diff --git a/.storybook/preview.tsx b/.storybook/preview.tsx index fcba92303..d66f0fb11 100644 --- a/.storybook/preview.tsx +++ b/.storybook/preview.tsx @@ -50,9 +50,13 @@ const preview: Preview = { locale: 'en', }, decorators: [ - (Story, { parameters, globals }) => { + (Story, { parameters, globals, args }) => { + // Get the locale from the global toolbar + // and merge it with any parameters or args state. const { locale } = globals as { locale: string }; const { state = {} } = parameters; + const { state: argsState = {} } = args; + const reducer = reducerWithInitialState( { meta: { @@ -60,7 +64,9 @@ const preview: Preview = { }, }, state as Record, + argsState as Record, ); + const store = configureStore({ reducer, middleware(getDefaultMiddleware) { diff --git a/app/javascript/mastodon/components/emoji/emoji.stories.tsx b/app/javascript/mastodon/components/emoji/emoji.stories.tsx new file mode 100644 index 000000000..a5e283158 --- /dev/null +++ b/app/javascript/mastodon/components/emoji/emoji.stories.tsx @@ -0,0 +1,56 @@ +import type { ComponentProps } from 'react'; + +import type { Meta, StoryObj } from '@storybook/react-vite'; + +import { importCustomEmojiData } from '@/mastodon/features/emoji/loader'; + +import { Emoji } from './index'; + +type EmojiProps = ComponentProps & { state: string }; + +const meta = { + title: 'Components/Emoji', + component: Emoji, + args: { + code: 'πŸ–€', + state: 'auto', + }, + argTypes: { + code: { + name: 'Emoji', + }, + state: { + control: { + type: 'select', + labels: { + auto: 'Auto', + native: 'Native', + twemoji: 'Twemoji', + }, + }, + options: ['auto', 'native', 'twemoji'], + name: 'Emoji Style', + mapping: { + auto: { meta: { emoji_style: 'auto' } }, + native: { meta: { emoji_style: 'native' } }, + twemoji: { meta: { emoji_style: 'twemoji' } }, + }, + }, + }, + render(args) { + void importCustomEmojiData(); + return ; + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +export const Default: Story = {}; + +export const CustomEmoji: Story = { + args: { + code: ':custom:', + }, +}; diff --git a/app/javascript/mastodon/components/emoji/html.tsx b/app/javascript/mastodon/components/emoji/html.tsx index b462a2ee6..628385f64 100644 --- a/app/javascript/mastodon/components/emoji/html.tsx +++ b/app/javascript/mastodon/components/emoji/html.tsx @@ -14,7 +14,7 @@ import { polymorphicForwardRef } from '@/types/polymorphic'; import { AnimateEmojiProvider, CustomEmojiProvider } from './context'; import { textToEmojis } from './index'; -interface EmojiHTMLProps { +export interface EmojiHTMLProps { htmlString: string; extraEmojis?: CustomEmojiMapArg; className?: string; diff --git a/app/javascript/mastodon/components/emoji/index.tsx b/app/javascript/mastodon/components/emoji/index.tsx index e070eb30d..0dff8314f 100644 --- a/app/javascript/mastodon/components/emoji/index.tsx +++ b/app/javascript/mastodon/components/emoji/index.tsx @@ -2,7 +2,7 @@ import type { FC } from 'react'; import { useContext, useEffect, useState } from 'react'; import { EMOJI_TYPE_CUSTOM } from '@/mastodon/features/emoji/constants'; -import { useEmojiAppState } from '@/mastodon/features/emoji/hooks'; +import { useEmojiAppState } from '@/mastodon/features/emoji/mode'; import { unicodeHexToUrl } from '@/mastodon/features/emoji/normalize'; import { isStateLoaded, diff --git a/app/javascript/mastodon/components/html_block/html_block.stories.tsx b/app/javascript/mastodon/components/html_block/html_block.stories.tsx index 9c104ba45..6fb3206df 100644 --- a/app/javascript/mastodon/components/html_block/html_block.stories.tsx +++ b/app/javascript/mastodon/components/html_block/html_block.stories.tsx @@ -7,23 +7,49 @@ const meta = { title: 'Components/HTMLBlock', component: HTMLBlock, args: { - contents: - '

Hello, world!

\n

A link

\n

This should be filtered out:

', + htmlString: `

Hello, world!

+

A link

+

This should be filtered out:

+

This also has emoji: πŸ–€

`, + }, + argTypes: { + extraEmojis: { + table: { + disable: true, + }, + }, + onElement: { + table: { + disable: true, + }, + }, + onAttribute: { + table: { + disable: true, + }, + }, }, render(args) { return ( // Just for visual clarity in Storybook. -
- -
+ /> ); }, + // Force Twemoji to demonstrate emoji rendering. + parameters: { + state: { + meta: { + emoji_style: 'twemoji', + }, + }, + }, } satisfies Meta; export default meta; diff --git a/app/javascript/mastodon/components/html_block/index.tsx b/app/javascript/mastodon/components/html_block/index.tsx index 51baea614..69fa1d9bf 100644 --- a/app/javascript/mastodon/components/html_block/index.tsx +++ b/app/javascript/mastodon/components/html_block/index.tsx @@ -1,50 +1,30 @@ -import type { FC, ReactNode } from 'react'; -import { useMemo } from 'react'; +import { useCallback } from 'react'; -import { cleanExtraEmojis } from '@/mastodon/features/emoji/normalize'; -import type { CustomEmojiMapArg } from '@/mastodon/features/emoji/types'; -import { createLimitedCache } from '@/mastodon/utils/cache'; +import type { OnElementHandler } from '@/mastodon/utils/html'; +import { polymorphicForwardRef } from '@/types/polymorphic'; -import { htmlStringToComponents } from '../../utils/html'; +import type { EmojiHTMLProps } from '../emoji/html'; +import { ModernEmojiHTML } from '../emoji/html'; +import { useElementHandledLink } from '../status/handled_link'; -// Use a module-level cache to avoid re-rendering the same HTML multiple times. -const cache = createLimitedCache({ maxSize: 1000 }); - -interface HTMLBlockProps { - contents: string; - extraEmojis?: CustomEmojiMapArg; -} - -export const HTMLBlock: FC = ({ - contents: raw, - extraEmojis, -}) => { - const customEmojis = useMemo( - () => cleanExtraEmojis(extraEmojis), - [extraEmojis], - ); - const contents = useMemo(() => { - const key = JSON.stringify({ raw, customEmojis }); - if (cache.has(key)) { - return cache.get(key); - } - - const rendered = htmlStringToComponents(raw, { - onText, - extraArgs: { customEmojis }, +export const HTMLBlock = polymorphicForwardRef< + 'div', + EmojiHTMLProps & Parameters[0] +>( + ({ + onElement: onParentElement, + hrefToMention, + hashtagAccountId, + ...props + }) => { + const { onElement: onLinkElement } = useElementHandledLink({ + hrefToMention, + hashtagAccountId, }); - - cache.set(key, rendered); - return rendered; - }, [raw, customEmojis]); - - return contents; -}; - -function onText( - text: string, - // eslint-disable-next-line @typescript-eslint/no-unused-vars -- Doesn't do anything, just showing how typing would work. - { customEmojis }: { customEmojis: CustomEmojiMapArg | null }, -) { - return text; -} + const onElement: OnElementHandler = useCallback( + (...args) => onParentElement?.(...args) ?? onLinkElement(...args), + [onLinkElement, onParentElement], + ); + return ; + }, +); diff --git a/app/javascript/mastodon/features/emoji/emoji_picker.tsx b/app/javascript/mastodon/features/emoji/emoji_picker.tsx index 6dcfe37ac..f9367c775 100644 --- a/app/javascript/mastodon/features/emoji/emoji_picker.tsx +++ b/app/javascript/mastodon/features/emoji/emoji_picker.tsx @@ -7,7 +7,7 @@ import { assetHost } from 'mastodon/utils/config'; import { EMOJI_MODE_NATIVE } from './constants'; import EmojiData from './emoji_data.json'; -import { useEmojiAppState } from './hooks'; +import { useEmojiAppState } from './mode'; const backgroundImageFnDefault = () => `${assetHost}/emoji/sheet_15_1.png`; diff --git a/app/javascript/mastodon/features/emoji/hooks.ts b/app/javascript/mastodon/features/emoji/hooks.ts deleted file mode 100644 index dbd20d304..000000000 --- a/app/javascript/mastodon/features/emoji/hooks.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { useCallback, useLayoutEffect, useMemo, useState } from 'react'; - -import { createAppSelector, useAppSelector } from '@/mastodon/store'; -import { isModernEmojiEnabled } from '@/mastodon/utils/environment'; - -import { toSupportedLocale } from './locale'; -import { determineEmojiMode } from './mode'; -import { cleanExtraEmojis } from './normalize'; -import { emojifyElement, emojifyText } from './render'; -import type { CustomEmojiMapArg, EmojiAppState } from './types'; -import { stringHasAnyEmoji } from './utils'; - -interface UseEmojifyOptions { - text: string; - extraEmojis?: CustomEmojiMapArg; - deep?: boolean; -} - -export function useEmojify({ - text, - extraEmojis, - deep = true, -}: UseEmojifyOptions) { - const [emojifiedText, setEmojifiedText] = useState(null); - - const appState = useEmojiAppState(); - const extra = useMemo(() => cleanExtraEmojis(extraEmojis), [extraEmojis]); - - const emojify = useCallback( - async (input: string) => { - let result: string | null = null; - if (deep) { - const wrapper = document.createElement('div'); - wrapper.innerHTML = input; - if (await emojifyElement(wrapper, appState, extra ?? {})) { - result = wrapper.innerHTML; - } - } else { - result = await emojifyText(text, appState, extra ?? {}); - } - if (result) { - setEmojifiedText(result); - } else { - setEmojifiedText(input); - } - }, - [appState, deep, extra, text], - ); - useLayoutEffect(() => { - if (isModernEmojiEnabled() && !!text.trim() && stringHasAnyEmoji(text)) { - void emojify(text); - } else { - // If no emoji or we don't want to render, fall back. - setEmojifiedText(text); - } - }, [emojify, text]); - - return emojifiedText; -} - -const modeSelector = createAppSelector( - [(state) => state.meta.get('emoji_style') as string], - (emoji_style) => determineEmojiMode(emoji_style), -); - -export function useEmojiAppState(): EmojiAppState { - const locale = useAppSelector((state) => - toSupportedLocale(state.meta.get('locale') as string), - ); - const mode = useAppSelector(modeSelector); - - return { - currentLocale: locale, - locales: [locale], - mode, - darkTheme: document.body.classList.contains('theme-default'), - }; -} diff --git a/app/javascript/mastodon/features/emoji/index.ts b/app/javascript/mastodon/features/emoji/index.ts index d128da6b5..3701ad676 100644 --- a/app/javascript/mastodon/features/emoji/index.ts +++ b/app/javascript/mastodon/features/emoji/index.ts @@ -10,6 +10,8 @@ let worker: Worker | null = null; const log = emojiLogger('index'); +const WORKER_TIMEOUT = 1_000; // 1 second + export function initializeEmoji() { log('initializing emojis'); if (!worker && 'Worker' in window) { @@ -29,7 +31,7 @@ export function initializeEmoji() { log('worker is not ready after timeout'); worker = null; void fallbackLoad(); - }, 500); + }, WORKER_TIMEOUT); thisWorker.addEventListener('message', (event: MessageEvent) => { const { data: message } = event; if (message === 'ready') { diff --git a/app/javascript/mastodon/features/emoji/mode.ts b/app/javascript/mastodon/features/emoji/mode.ts index 0f581d8b5..afb8a78eb 100644 --- a/app/javascript/mastodon/features/emoji/mode.ts +++ b/app/javascript/mastodon/features/emoji/mode.ts @@ -1,6 +1,7 @@ // Credit to Nolan Lawson for the original implementation. // See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js +import { createAppSelector, useAppSelector } from '@/mastodon/store'; import { isDevelopment } from '@/mastodon/utils/environment'; import { @@ -8,7 +9,27 @@ import { EMOJI_MODE_NATIVE_WITH_FLAGS, EMOJI_MODE_TWEMOJI, } from './constants'; -import type { EmojiMode } from './types'; +import { toSupportedLocale } from './locale'; +import type { EmojiAppState, EmojiMode } from './types'; + +const modeSelector = createAppSelector( + [(state) => state.meta.get('emoji_style') as string], + (emoji_style) => determineEmojiMode(emoji_style), +); + +export function useEmojiAppState(): EmojiAppState { + const locale = useAppSelector((state) => + toSupportedLocale(state.meta.get('locale') as string), + ); + const mode = useAppSelector(modeSelector); + + return { + currentLocale: locale, + locales: [locale], + mode, + darkTheme: document.body.classList.contains('theme-default'), + }; +} type Feature = Uint8ClampedArray; diff --git a/app/javascript/mastodon/features/emoji/render.test.ts b/app/javascript/mastodon/features/emoji/render.test.ts index 108cf7475..05dbc388c 100644 --- a/app/javascript/mastodon/features/emoji/render.test.ts +++ b/app/javascript/mastodon/features/emoji/render.test.ts @@ -1,101 +1,12 @@ import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories'; -import { EMOJI_MODE_TWEMOJI } from './constants'; import * as db from './database'; +import * as loader from './loader'; import { - emojifyElement, - emojifyText, - testCacheClear, + loadEmojiDataToState, + stringToEmojiState, tokenizeText, } from './render'; -import type { EmojiAppState } from './types'; - -function mockDatabase() { - return { - searchCustomEmojisByShortcodes: vi - .spyOn(db, 'searchCustomEmojisByShortcodes') - .mockResolvedValue([customEmojiFactory()]), - searchEmojisByHexcodes: vi - .spyOn(db, 'searchEmojisByHexcodes') - .mockResolvedValue([ - unicodeEmojiFactory({ - hexcode: '1F60A', - label: 'smiling face with smiling eyes', - unicode: '😊', - }), - unicodeEmojiFactory({ - hexcode: '1F1EA-1F1FA', - label: 'flag-eu', - unicode: 'πŸ‡ͺπŸ‡Ί', - }), - ]), - }; -} - -const expectedSmileImage = - '😊'; -const expectedFlagImage = - 'πŸ‡ͺπŸ‡Ί'; - -function testAppState(state: Partial = {}) { - return { - locales: ['en'], - mode: EMOJI_MODE_TWEMOJI, - currentLocale: 'en', - darkTheme: false, - ...state, - } satisfies EmojiAppState; -} - -describe('emojifyElement', () => { - function testElement(text = '

Hello 😊πŸ‡ͺπŸ‡Ί!

:custom:

') { - const testElement = document.createElement('div'); - testElement.innerHTML = text; - return testElement; - } - - afterEach(() => { - testCacheClear(); - vi.restoreAllMocks(); - }); - - test('caches element rendering results', async () => { - const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } = - mockDatabase(); - await emojifyElement(testElement(), testAppState()); - await emojifyElement(testElement(), testAppState()); - await emojifyElement(testElement(), testAppState()); - expect(searchEmojisByHexcodes).toHaveBeenCalledExactlyOnceWith( - ['1F1EA-1F1FA', '1F60A'], - 'en', - ); - expect(searchCustomEmojisByShortcodes).toHaveBeenCalledExactlyOnceWith([ - ':custom:', - ]); - }); - - test('returns null when no emoji are found', async () => { - mockDatabase(); - const actual = await emojifyElement( - testElement('

here is just text :)

'), - testAppState(), - ); - expect(actual).toBeNull(); - }); -}); - -describe('emojifyText', () => { - test('returns original input when no emoji are in string', async () => { - const actual = await emojifyText('nothing here', testAppState()); - expect(actual).toBe('nothing here'); - }); - - test('renders Unicode emojis to twemojis', async () => { - mockDatabase(); - const actual = await emojifyText('Hello 😊πŸ‡ͺπŸ‡Ί!', testAppState()); - expect(actual).toBe(`Hello ${expectedSmileImage}${expectedFlagImage}!`); - }); -}); describe('tokenizeText', () => { test('returns an array of text to be a single token', () => { @@ -162,3 +73,106 @@ describe('tokenizeText', () => { ]); }); }); + +describe('stringToEmojiState', () => { + test('returns unicode emoji state for valid unicode emoji', () => { + expect(stringToEmojiState('😊')).toEqual({ + type: 'unicode', + code: '1F60A', + }); + }); + + test('returns custom emoji state for valid custom emoji', () => { + expect(stringToEmojiState(':smile:')).toEqual({ + type: 'custom', + code: 'smile', + data: undefined, + }); + }); + + test('returns custom emoji state with data when provided', () => { + const customEmoji = { + smile: customEmojiFactory({ + shortcode: 'smile', + url: 'https://example.com/smile.png', + static_url: 'https://example.com/smile_static.png', + }), + }; + expect(stringToEmojiState(':smile:', customEmoji)).toEqual({ + type: 'custom', + code: 'smile', + data: customEmoji.smile, + }); + }); + + test('returns null for invalid emoji strings', () => { + expect(stringToEmojiState('notanemoji')).toBeNull(); + expect(stringToEmojiState(':invalid-emoji:')).toBeNull(); + }); +}); + +describe('loadEmojiDataToState', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + test('loads unicode data into state', async () => { + const dbCall = vi + .spyOn(db, 'loadEmojiByHexcode') + .mockResolvedValue(unicodeEmojiFactory()); + const unicodeState = { type: 'unicode', code: '1F60A' } as const; + const result = await loadEmojiDataToState(unicodeState, 'en'); + expect(dbCall).toHaveBeenCalledWith('1F60A', 'en'); + expect(result).toEqual({ + type: 'unicode', + code: '1F60A', + data: unicodeEmojiFactory(), + }); + }); + + test('loads custom emoji data into state', async () => { + const dbCall = vi + .spyOn(db, 'loadCustomEmojiByShortcode') + .mockResolvedValueOnce(customEmojiFactory()); + const customState = { type: 'custom', code: 'smile' } as const; + const result = await loadEmojiDataToState(customState, 'en'); + expect(dbCall).toHaveBeenCalledWith('smile'); + expect(result).toEqual({ + type: 'custom', + code: 'smile', + data: customEmojiFactory(), + }); + }); + + test('returns null if unicode emoji not found in database', async () => { + vi.spyOn(db, 'loadEmojiByHexcode').mockResolvedValueOnce(undefined); + const unicodeState = { type: 'unicode', code: '1F60A' } as const; + const result = await loadEmojiDataToState(unicodeState, 'en'); + expect(result).toBeNull(); + }); + + test('returns null if custom emoji not found in database', async () => { + vi.spyOn(db, 'loadCustomEmojiByShortcode').mockResolvedValueOnce(undefined); + const customState = { type: 'custom', code: 'smile' } as const; + const result = await loadEmojiDataToState(customState, 'en'); + expect(result).toBeNull(); + }); + + test('retries loading emoji data once if initial load fails', async () => { + const dbCall = vi + .spyOn(db, 'loadEmojiByHexcode') + .mockRejectedValue(new db.LocaleNotLoadedError('en')); + vi.spyOn(loader, 'importEmojiData').mockResolvedValueOnce(); + const consoleCall = vi + .spyOn(console, 'warn') + .mockImplementationOnce(() => null); + + const unicodeState = { type: 'unicode', code: '1F60A' } as const; + const result = await loadEmojiDataToState(unicodeState, 'en'); + + expect(dbCall).toHaveBeenCalledTimes(2); + expect(loader.importEmojiData).toHaveBeenCalledWith('en'); + expect(consoleCall).toHaveBeenCalled(); + expect(result).toBeNull(); + }); +}); diff --git a/app/javascript/mastodon/features/emoji/render.ts b/app/javascript/mastodon/features/emoji/render.ts index e0c8fd8dc..574d5ef59 100644 --- a/app/javascript/mastodon/features/emoji/render.ts +++ b/app/javascript/mastodon/features/emoji/render.ts @@ -1,7 +1,3 @@ -import { autoPlayGif } from '@/mastodon/initial_state'; -import { createLimitedCache } from '@/mastodon/utils/cache'; -import * as perf from '@/mastodon/utils/performance'; - import { EMOJI_MODE_NATIVE, EMOJI_MODE_NATIVE_WITH_FLAGS, @@ -12,33 +8,69 @@ import { loadCustomEmojiByShortcode, loadEmojiByHexcode, LocaleNotLoadedError, - searchCustomEmojisByShortcodes, - searchEmojisByHexcodes, } from './database'; import { importEmojiData } from './loader'; -import { emojiToUnicodeHex, unicodeHexToUrl } from './normalize'; +import { emojiToUnicodeHex } from './normalize'; import type { - EmojiAppState, EmojiLoadedState, EmojiMode, EmojiState, EmojiStateCustom, - EmojiStateMap, EmojiStateUnicode, ExtraCustomEmojiMap, - LocaleOrCustom, } from './types'; import { anyEmojiRegex, emojiLogger, isCustomEmoji, isUnicodeEmoji, - stringHasAnyEmoji, stringHasUnicodeFlags, } from './utils'; const log = emojiLogger('render'); +type TokenizedText = (string | EmojiState)[]; + +/** + * Tokenizes text into strings and emoji states. + * @param text Text to tokenize. + * @returns Array of strings and emoji states. + */ +export function tokenizeText(text: string): TokenizedText { + if (!text.trim()) { + return [text]; + } + + const tokens = []; + let lastIndex = 0; + for (const match of text.matchAll(anyEmojiRegex())) { + if (match.index > lastIndex) { + tokens.push(text.slice(lastIndex, match.index)); + } + + const code = match[0]; + + if (code.startsWith(':') && code.endsWith(':')) { + // Custom emoji + tokens.push({ + type: EMOJI_TYPE_CUSTOM, + code, + } satisfies EmojiStateCustom); + } else { + // Unicode emoji + tokens.push({ + type: EMOJI_TYPE_UNICODE, + code: code, + } satisfies EmojiStateUnicode); + } + lastIndex = match.index + code.length; + } + if (lastIndex < text.length) { + tokens.push(text.slice(lastIndex)); + } + return tokens; +} + /** * Parses emoji string to extract emoji state. * @param code Hex code or custom shortcode. @@ -132,305 +164,19 @@ export function isStateLoaded(state: EmojiState): state is EmojiLoadedState { } /** - * Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions. + * Determines if the given token should be rendered as an image based on the emoji mode. + * @param state Emoji state to parse. + * @param mode Rendering mode. + * @returns Whether to render as an image. */ -export async function emojifyElement( - element: Element, - appState: EmojiAppState, - extraEmojis: ExtraCustomEmojiMap = {}, -): Promise { - const cacheKey = createCacheKey(element, appState, extraEmojis); - const cached = getCached(cacheKey); - if (cached !== undefined) { - log('Cache hit on %s', element.outerHTML); - if (cached === null) { - return null; - } - element.innerHTML = cached; - return element; - } - if (!stringHasAnyEmoji(element.innerHTML)) { - updateCache(cacheKey, null); - return null; - } - perf.start('emojifyElement()'); - const queue: (HTMLElement | Text)[] = [element]; - while (queue.length > 0) { - const current = queue.shift(); - if ( - !current || - current instanceof HTMLScriptElement || - current instanceof HTMLStyleElement - ) { - continue; - } - - if ( - current.textContent && - (current instanceof Text || !current.hasChildNodes()) - ) { - const renderedContent = await textToElementArray( - current.textContent, - appState, - extraEmojis, - ); - if (renderedContent) { - if (!(current instanceof Text)) { - current.textContent = null; // Clear the text content if it's not a Text node. - } - current.replaceWith(renderedToHTML(renderedContent)); - } - continue; - } - - for (const child of current.childNodes) { - if (child instanceof HTMLElement || child instanceof Text) { - queue.push(child); - } - } - } - updateCache(cacheKey, element.innerHTML); - perf.stop('emojifyElement()'); - return element; -} - -export async function emojifyText( - text: string, - appState: EmojiAppState, - extraEmojis: ExtraCustomEmojiMap = {}, -): Promise { - const cacheKey = createCacheKey(text, appState, extraEmojis); - const cached = getCached(cacheKey); - if (cached !== undefined) { - log('Cache hit on %s', text); - return cached ?? text; - } - if (!stringHasAnyEmoji(text)) { - updateCache(cacheKey, null); - return text; - } - const eleArray = await textToElementArray(text, appState, extraEmojis); - if (!eleArray) { - updateCache(cacheKey, null); - return text; - } - const rendered = renderedToHTML(eleArray, document.createElement('div')); - updateCache(cacheKey, rendered.innerHTML); - return rendered.innerHTML; -} - -// Private functions - -const { - set: updateCache, - get: getCached, - clear: cacheClear, -} = createLimitedCache({ log: log.extend('cache') }); - -function createCacheKey( - input: HTMLElement | string, - appState: EmojiAppState, - extraEmojis: ExtraCustomEmojiMap, -) { - return JSON.stringify([ - input instanceof HTMLElement ? input.outerHTML : input, - appState, - extraEmojis, - ]); -} - -type EmojifiedTextArray = (string | HTMLImageElement)[]; - -async function textToElementArray( - text: string, - appState: EmojiAppState, - extraEmojis: ExtraCustomEmojiMap = {}, -): Promise { - // Exit if no text to convert. - if (!text.trim()) { - return null; - } - - const tokens = tokenizeText(text); - - // If only one token and it's a string, exit early. - if (tokens.length === 1 && typeof tokens[0] === 'string') { - return null; - } - - // Get all emoji from the state map, loading any missing ones. - await loadMissingEmojiIntoCache(tokens, appState, extraEmojis); - - const renderedFragments: EmojifiedTextArray = []; - for (const token of tokens) { - if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) { - let state: EmojiState | undefined; - if (token.type === EMOJI_TYPE_CUSTOM) { - const extraEmojiData = extraEmojis[token.code]; - if (extraEmojiData) { - state = { - type: EMOJI_TYPE_CUSTOM, - data: extraEmojiData, - code: token.code, - }; - } else { - state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM); - } - } else { - state = emojiForLocale( - emojiToUnicodeHex(token.code), - appState.currentLocale, - ); - } - - // If the state is valid, create an image element. Otherwise, just append as text. - if (state && typeof state !== 'string' && isStateLoaded(state)) { - const image = stateToImage(state, appState); - renderedFragments.push(image); - continue; - } - } - const text = typeof token === 'string' ? token : token.code; - renderedFragments.push(text); - } - - return renderedFragments; -} - -type TokenizedText = (string | EmojiState)[]; - -export function tokenizeText(text: string): TokenizedText { - if (!text.trim()) { - return [text]; - } - - const tokens = []; - let lastIndex = 0; - for (const match of text.matchAll(anyEmojiRegex())) { - if (match.index > lastIndex) { - tokens.push(text.slice(lastIndex, match.index)); - } - - const code = match[0]; - - if (code.startsWith(':') && code.endsWith(':')) { - // Custom emoji - tokens.push({ - type: EMOJI_TYPE_CUSTOM, - code, - } satisfies EmojiStateCustom); - } else { - // Unicode emoji - tokens.push({ - type: EMOJI_TYPE_UNICODE, - code: code, - } satisfies EmojiStateUnicode); - } - lastIndex = match.index + code.length; - } - if (lastIndex < text.length) { - tokens.push(text.slice(lastIndex)); - } - return tokens; -} - -const localeCacheMap = new Map([ - [ - EMOJI_TYPE_CUSTOM, - createLimitedCache({ log: log.extend('custom') }), - ], -]); - -function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap { - return ( - localeCacheMap.get(locale) ?? - createLimitedCache({ log: log.extend(locale) }) - ); -} - -function emojiForLocale( - code: string, - locale: LocaleOrCustom, -): EmojiState | undefined { - const cache = cacheForLocale(locale); - return cache.get(code); -} - -async function loadMissingEmojiIntoCache( - tokens: TokenizedText, - { mode, currentLocale }: EmojiAppState, - extraEmojis: ExtraCustomEmojiMap, -) { - const missingUnicodeEmoji = new Set(); - const missingCustomEmoji = new Set(); - - // Iterate over tokens and check if they are in the cache already. - for (const token of tokens) { - if (typeof token === 'string') { - continue; // Skip plain strings. - } - - // If this is a custom emoji, check it separately. - if (token.type === EMOJI_TYPE_CUSTOM) { - const code = token.code; - if (code in extraEmojis) { - continue; // We don't care about extra emoji. - } - const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM); - if (!emojiState) { - missingCustomEmoji.add(code); - } - // Otherwise this is a unicode emoji, so check it against all locales. - } else if (shouldRenderImage(token, mode)) { - const code = emojiToUnicodeHex(token.code); - if (missingUnicodeEmoji.has(code)) { - continue; // Already marked as missing. - } - const emojiState = emojiForLocale(code, currentLocale); - if (!emojiState) { - // If it's missing in one locale, we consider it missing for all. - missingUnicodeEmoji.add(code); - } - } - } - - if (missingUnicodeEmoji.size > 0) { - const missingEmojis = Array.from(missingUnicodeEmoji).toSorted(); - const emojis = await searchEmojisByHexcodes(missingEmojis, currentLocale); - const cache = cacheForLocale(currentLocale); - for (const emoji of emojis) { - cache.set(emoji.hexcode, { - type: EMOJI_TYPE_UNICODE, - data: emoji, - code: emoji.hexcode, - }); - } - localeCacheMap.set(currentLocale, cache); - } - - if (missingCustomEmoji.size > 0) { - const missingEmojis = Array.from(missingCustomEmoji).toSorted(); - const emojis = await searchCustomEmojisByShortcodes(missingEmojis); - const cache = cacheForLocale(EMOJI_TYPE_CUSTOM); - for (const emoji of emojis) { - cache.set(emoji.shortcode, { - type: EMOJI_TYPE_CUSTOM, - data: emoji, - code: emoji.shortcode, - }); - } - localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache); - } -} - -export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean { - if (token.type === EMOJI_TYPE_UNICODE) { +export function shouldRenderImage(state: EmojiState, mode: EmojiMode): boolean { + if (state.type === EMOJI_TYPE_UNICODE) { // If the mode is native or native with flags for non-flag emoji // we can just append the text node directly. if ( mode === EMOJI_MODE_NATIVE || (mode === EMOJI_MODE_NATIVE_WITH_FLAGS && - !stringHasUnicodeFlags(token.code)) + !stringHasUnicodeFlags(state.code)) ) { return false; } @@ -438,52 +184,3 @@ export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean { return true; } - -function stateToImage(state: EmojiLoadedState, appState: EmojiAppState) { - const image = document.createElement('img'); - image.draggable = false; - image.classList.add('emojione'); - - if (state.type === EMOJI_TYPE_UNICODE) { - image.alt = state.data.unicode; - image.title = state.data.label; - image.src = unicodeHexToUrl(state.data.hexcode, appState.darkTheme); - } else { - // Custom emoji - const shortCode = `:${state.data.shortcode}:`; - image.classList.add('custom-emoji'); - image.alt = shortCode; - image.title = shortCode; - image.src = autoPlayGif ? state.data.url : state.data.static_url; - image.dataset.original = state.data.url; - image.dataset.static = state.data.static_url; - } - - return image; -} - -function renderedToHTML(renderedArray: EmojifiedTextArray): DocumentFragment; -function renderedToHTML( - renderedArray: EmojifiedTextArray, - parent: ParentType, -): ParentType; -function renderedToHTML( - renderedArray: EmojifiedTextArray, - parent: ParentNode | null = null, -) { - const fragment = parent ?? document.createDocumentFragment(); - for (const fragmentItem of renderedArray) { - if (typeof fragmentItem === 'string') { - fragment.appendChild(document.createTextNode(fragmentItem)); - } else if (fragmentItem instanceof HTMLImageElement) { - fragment.appendChild(fragmentItem); - } - } - return fragment; -} - -// Testing helpers -export const testCacheClear = () => { - cacheClear(); - localeCacheMap.clear(); -}; diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index b55cefb0a..8cd0902aa 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -4,7 +4,6 @@ import type { FlatCompactEmoji, Locale } from 'emojibase'; import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; -import type { LimitedCache } from '@/mastodon/utils/cache'; import type { EMOJI_MODE_NATIVE, @@ -48,12 +47,11 @@ export interface EmojiStateCustom { data?: CustomEmojiRenderFields; } export type EmojiState = EmojiStateUnicode | EmojiStateCustom; + export type EmojiLoadedState = | Required | Required; -export type EmojiStateMap = LimitedCache; - export type CustomEmojiMapArg = | ExtraCustomEmojiMap | ImmutableList @@ -64,9 +62,3 @@ export type ExtraCustomEmojiMap = Record< string, Pick >; - -export interface TwemojiBorderInfo { - hexCode: string; - hasLightBorder: boolean; - hasDarkBorder: boolean; -} diff --git a/app/javascript/mastodon/features/emoji/utils.test.ts b/app/javascript/mastodon/features/emoji/utils.test.ts index b9062294c..3844c814a 100644 --- a/app/javascript/mastodon/features/emoji/utils.test.ts +++ b/app/javascript/mastodon/features/emoji/utils.test.ts @@ -1,35 +1,31 @@ -import { - stringHasAnyEmoji, - stringHasCustomEmoji, - stringHasUnicodeEmoji, - stringHasUnicodeFlags, -} from './utils'; +import { isCustomEmoji, isUnicodeEmoji, stringHasUnicodeFlags } from './utils'; -describe('stringHasUnicodeEmoji', () => { +describe('isUnicodeEmoji', () => { test.concurrent.for([ - ['only text', false], - ['text with non-emoji symbols β„’Β©', false], - ['text with emoji πŸ˜€', true], - ['multiple emojis πŸ˜€πŸ˜ƒπŸ˜„', true], - ['emoji with skin tone πŸ‘πŸ½', true], - ['emoji with ZWJ πŸ‘©β€β€οΈβ€πŸ‘¨', true], - ['emoji with variation selector ✊️', true], - ['emoji with keycap 1️⃣', true], - ['emoji with flags πŸ‡ΊπŸ‡Έ', true], - ['emoji with regional indicators πŸ‡¦πŸ‡Ί', true], - ['emoji with gender πŸ‘©β€βš•οΈ', true], - ['emoji with family πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], - ['emoji with zero width joiner πŸ‘©β€πŸ”¬', true], - ['emoji with non-BMP codepoint πŸ§‘β€πŸš€', true], - ['emoji with combining marks πŸ‘¨β€πŸ‘©β€πŸ‘§β€πŸ‘¦', true], - ['emoji with enclosing keycap #️⃣', true], - ['emoji with no visible glyph \u200D', false], - ] as const)( - 'stringHasUnicodeEmoji has emojis in "%s": %o', - ([text, expected], { expect }) => { - expect(stringHasUnicodeEmoji(text)).toBe(expected); - }, - ); + ['😊', true], + ['πŸ‡ΏπŸ‡Ό', true], + ['πŸ΄β€β˜ οΈ', true], + ['πŸ³οΈβ€πŸŒˆ', true], + ['foo', false], + [':smile:', false], + ['😊foo', false], + ] as const)('isUnicodeEmoji("%s") is %o', ([input, expected], { expect }) => { + expect(isUnicodeEmoji(input)).toBe(expected); + }); +}); + +describe('isCustomEmoji', () => { + test.concurrent.for([ + [':smile:', true], + [':smile_123:', true], + [':SMILE:', true], + ['😊', false], + ['foo', false], + [':smile', false], + ['smile:', false], + ] as const)('isCustomEmoji("%s") is %o', ([input, expected], { expect }) => { + expect(isCustomEmoji(input)).toBe(expected); + }); }); describe('stringHasUnicodeFlags', () => { @@ -51,27 +47,3 @@ describe('stringHasUnicodeFlags', () => { }, ); }); - -describe('stringHasCustomEmoji', () => { - test('string with custom emoji returns true', () => { - expect(stringHasCustomEmoji(':custom: :test:')).toBeTruthy(); - }); - test('string without custom emoji returns false', () => { - expect(stringHasCustomEmoji('πŸ³οΈβ€πŸŒˆ :πŸ³οΈβ€πŸŒˆ: text β„’')).toBeFalsy(); - }); -}); - -describe('stringHasAnyEmoji', () => { - test('string without any emoji or characters', () => { - expect(stringHasAnyEmoji('normal text. 12356?!')).toBeFalsy(); - }); - test('string with non-emoji characters', () => { - expect(stringHasAnyEmoji('β„’Β©')).toBeFalsy(); - }); - test('has unicode emoji', () => { - expect(stringHasAnyEmoji('πŸ³οΈβ€πŸŒˆπŸ”₯πŸ‡ΈπŸ‡Ή πŸ‘©β€πŸ”¬')).toBeTruthy(); - }); - test('has custom emoji', () => { - expect(stringHasAnyEmoji(':test: :custom:')).toBeTruthy(); - }); -}); diff --git a/app/javascript/mastodon/features/emoji/utils.ts b/app/javascript/mastodon/features/emoji/utils.ts index e811565c2..c567afc2c 100644 --- a/app/javascript/mastodon/features/emoji/utils.ts +++ b/app/javascript/mastodon/features/emoji/utils.ts @@ -6,10 +6,6 @@ export function emojiLogger(segment: string) { return debug(`emojis:${segment}`); } -export function stringHasUnicodeEmoji(input: string): boolean { - return new RegExp(EMOJI_REGEX, supportedFlags()).test(input); -} - export function isUnicodeEmoji(input: string): boolean { return ( input.length > 0 && @@ -34,19 +30,13 @@ export function stringHasUnicodeFlags(input: string): boolean { // Constant as this is supported by all browsers. const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i; +// Use the polyfill regex or the Unicode property escapes if supported. +const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}'; export function isCustomEmoji(input: string): boolean { return new RegExp(`^${CUSTOM_EMOJI_REGEX.source}$`, 'i').test(input); } -export function stringHasCustomEmoji(input: string) { - return CUSTOM_EMOJI_REGEX.test(input); -} - -export function stringHasAnyEmoji(input: string) { - return stringHasUnicodeEmoji(input) || stringHasCustomEmoji(input); -} - export function anyEmojiRegex() { return new RegExp( `${EMOJI_REGEX}|${CUSTOM_EMOJI_REGEX.source}`, @@ -64,5 +54,3 @@ function supportedFlags(flags = '') { } return flags; } - -const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}'; diff --git a/app/javascript/testing/api.ts b/app/javascript/testing/api.ts index 4948d7199..dd45b7e7b 100644 --- a/app/javascript/testing/api.ts +++ b/app/javascript/testing/api.ts @@ -1,7 +1,10 @@ +import type { CompactEmoji } from 'emojibase'; import { http, HttpResponse } from 'msw'; import { action } from 'storybook/actions'; -import { relationshipsFactory } from './factories'; +import { toSupportedLocale } from '@/mastodon/features/emoji/locale'; + +import { customEmojiFactory, relationshipsFactory } from './factories'; export const mockHandlers = { mute: http.post<{ id: string }>('/api/v1/accounts/:id/mute', ({ params }) => { @@ -40,6 +43,24 @@ export const mockHandlers = { ); }, ), + emojiCustomData: http.get('/api/v1/custom_emojis', () => { + action('fetching custom emoji data')(); + return HttpResponse.json([customEmojiFactory()]); + }), + emojiData: http.get<{ locale: string }>( + '/packs-dev/emoji/:locale.json', + async ({ params }) => { + const locale = toSupportedLocale(params.locale); + action('fetching emoji data')(locale); + const { default: data } = (await import( + `emojibase-data/${locale}/compact.json` + )) as { + default: CompactEmoji[]; + }; + + return HttpResponse.json([data]); + }, + ), }; export const unhandledRequestHandler = ({ url }: Request) => {