Emoji: Cleanup new code (#36402)

.storybook/preview.tsx

@@ -50,9 +50,13 @@ const preview: Preview = {
     locale: 'en',
   },
   decorators: [
-    (Story, { parameters, globals }) => {
+    (Story, { parameters, globals, args }) => {
+      // Get the locale from the global toolbar
+      // and merge it with any parameters or args state.
       const { locale } = globals as { locale: string };
       const { state = {} } = parameters;
+      const { state: argsState = {} } = args;
+
       const reducer = reducerWithInitialState(
         {
           meta: {
@@ -60,7 +64,9 @@ const preview: Preview = {
           },
         },
         state as Record<string, unknown>,
+        argsState as Record<string, unknown>,
       );
+
       const store = configureStore({
         reducer,
         middleware(getDefaultMiddleware) {

app/javascript/mastodon/components/emoji/emoji.stories.tsx

@@ -0,0 +1,56 @@
+import type { ComponentProps } from 'react';
+
+import type { Meta, StoryObj } from '@storybook/react-vite';
+
+import { importCustomEmojiData } from '@/mastodon/features/emoji/loader';
+
+import { Emoji } from './index';
+
+type EmojiProps = ComponentProps<typeof Emoji> & { state: string };
+
+const meta = {
+  title: 'Components/Emoji',
+  component: Emoji,
+  args: {
+    code: '🖤',
+    state: 'auto',
+  },
+  argTypes: {
+    code: {
+      name: 'Emoji',
+    },
+    state: {
+      control: {
+        type: 'select',
+        labels: {
+          auto: 'Auto',
+          native: 'Native',
+          twemoji: 'Twemoji',
+        },
+      },
+      options: ['auto', 'native', 'twemoji'],
+      name: 'Emoji Style',
+      mapping: {
+        auto: { meta: { emoji_style: 'auto' } },
+        native: { meta: { emoji_style: 'native' } },
+        twemoji: { meta: { emoji_style: 'twemoji' } },
+      },
+    },
+  },
+  render(args) {
+    void importCustomEmojiData();
+    return <Emoji {...args} />;
+  },
+} satisfies Meta<EmojiProps>;
+
+export default meta;
+
+type Story = StoryObj<typeof meta>;
+
+export const Default: Story = {};
+
+export const CustomEmoji: Story = {
+  args: {
+    code: ':custom:',
+  },
+};

app/javascript/mastodon/components/emoji/html.tsx

@@ -14,7 +14,7 @@ import { polymorphicForwardRef } from '@/types/polymorphic';
 import { AnimateEmojiProvider, CustomEmojiProvider } from './context';
 import { textToEmojis } from './index';
 
-interface EmojiHTMLProps {
+export interface EmojiHTMLProps {
   htmlString: string;
   extraEmojis?: CustomEmojiMapArg;
   className?: string;

app/javascript/mastodon/components/emoji/index.tsx

@@ -2,7 +2,7 @@ import type { FC } from 'react';
 import { useContext, useEffect, useState } from 'react';
 
 import { EMOJI_TYPE_CUSTOM } from '@/mastodon/features/emoji/constants';
-import { useEmojiAppState } from '@/mastodon/features/emoji/hooks';
+import { useEmojiAppState } from '@/mastodon/features/emoji/mode';
 import { unicodeHexToUrl } from '@/mastodon/features/emoji/normalize';
 import {
   isStateLoaded,

app/javascript/mastodon/components/html_block/html_block.stories.tsx

@@ -7,23 +7,49 @@ const meta = {
   title: 'Components/HTMLBlock',
   component: HTMLBlock,
   args: {
-    contents:
+    htmlString: `<p>Hello, world!</p>
-      '<p>Hello, world!</p>\n<p><a href="#">A link</a></p>\n<p>This should be filtered out: <button>Bye!</button></p>',
+<p><a href="#">A link</a></p>
+<p>This should be filtered out: <button>Bye!</button></p>
+<p>This also has emoji: 🖤</p>`,
+  },
+  argTypes: {
+    extraEmojis: {
+      table: {
+        disable: true,
+      },
+    },
+    onElement: {
+      table: {
+        disable: true,
+      },
+    },
+    onAttribute: {
+      table: {
+        disable: true,
+      },
+    },
   },
   render(args) {
     return (
       // Just for visual clarity in Storybook.
-      <div
+      <HTMLBlock
+        {...args}
         style={{
           border: '1px solid black',
           padding: '1rem',
           minWidth: '300px',
         }}
-      >
+      />
-        <HTMLBlock {...args} />
-      </div>
     );
   },
+  // Force Twemoji to demonstrate emoji rendering.
+  parameters: {
+    state: {
+      meta: {
+        emoji_style: 'twemoji',
+      },
+    },
+  },
 } satisfies Meta<typeof HTMLBlock>;
 
 export default meta;

app/javascript/mastodon/components/html_block/index.tsx

@@ -1,50 +1,30 @@
-import type { FC, ReactNode } from 'react';
+import { useCallback } from 'react';
-import { useMemo } from 'react';
 
-import { cleanExtraEmojis } from '@/mastodon/features/emoji/normalize';
+import type { OnElementHandler } from '@/mastodon/utils/html';
-import type { CustomEmojiMapArg } from '@/mastodon/features/emoji/types';
+import { polymorphicForwardRef } from '@/types/polymorphic';
-import { createLimitedCache } from '@/mastodon/utils/cache';
 
-import { htmlStringToComponents } from '../../utils/html';
+import type { EmojiHTMLProps } from '../emoji/html';
+import { ModernEmojiHTML } from '../emoji/html';
+import { useElementHandledLink } from '../status/handled_link';
 
-// Use a module-level cache to avoid re-rendering the same HTML multiple times.
+export const HTMLBlock = polymorphicForwardRef<
-const cache = createLimitedCache<ReactNode>({ maxSize: 1000 });
+  'div',
-
+  EmojiHTMLProps & Parameters<typeof useElementHandledLink>[0]
-interface HTMLBlockProps {
+>(
-  contents: string;
+  ({
-  extraEmojis?: CustomEmojiMapArg;
+    onElement: onParentElement,
-}
+    hrefToMention,
-
+    hashtagAccountId,
-export const HTMLBlock: FC<HTMLBlockProps> = ({
+    ...props
-  contents: raw,
+  }) => {
-  extraEmojis,
+    const { onElement: onLinkElement } = useElementHandledLink({
-}) => {
+      hrefToMention,
-  const customEmojis = useMemo(
+      hashtagAccountId,
-    () => cleanExtraEmojis(extraEmojis),
-    [extraEmojis],
-  );
-  const contents = useMemo(() => {
-    const key = JSON.stringify({ raw, customEmojis });
-    if (cache.has(key)) {
-      return cache.get(key);
-    }
-
-    const rendered = htmlStringToComponents(raw, {
-      onText,
-      extraArgs: { customEmojis },
     });
-
+    const onElement: OnElementHandler = useCallback(
-    cache.set(key, rendered);
+      (...args) => onParentElement?.(...args) ?? onLinkElement(...args),
-    return rendered;
+      [onLinkElement, onParentElement],
-  }, [raw, customEmojis]);
+    );
-
+    return <ModernEmojiHTML {...props} onElement={onElement} />;
-  return contents;
+  },
-};
+);
-
-function onText(
-  text: string,
-  // eslint-disable-next-line @typescript-eslint/no-unused-vars -- Doesn't do anything, just showing how typing would work.
-  { customEmojis }: { customEmojis: CustomEmojiMapArg | null },
-) {
-  return text;
-}

app/javascript/mastodon/features/emoji/emoji_picker.tsx

@@ -7,7 +7,7 @@ import { assetHost } from 'mastodon/utils/config';
 
 import { EMOJI_MODE_NATIVE } from './constants';
 import EmojiData from './emoji_data.json';
-import { useEmojiAppState } from './hooks';
+import { useEmojiAppState } from './mode';
 
 const backgroundImageFnDefault = () => `${assetHost}/emoji/sheet_15_1.png`;
 

app/javascript/mastodon/features/emoji/hooks.ts

@@ -1,78 +0,0 @@
-import { useCallback, useLayoutEffect, useMemo, useState } from 'react';
-
-import { createAppSelector, useAppSelector } from '@/mastodon/store';
-import { isModernEmojiEnabled } from '@/mastodon/utils/environment';
-
-import { toSupportedLocale } from './locale';
-import { determineEmojiMode } from './mode';
-import { cleanExtraEmojis } from './normalize';
-import { emojifyElement, emojifyText } from './render';
-import type { CustomEmojiMapArg, EmojiAppState } from './types';
-import { stringHasAnyEmoji } from './utils';
-
-interface UseEmojifyOptions {
-  text: string;
-  extraEmojis?: CustomEmojiMapArg;
-  deep?: boolean;
-}
-
-export function useEmojify({
-  text,
-  extraEmojis,
-  deep = true,
-}: UseEmojifyOptions) {
-  const [emojifiedText, setEmojifiedText] = useState<string | null>(null);
-
-  const appState = useEmojiAppState();
-  const extra = useMemo(() => cleanExtraEmojis(extraEmojis), [extraEmojis]);
-
-  const emojify = useCallback(
-    async (input: string) => {
-      let result: string | null = null;
-      if (deep) {
-        const wrapper = document.createElement('div');
-        wrapper.innerHTML = input;
-        if (await emojifyElement(wrapper, appState, extra ?? {})) {
-          result = wrapper.innerHTML;
-        }
-      } else {
-        result = await emojifyText(text, appState, extra ?? {});
-      }
-      if (result) {
-        setEmojifiedText(result);
-      } else {
-        setEmojifiedText(input);
-      }
-    },
-    [appState, deep, extra, text],
-  );
-  useLayoutEffect(() => {
-    if (isModernEmojiEnabled() && !!text.trim() && stringHasAnyEmoji(text)) {
-      void emojify(text);
-    } else {
-      // If no emoji or we don't want to render, fall back.
-      setEmojifiedText(text);
-    }
-  }, [emojify, text]);
-
-  return emojifiedText;
-}
-
-const modeSelector = createAppSelector(
-  [(state) => state.meta.get('emoji_style') as string],
-  (emoji_style) => determineEmojiMode(emoji_style),
-);
-
-export function useEmojiAppState(): EmojiAppState {
-  const locale = useAppSelector((state) =>
-    toSupportedLocale(state.meta.get('locale') as string),
-  );
-  const mode = useAppSelector(modeSelector);
-
-  return {
-    currentLocale: locale,
-    locales: [locale],
-    mode,
-    darkTheme: document.body.classList.contains('theme-default'),
-  };
-}

app/javascript/mastodon/features/emoji/index.ts

@@ -10,6 +10,8 @@ let worker: Worker | null = null;
 
 const log = emojiLogger('index');
 
+const WORKER_TIMEOUT = 1_000; // 1 second
+
 export function initializeEmoji() {
   log('initializing emojis');
   if (!worker && 'Worker' in window) {
@@ -29,7 +31,7 @@ export function initializeEmoji() {
       log('worker is not ready after timeout');
       worker = null;
       void fallbackLoad();
-    }, 500);
+    }, WORKER_TIMEOUT);
     thisWorker.addEventListener('message', (event: MessageEvent<string>) => {
       const { data: message } = event;
       if (message === 'ready') {

app/javascript/mastodon/features/emoji/mode.ts

@@ -1,6 +1,7 @@
 // Credit to Nolan Lawson for the original implementation.
 // See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js
 
+import { createAppSelector, useAppSelector } from '@/mastodon/store';
 import { isDevelopment } from '@/mastodon/utils/environment';
 
 import {
@@ -8,7 +9,27 @@ import {
   EMOJI_MODE_NATIVE_WITH_FLAGS,
   EMOJI_MODE_TWEMOJI,
 } from './constants';
-import type { EmojiMode } from './types';
+import { toSupportedLocale } from './locale';
+import type { EmojiAppState, EmojiMode } from './types';
+
+const modeSelector = createAppSelector(
+  [(state) => state.meta.get('emoji_style') as string],
+  (emoji_style) => determineEmojiMode(emoji_style),
+);
+
+export function useEmojiAppState(): EmojiAppState {
+  const locale = useAppSelector((state) =>
+    toSupportedLocale(state.meta.get('locale') as string),
+  );
+  const mode = useAppSelector(modeSelector);
+
+  return {
+    currentLocale: locale,
+    locales: [locale],
+    mode,
+    darkTheme: document.body.classList.contains('theme-default'),
+  };
+}
 
 type Feature = Uint8ClampedArray;
 

app/javascript/mastodon/features/emoji/render.test.ts

@@ -1,101 +1,12 @@
 import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
 
-import { EMOJI_MODE_TWEMOJI } from './constants';
 import * as db from './database';
+import * as loader from './loader';
 import {
-  emojifyElement,
+  loadEmojiDataToState,
-  emojifyText,
+  stringToEmojiState,
-  testCacheClear,
   tokenizeText,
 } from './render';
-import type { EmojiAppState } from './types';
-
-function mockDatabase() {
-  return {
-    searchCustomEmojisByShortcodes: vi
-      .spyOn(db, 'searchCustomEmojisByShortcodes')
-      .mockResolvedValue([customEmojiFactory()]),
-    searchEmojisByHexcodes: vi
-      .spyOn(db, 'searchEmojisByHexcodes')
-      .mockResolvedValue([
-        unicodeEmojiFactory({
-          hexcode: '1F60A',
-          label: 'smiling face with smiling eyes',
-          unicode: '😊',
-        }),
-        unicodeEmojiFactory({
-          hexcode: '1F1EA-1F1FA',
-          label: 'flag-eu',
-          unicode: '🇪🇺',
-        }),
-      ]),
-  };
-}
-
-const expectedSmileImage =
-  '<img draggable="false" class="emojione" alt="😊" title="smiling face with smiling eyes" src="/emoji/1f60a.svg">';
-const expectedFlagImage =
-  '<img draggable="false" class="emojione" alt="🇪🇺" title="flag-eu" src="/emoji/1f1ea-1f1fa.svg">';
-
-function testAppState(state: Partial<EmojiAppState> = {}) {
-  return {
-    locales: ['en'],
-    mode: EMOJI_MODE_TWEMOJI,
-    currentLocale: 'en',
-    darkTheme: false,
-    ...state,
-  } satisfies EmojiAppState;
-}
-
-describe('emojifyElement', () => {
-  function testElement(text = '<p>Hello 😊🇪🇺!</p><p>:custom:</p>') {
-    const testElement = document.createElement('div');
-    testElement.innerHTML = text;
-    return testElement;
-  }
-
-  afterEach(() => {
-    testCacheClear();
-    vi.restoreAllMocks();
-  });
-
-  test('caches element rendering results', async () => {
-    const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } =
-      mockDatabase();
-    await emojifyElement(testElement(), testAppState());
-    await emojifyElement(testElement(), testAppState());
-    await emojifyElement(testElement(), testAppState());
-    expect(searchEmojisByHexcodes).toHaveBeenCalledExactlyOnceWith(
-      ['1F1EA-1F1FA', '1F60A'],
-      'en',
-    );
-    expect(searchCustomEmojisByShortcodes).toHaveBeenCalledExactlyOnceWith([
-      ':custom:',
-    ]);
-  });
-
-  test('returns null when no emoji are found', async () => {
-    mockDatabase();
-    const actual = await emojifyElement(
-      testElement('<p>here is just text :)</p>'),
-      testAppState(),
-    );
-    expect(actual).toBeNull();
-  });
-});
-
-describe('emojifyText', () => {
-  test('returns original input when no emoji are in string', async () => {
-    const actual = await emojifyText('nothing here', testAppState());
-    expect(actual).toBe('nothing here');
-  });
-
-  test('renders Unicode emojis to twemojis', async () => {
-    mockDatabase();
-    const actual = await emojifyText('Hello 😊🇪🇺!', testAppState());
-    expect(actual).toBe(`Hello ${expectedSmileImage}${expectedFlagImage}!`);
-  });
-});
 
 describe('tokenizeText', () => {
   test('returns an array of text to be a single token', () => {
@@ -162,3 +73,106 @@ describe('tokenizeText', () => {
     ]);
   });
 });
+
+describe('stringToEmojiState', () => {
+  test('returns unicode emoji state for valid unicode emoji', () => {
+    expect(stringToEmojiState('😊')).toEqual({
+      type: 'unicode',
+      code: '1F60A',
+    });
+  });
+
+  test('returns custom emoji state for valid custom emoji', () => {
+    expect(stringToEmojiState(':smile:')).toEqual({
+      type: 'custom',
+      code: 'smile',
+      data: undefined,
+    });
+  });
+
+  test('returns custom emoji state with data when provided', () => {
+    const customEmoji = {
+      smile: customEmojiFactory({
+        shortcode: 'smile',
+        url: 'https://example.com/smile.png',
+        static_url: 'https://example.com/smile_static.png',
+      }),
+    };
+    expect(stringToEmojiState(':smile:', customEmoji)).toEqual({
+      type: 'custom',
+      code: 'smile',
+      data: customEmoji.smile,
+    });
+  });
+
+  test('returns null for invalid emoji strings', () => {
+    expect(stringToEmojiState('notanemoji')).toBeNull();
+    expect(stringToEmojiState(':invalid-emoji:')).toBeNull();
+  });
+});
+
+describe('loadEmojiDataToState', () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  test('loads unicode data into state', async () => {
+    const dbCall = vi
+      .spyOn(db, 'loadEmojiByHexcode')
+      .mockResolvedValue(unicodeEmojiFactory());
+    const unicodeState = { type: 'unicode', code: '1F60A' } as const;
+    const result = await loadEmojiDataToState(unicodeState, 'en');
+    expect(dbCall).toHaveBeenCalledWith('1F60A', 'en');
+    expect(result).toEqual({
+      type: 'unicode',
+      code: '1F60A',
+      data: unicodeEmojiFactory(),
+    });
+  });
+
+  test('loads custom emoji data into state', async () => {
+    const dbCall = vi
+      .spyOn(db, 'loadCustomEmojiByShortcode')
+      .mockResolvedValueOnce(customEmojiFactory());
+    const customState = { type: 'custom', code: 'smile' } as const;
+    const result = await loadEmojiDataToState(customState, 'en');
+    expect(dbCall).toHaveBeenCalledWith('smile');
+    expect(result).toEqual({
+      type: 'custom',
+      code: 'smile',
+      data: customEmojiFactory(),
+    });
+  });
+
+  test('returns null if unicode emoji not found in database', async () => {
+    vi.spyOn(db, 'loadEmojiByHexcode').mockResolvedValueOnce(undefined);
+    const unicodeState = { type: 'unicode', code: '1F60A' } as const;
+    const result = await loadEmojiDataToState(unicodeState, 'en');
+    expect(result).toBeNull();
+  });
+
+  test('returns null if custom emoji not found in database', async () => {
+    vi.spyOn(db, 'loadCustomEmojiByShortcode').mockResolvedValueOnce(undefined);
+    const customState = { type: 'custom', code: 'smile' } as const;
+    const result = await loadEmojiDataToState(customState, 'en');
+    expect(result).toBeNull();
+  });
+
+  test('retries loading emoji data once if initial load fails', async () => {
+    const dbCall = vi
+      .spyOn(db, 'loadEmojiByHexcode')
+      .mockRejectedValue(new db.LocaleNotLoadedError('en'));
+    vi.spyOn(loader, 'importEmojiData').mockResolvedValueOnce();
+    const consoleCall = vi
+      .spyOn(console, 'warn')
+      .mockImplementationOnce(() => null);
+
+    const unicodeState = { type: 'unicode', code: '1F60A' } as const;
+    const result = await loadEmojiDataToState(unicodeState, 'en');
+
+    expect(dbCall).toHaveBeenCalledTimes(2);
+    expect(loader.importEmojiData).toHaveBeenCalledWith('en');
+    expect(consoleCall).toHaveBeenCalled();
+    expect(result).toBeNull();
+  });
+});

app/javascript/mastodon/features/emoji/render.ts

@@ -1,7 +1,3 @@
-import { autoPlayGif } from '@/mastodon/initial_state';
-import { createLimitedCache } from '@/mastodon/utils/cache';
-import * as perf from '@/mastodon/utils/performance';
-
 import {
   EMOJI_MODE_NATIVE,
   EMOJI_MODE_NATIVE_WITH_FLAGS,
@@ -12,33 +8,69 @@ import {
   loadCustomEmojiByShortcode,
   loadEmojiByHexcode,
   LocaleNotLoadedError,
-  searchCustomEmojisByShortcodes,
-  searchEmojisByHexcodes,
 } from './database';
 import { importEmojiData } from './loader';
-import { emojiToUnicodeHex, unicodeHexToUrl } from './normalize';
+import { emojiToUnicodeHex } from './normalize';
 import type {
-  EmojiAppState,
   EmojiLoadedState,
   EmojiMode,
   EmojiState,
   EmojiStateCustom,
-  EmojiStateMap,
   EmojiStateUnicode,
   ExtraCustomEmojiMap,
-  LocaleOrCustom,
 } from './types';
 import {
   anyEmojiRegex,
   emojiLogger,
   isCustomEmoji,
   isUnicodeEmoji,
-  stringHasAnyEmoji,
   stringHasUnicodeFlags,
 } from './utils';
 
 const log = emojiLogger('render');
 
+type TokenizedText = (string | EmojiState)[];
+
+/**
+ * Tokenizes text into strings and emoji states.
+ * @param text Text to tokenize.
+ * @returns Array of strings and emoji states.
+ */
+export function tokenizeText(text: string): TokenizedText {
+  if (!text.trim()) {
+    return [text];
+  }
+
+  const tokens = [];
+  let lastIndex = 0;
+  for (const match of text.matchAll(anyEmojiRegex())) {
+    if (match.index > lastIndex) {
+      tokens.push(text.slice(lastIndex, match.index));
+    }
+
+    const code = match[0];
+
+    if (code.startsWith(':') && code.endsWith(':')) {
+      // Custom emoji
+      tokens.push({
+        type: EMOJI_TYPE_CUSTOM,
+        code,
+      } satisfies EmojiStateCustom);
+    } else {
+      // Unicode emoji
+      tokens.push({
+        type: EMOJI_TYPE_UNICODE,
+        code: code,
+      } satisfies EmojiStateUnicode);
+    }
+    lastIndex = match.index + code.length;
+  }
+  if (lastIndex < text.length) {
+    tokens.push(text.slice(lastIndex));
+  }
+  return tokens;
+}
+
 /**
  * Parses emoji string to extract emoji state.
  * @param code Hex code or custom shortcode.
@@ -132,305 +164,19 @@ export function isStateLoaded(state: EmojiState): state is EmojiLoadedState {
 }
 
 /**
- * Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions.
+ * Determines if the given token should be rendered as an image based on the emoji mode.
+ * @param state Emoji state to parse.
+ * @param mode Rendering mode.
+ * @returns Whether to render as an image.
  */
-export async function emojifyElement<Element extends HTMLElement>(
+export function shouldRenderImage(state: EmojiState, mode: EmojiMode): boolean {
-  element: Element,
+  if (state.type === EMOJI_TYPE_UNICODE) {
-  appState: EmojiAppState,
-  extraEmojis: ExtraCustomEmojiMap = {},
-): Promise<Element | null> {
-  const cacheKey = createCacheKey(element, appState, extraEmojis);
-  const cached = getCached(cacheKey);
-  if (cached !== undefined) {
-    log('Cache hit on %s', element.outerHTML);
-    if (cached === null) {
-      return null;
-    }
-    element.innerHTML = cached;
-    return element;
-  }
-  if (!stringHasAnyEmoji(element.innerHTML)) {
-    updateCache(cacheKey, null);
-    return null;
-  }
-  perf.start('emojifyElement()');
-  const queue: (HTMLElement | Text)[] = [element];
-  while (queue.length > 0) {
-    const current = queue.shift();
-    if (
-      !current ||
-      current instanceof HTMLScriptElement ||
-      current instanceof HTMLStyleElement
-    ) {
-      continue;
-    }
-
-    if (
-      current.textContent &&
-      (current instanceof Text || !current.hasChildNodes())
-    ) {
-      const renderedContent = await textToElementArray(
-        current.textContent,
-        appState,
-        extraEmojis,
-      );
-      if (renderedContent) {
-        if (!(current instanceof Text)) {
-          current.textContent = null; // Clear the text content if it's not a Text node.
-        }
-        current.replaceWith(renderedToHTML(renderedContent));
-      }
-      continue;
-    }
-
-    for (const child of current.childNodes) {
-      if (child instanceof HTMLElement || child instanceof Text) {
-        queue.push(child);
-      }
-    }
-  }
-  updateCache(cacheKey, element.innerHTML);
-  perf.stop('emojifyElement()');
-  return element;
-}
-
-export async function emojifyText(
-  text: string,
-  appState: EmojiAppState,
-  extraEmojis: ExtraCustomEmojiMap = {},
-): Promise<string | null> {
-  const cacheKey = createCacheKey(text, appState, extraEmojis);
-  const cached = getCached(cacheKey);
-  if (cached !== undefined) {
-    log('Cache hit on %s', text);
-    return cached ?? text;
-  }
-  if (!stringHasAnyEmoji(text)) {
-    updateCache(cacheKey, null);
-    return text;
-  }
-  const eleArray = await textToElementArray(text, appState, extraEmojis);
-  if (!eleArray) {
-    updateCache(cacheKey, null);
-    return text;
-  }
-  const rendered = renderedToHTML(eleArray, document.createElement('div'));
-  updateCache(cacheKey, rendered.innerHTML);
-  return rendered.innerHTML;
-}
-
-// Private functions
-
-const {
-  set: updateCache,
-  get: getCached,
-  clear: cacheClear,
-} = createLimitedCache<string | null>({ log: log.extend('cache') });
-
-function createCacheKey(
-  input: HTMLElement | string,
-  appState: EmojiAppState,
-  extraEmojis: ExtraCustomEmojiMap,
-) {
-  return JSON.stringify([
-    input instanceof HTMLElement ? input.outerHTML : input,
-    appState,
-    extraEmojis,
-  ]);
-}
-
-type EmojifiedTextArray = (string | HTMLImageElement)[];
-
-async function textToElementArray(
-  text: string,
-  appState: EmojiAppState,
-  extraEmojis: ExtraCustomEmojiMap = {},
-): Promise<EmojifiedTextArray | null> {
-  // Exit if no text to convert.
-  if (!text.trim()) {
-    return null;
-  }
-
-  const tokens = tokenizeText(text);
-
-  // If only one token and it's a string, exit early.
-  if (tokens.length === 1 && typeof tokens[0] === 'string') {
-    return null;
-  }
-
-  // Get all emoji from the state map, loading any missing ones.
-  await loadMissingEmojiIntoCache(tokens, appState, extraEmojis);
-
-  const renderedFragments: EmojifiedTextArray = [];
-  for (const token of tokens) {
-    if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) {
-      let state: EmojiState | undefined;
-      if (token.type === EMOJI_TYPE_CUSTOM) {
-        const extraEmojiData = extraEmojis[token.code];
-        if (extraEmojiData) {
-          state = {
-            type: EMOJI_TYPE_CUSTOM,
-            data: extraEmojiData,
-            code: token.code,
-          };
-        } else {
-          state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM);
-        }
-      } else {
-        state = emojiForLocale(
-          emojiToUnicodeHex(token.code),
-          appState.currentLocale,
-        );
-      }
-
-      // If the state is valid, create an image element. Otherwise, just append as text.
-      if (state && typeof state !== 'string' && isStateLoaded(state)) {
-        const image = stateToImage(state, appState);
-        renderedFragments.push(image);
-        continue;
-      }
-    }
-    const text = typeof token === 'string' ? token : token.code;
-    renderedFragments.push(text);
-  }
-
-  return renderedFragments;
-}
-
-type TokenizedText = (string | EmojiState)[];
-
-export function tokenizeText(text: string): TokenizedText {
-  if (!text.trim()) {
-    return [text];
-  }
-
-  const tokens = [];
-  let lastIndex = 0;
-  for (const match of text.matchAll(anyEmojiRegex())) {
-    if (match.index > lastIndex) {
-      tokens.push(text.slice(lastIndex, match.index));
-    }
-
-    const code = match[0];
-
-    if (code.startsWith(':') && code.endsWith(':')) {
-      // Custom emoji
-      tokens.push({
-        type: EMOJI_TYPE_CUSTOM,
-        code,
-      } satisfies EmojiStateCustom);
-    } else {
-      // Unicode emoji
-      tokens.push({
-        type: EMOJI_TYPE_UNICODE,
-        code: code,
-      } satisfies EmojiStateUnicode);
-    }
-    lastIndex = match.index + code.length;
-  }
-  if (lastIndex < text.length) {
-    tokens.push(text.slice(lastIndex));
-  }
-  return tokens;
-}
-
-const localeCacheMap = new Map<LocaleOrCustom, EmojiStateMap>([
-  [
-    EMOJI_TYPE_CUSTOM,
-    createLimitedCache<EmojiState>({ log: log.extend('custom') }),
-  ],
-]);
-
-function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap {
-  return (
-    localeCacheMap.get(locale) ??
-    createLimitedCache<EmojiState>({ log: log.extend(locale) })
-  );
-}
-
-function emojiForLocale(
-  code: string,
-  locale: LocaleOrCustom,
-): EmojiState | undefined {
-  const cache = cacheForLocale(locale);
-  return cache.get(code);
-}
-
-async function loadMissingEmojiIntoCache(
-  tokens: TokenizedText,
-  { mode, currentLocale }: EmojiAppState,
-  extraEmojis: ExtraCustomEmojiMap,
-) {
-  const missingUnicodeEmoji = new Set<string>();
-  const missingCustomEmoji = new Set<string>();
-
-  // Iterate over tokens and check if they are in the cache already.
-  for (const token of tokens) {
-    if (typeof token === 'string') {
-      continue; // Skip plain strings.
-    }
-
-    // If this is a custom emoji, check it separately.
-    if (token.type === EMOJI_TYPE_CUSTOM) {
-      const code = token.code;
-      if (code in extraEmojis) {
-        continue; // We don't care about extra emoji.
-      }
-      const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM);
-      if (!emojiState) {
-        missingCustomEmoji.add(code);
-      }
-      // Otherwise this is a unicode emoji, so check it against all locales.
-    } else if (shouldRenderImage(token, mode)) {
-      const code = emojiToUnicodeHex(token.code);
-      if (missingUnicodeEmoji.has(code)) {
-        continue; // Already marked as missing.
-      }
-      const emojiState = emojiForLocale(code, currentLocale);
-      if (!emojiState) {
-        // If it's missing in one locale, we consider it missing for all.
-        missingUnicodeEmoji.add(code);
-      }
-    }
-  }
-
-  if (missingUnicodeEmoji.size > 0) {
-    const missingEmojis = Array.from(missingUnicodeEmoji).toSorted();
-    const emojis = await searchEmojisByHexcodes(missingEmojis, currentLocale);
-    const cache = cacheForLocale(currentLocale);
-    for (const emoji of emojis) {
-      cache.set(emoji.hexcode, {
-        type: EMOJI_TYPE_UNICODE,
-        data: emoji,
-        code: emoji.hexcode,
-      });
-    }
-    localeCacheMap.set(currentLocale, cache);
-  }
-
-  if (missingCustomEmoji.size > 0) {
-    const missingEmojis = Array.from(missingCustomEmoji).toSorted();
-    const emojis = await searchCustomEmojisByShortcodes(missingEmojis);
-    const cache = cacheForLocale(EMOJI_TYPE_CUSTOM);
-    for (const emoji of emojis) {
-      cache.set(emoji.shortcode, {
-        type: EMOJI_TYPE_CUSTOM,
-        data: emoji,
-        code: emoji.shortcode,
-      });
-    }
-    localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache);
-  }
-}
-
-export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean {
-  if (token.type === EMOJI_TYPE_UNICODE) {
     // If the mode is native or native with flags for non-flag emoji
     // we can just append the text node directly.
     if (
       mode === EMOJI_MODE_NATIVE ||
       (mode === EMOJI_MODE_NATIVE_WITH_FLAGS &&
-        !stringHasUnicodeFlags(token.code))
+        !stringHasUnicodeFlags(state.code))
     ) {
       return false;
     }
@@ -438,52 +184,3 @@ export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean {
 
   return true;
 }
-
-function stateToImage(state: EmojiLoadedState, appState: EmojiAppState) {
-  const image = document.createElement('img');
-  image.draggable = false;
-  image.classList.add('emojione');
-
-  if (state.type === EMOJI_TYPE_UNICODE) {
-    image.alt = state.data.unicode;
-    image.title = state.data.label;
-    image.src = unicodeHexToUrl(state.data.hexcode, appState.darkTheme);
-  } else {
-    // Custom emoji
-    const shortCode = `:${state.data.shortcode}:`;
-    image.classList.add('custom-emoji');
-    image.alt = shortCode;
-    image.title = shortCode;
-    image.src = autoPlayGif ? state.data.url : state.data.static_url;
-    image.dataset.original = state.data.url;
-    image.dataset.static = state.data.static_url;
-  }
-
-  return image;
-}
-
-function renderedToHTML(renderedArray: EmojifiedTextArray): DocumentFragment;
-function renderedToHTML<ParentType extends ParentNode>(
-  renderedArray: EmojifiedTextArray,
-  parent: ParentType,
-): ParentType;
-function renderedToHTML(
-  renderedArray: EmojifiedTextArray,
-  parent: ParentNode | null = null,
-) {
-  const fragment = parent ?? document.createDocumentFragment();
-  for (const fragmentItem of renderedArray) {
-    if (typeof fragmentItem === 'string') {
-      fragment.appendChild(document.createTextNode(fragmentItem));
-    } else if (fragmentItem instanceof HTMLImageElement) {
-      fragment.appendChild(fragmentItem);
-    }
-  }
-  return fragment;
-}
-
-// Testing helpers
-export const testCacheClear = () => {
-  cacheClear();
-  localeCacheMap.clear();
-};

app/javascript/mastodon/features/emoji/types.ts

@@ -4,7 +4,6 @@ import type { FlatCompactEmoji, Locale } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
 import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
-import type { LimitedCache } from '@/mastodon/utils/cache';
 
 import type {
   EMOJI_MODE_NATIVE,
@@ -48,12 +47,11 @@ export interface EmojiStateCustom {
   data?: CustomEmojiRenderFields;
 }
 export type EmojiState = EmojiStateUnicode | EmojiStateCustom;
+
 export type EmojiLoadedState =
   | Required<EmojiStateUnicode>
   | Required<EmojiStateCustom>;
 
-export type EmojiStateMap = LimitedCache<string, EmojiState>;
-
 export type CustomEmojiMapArg =
   | ExtraCustomEmojiMap
   | ImmutableList<CustomEmoji>
@@ -64,9 +62,3 @@ export type ExtraCustomEmojiMap = Record<
   string,
   Pick<CustomEmojiData, 'shortcode' | 'static_url' | 'url'>
 >;
-
-export interface TwemojiBorderInfo {
-  hexCode: string;
-  hasLightBorder: boolean;
-  hasDarkBorder: boolean;
-}

app/javascript/mastodon/features/emoji/utils.test.ts

@@ -1,35 +1,31 @@
-import {
+import { isCustomEmoji, isUnicodeEmoji, stringHasUnicodeFlags } from './utils';
-  stringHasAnyEmoji,
-  stringHasCustomEmoji,
-  stringHasUnicodeEmoji,
-  stringHasUnicodeFlags,
-} from './utils';
 
-describe('stringHasUnicodeEmoji', () => {
+describe('isUnicodeEmoji', () => {
   test.concurrent.for([
-    ['only text', false],
+    ['😊', true],
-    ['text with non-emoji symbols ™©', false],
+    ['🇿🇼', true],
-    ['text with emoji 😀', true],
+    ['🏴‍☠️', true],
-    ['multiple emojis 😀😃😄', true],
+    ['🏳️‍🌈', true],
-    ['emoji with skin tone 👍🏽', true],
+    ['foo', false],
-    ['emoji with ZWJ 👩‍❤️‍👨', true],
+    [':smile:', false],
-    ['emoji with variation selector ✊️', true],
+    ['😊foo', false],
-    ['emoji with keycap 1️⃣', true],
+  ] as const)('isUnicodeEmoji("%s") is %o', ([input, expected], { expect }) => {
-    ['emoji with flags 🇺🇸', true],
+    expect(isUnicodeEmoji(input)).toBe(expected);
-    ['emoji with regional indicators 🇦🇺', true],
+  });
-    ['emoji with gender 👩‍⚕️', true],
+});
-    ['emoji with family 👨‍👩‍👧‍👦', true],
+
-    ['emoji with zero width joiner 👩‍🔬', true],
+describe('isCustomEmoji', () => {
-    ['emoji with non-BMP codepoint 🧑‍🚀', true],
+  test.concurrent.for([
-    ['emoji with combining marks 👨‍👩‍👧‍👦', true],
+    [':smile:', true],
-    ['emoji with enclosing keycap #️⃣', true],
+    [':smile_123:', true],
-    ['emoji with no visible glyph \u200D', false],
+    [':SMILE:', true],
-  ] as const)(
+    ['😊', false],
-    'stringHasUnicodeEmoji has emojis in "%s": %o',
+    ['foo', false],
-    ([text, expected], { expect }) => {
+    [':smile', false],
-      expect(stringHasUnicodeEmoji(text)).toBe(expected);
+    ['smile:', false],
-    },
+  ] as const)('isCustomEmoji("%s") is %o', ([input, expected], { expect }) => {
-  );
+    expect(isCustomEmoji(input)).toBe(expected);
+  });
 });
 
 describe('stringHasUnicodeFlags', () => {
@@ -51,27 +47,3 @@ describe('stringHasUnicodeFlags', () => {
     },
   );
 });
-
-describe('stringHasCustomEmoji', () => {
-  test('string with custom emoji returns true', () => {
-    expect(stringHasCustomEmoji(':custom: :test:')).toBeTruthy();
-  });
-  test('string without custom emoji returns false', () => {
-    expect(stringHasCustomEmoji('🏳️‍🌈 :🏳️‍🌈: text ™')).toBeFalsy();
-  });
-});
-
-describe('stringHasAnyEmoji', () => {
-  test('string without any emoji or characters', () => {
-    expect(stringHasAnyEmoji('normal text. 12356?!')).toBeFalsy();
-  });
-  test('string with non-emoji characters', () => {
-    expect(stringHasAnyEmoji('™©')).toBeFalsy();
-  });
-  test('has unicode emoji', () => {
-    expect(stringHasAnyEmoji('🏳️‍🌈🔥🇸🇹 👩‍🔬')).toBeTruthy();
-  });
-  test('has custom emoji', () => {
-    expect(stringHasAnyEmoji(':test: :custom:')).toBeTruthy();
-  });
-});

app/javascript/mastodon/features/emoji/utils.ts

@@ -6,10 +6,6 @@ export function emojiLogger(segment: string) {
   return debug(`emojis:${segment}`);
 }
 
-export function stringHasUnicodeEmoji(input: string): boolean {
-  return new RegExp(EMOJI_REGEX, supportedFlags()).test(input);
-}
-
 export function isUnicodeEmoji(input: string): boolean {
   return (
     input.length > 0 &&
@@ -34,19 +30,13 @@ export function stringHasUnicodeFlags(input: string): boolean {
 
 // Constant as this is supported by all browsers.
 const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i;
+// Use the polyfill regex or the Unicode property escapes if supported.
+const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}';
 
 export function isCustomEmoji(input: string): boolean {
   return new RegExp(`^${CUSTOM_EMOJI_REGEX.source}$`, 'i').test(input);
 }
 
-export function stringHasCustomEmoji(input: string) {
-  return CUSTOM_EMOJI_REGEX.test(input);
-}
-
-export function stringHasAnyEmoji(input: string) {
-  return stringHasUnicodeEmoji(input) || stringHasCustomEmoji(input);
-}
-
 export function anyEmojiRegex() {
   return new RegExp(
     `${EMOJI_REGEX}|${CUSTOM_EMOJI_REGEX.source}`,
@@ -64,5 +54,3 @@ function supportedFlags(flags = '') {
   }
   return flags;
 }
-
-const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}';

app/javascript/testing/api.ts

@@ -1,7 +1,10 @@
+import type { CompactEmoji } from 'emojibase';
 import { http, HttpResponse } from 'msw';
 import { action } from 'storybook/actions';
 
-import { relationshipsFactory } from './factories';
+import { toSupportedLocale } from '@/mastodon/features/emoji/locale';
+
+import { customEmojiFactory, relationshipsFactory } from './factories';
 
 export const mockHandlers = {
   mute: http.post<{ id: string }>('/api/v1/accounts/:id/mute', ({ params }) => {
@@ -40,6 +43,24 @@ export const mockHandlers = {
       );
     },
   ),
+  emojiCustomData: http.get('/api/v1/custom_emojis', () => {
+    action('fetching custom emoji data')();
+    return HttpResponse.json([customEmojiFactory()]);
+  }),
+  emojiData: http.get<{ locale: string }>(
+    '/packs-dev/emoji/:locale.json',
+    async ({ params }) => {
+      const locale = toSupportedLocale(params.locale);
+      action('fetching emoji data')(locale);
+      const { default: data } = (await import(
+        `emojibase-data/${locale}/compact.json`
+      )) as {
+        default: CompactEmoji[];
+      };
+
+      return HttpResponse.json([data]);
+    },
+  ),
 };
 
 export const unhandledRequestHandler = ({ url }: Request) => {