2
0

Emoji: Cleanup new code (#36402)

This commit is contained in:
Echo
2025-10-14 11:36:25 +02:00
committed by GitHub
parent 9a001e7839
commit 0c64e7f75e
16 changed files with 356 additions and 659 deletions

View File

@@ -50,9 +50,13 @@ const preview: Preview = {
locale: 'en',
},
decorators: [
(Story, { parameters, globals }) => {
(Story, { parameters, globals, args }) => {
// Get the locale from the global toolbar
// and merge it with any parameters or args state.
const { locale } = globals as { locale: string };
const { state = {} } = parameters;
const { state: argsState = {} } = args;
const reducer = reducerWithInitialState(
{
meta: {
@@ -60,7 +64,9 @@ const preview: Preview = {
},
},
state as Record<string, unknown>,
argsState as Record<string, unknown>,
);
const store = configureStore({
reducer,
middleware(getDefaultMiddleware) {

View File

@@ -0,0 +1,56 @@
import type { ComponentProps } from 'react';
import type { Meta, StoryObj } from '@storybook/react-vite';
import { importCustomEmojiData } from '@/mastodon/features/emoji/loader';
import { Emoji } from './index';
type EmojiProps = ComponentProps<typeof Emoji> & { state: string };
const meta = {
title: 'Components/Emoji',
component: Emoji,
args: {
code: '🖤',
state: 'auto',
},
argTypes: {
code: {
name: 'Emoji',
},
state: {
control: {
type: 'select',
labels: {
auto: 'Auto',
native: 'Native',
twemoji: 'Twemoji',
},
},
options: ['auto', 'native', 'twemoji'],
name: 'Emoji Style',
mapping: {
auto: { meta: { emoji_style: 'auto' } },
native: { meta: { emoji_style: 'native' } },
twemoji: { meta: { emoji_style: 'twemoji' } },
},
},
},
render(args) {
void importCustomEmojiData();
return <Emoji {...args} />;
},
} satisfies Meta<EmojiProps>;
export default meta;
type Story = StoryObj<typeof meta>;
export const Default: Story = {};
export const CustomEmoji: Story = {
args: {
code: ':custom:',
},
};

View File

@@ -14,7 +14,7 @@ import { polymorphicForwardRef } from '@/types/polymorphic';
import { AnimateEmojiProvider, CustomEmojiProvider } from './context';
import { textToEmojis } from './index';
interface EmojiHTMLProps {
export interface EmojiHTMLProps {
htmlString: string;
extraEmojis?: CustomEmojiMapArg;
className?: string;

View File

@@ -2,7 +2,7 @@ import type { FC } from 'react';
import { useContext, useEffect, useState } from 'react';
import { EMOJI_TYPE_CUSTOM } from '@/mastodon/features/emoji/constants';
import { useEmojiAppState } from '@/mastodon/features/emoji/hooks';
import { useEmojiAppState } from '@/mastodon/features/emoji/mode';
import { unicodeHexToUrl } from '@/mastodon/features/emoji/normalize';
import {
isStateLoaded,

View File

@@ -7,23 +7,49 @@ const meta = {
title: 'Components/HTMLBlock',
component: HTMLBlock,
args: {
contents:
'<p>Hello, world!</p>\n<p><a href="#">A link</a></p>\n<p>This should be filtered out: <button>Bye!</button></p>',
htmlString: `<p>Hello, world!</p>
<p><a href="#">A link</a></p>
<p>This should be filtered out: <button>Bye!</button></p>
<p>This also has emoji: 🖤</p>`,
},
argTypes: {
extraEmojis: {
table: {
disable: true,
},
},
onElement: {
table: {
disable: true,
},
},
onAttribute: {
table: {
disable: true,
},
},
},
render(args) {
return (
// Just for visual clarity in Storybook.
<div
<HTMLBlock
{...args}
style={{
border: '1px solid black',
padding: '1rem',
minWidth: '300px',
}}
>
<HTMLBlock {...args} />
</div>
/>
);
},
// Force Twemoji to demonstrate emoji rendering.
parameters: {
state: {
meta: {
emoji_style: 'twemoji',
},
},
},
} satisfies Meta<typeof HTMLBlock>;
export default meta;

View File

@@ -1,50 +1,30 @@
import type { FC, ReactNode } from 'react';
import { useMemo } from 'react';
import { useCallback } from 'react';
import { cleanExtraEmojis } from '@/mastodon/features/emoji/normalize';
import type { CustomEmojiMapArg } from '@/mastodon/features/emoji/types';
import { createLimitedCache } from '@/mastodon/utils/cache';
import type { OnElementHandler } from '@/mastodon/utils/html';
import { polymorphicForwardRef } from '@/types/polymorphic';
import { htmlStringToComponents } from '../../utils/html';
import type { EmojiHTMLProps } from '../emoji/html';
import { ModernEmojiHTML } from '../emoji/html';
import { useElementHandledLink } from '../status/handled_link';
// Use a module-level cache to avoid re-rendering the same HTML multiple times.
const cache = createLimitedCache<ReactNode>({ maxSize: 1000 });
interface HTMLBlockProps {
contents: string;
extraEmojis?: CustomEmojiMapArg;
}
export const HTMLBlock: FC<HTMLBlockProps> = ({
contents: raw,
extraEmojis,
}) => {
const customEmojis = useMemo(
() => cleanExtraEmojis(extraEmojis),
[extraEmojis],
);
const contents = useMemo(() => {
const key = JSON.stringify({ raw, customEmojis });
if (cache.has(key)) {
return cache.get(key);
}
const rendered = htmlStringToComponents(raw, {
onText,
extraArgs: { customEmojis },
export const HTMLBlock = polymorphicForwardRef<
'div',
EmojiHTMLProps & Parameters<typeof useElementHandledLink>[0]
>(
({
onElement: onParentElement,
hrefToMention,
hashtagAccountId,
...props
}) => {
const { onElement: onLinkElement } = useElementHandledLink({
hrefToMention,
hashtagAccountId,
});
cache.set(key, rendered);
return rendered;
}, [raw, customEmojis]);
return contents;
};
function onText(
text: string,
// eslint-disable-next-line @typescript-eslint/no-unused-vars -- Doesn't do anything, just showing how typing would work.
{ customEmojis }: { customEmojis: CustomEmojiMapArg | null },
) {
return text;
}
const onElement: OnElementHandler = useCallback(
(...args) => onParentElement?.(...args) ?? onLinkElement(...args),
[onLinkElement, onParentElement],
);
return <ModernEmojiHTML {...props} onElement={onElement} />;
},
);

View File

@@ -7,7 +7,7 @@ import { assetHost } from 'mastodon/utils/config';
import { EMOJI_MODE_NATIVE } from './constants';
import EmojiData from './emoji_data.json';
import { useEmojiAppState } from './hooks';
import { useEmojiAppState } from './mode';
const backgroundImageFnDefault = () => `${assetHost}/emoji/sheet_15_1.png`;

View File

@@ -1,78 +0,0 @@
import { useCallback, useLayoutEffect, useMemo, useState } from 'react';
import { createAppSelector, useAppSelector } from '@/mastodon/store';
import { isModernEmojiEnabled } from '@/mastodon/utils/environment';
import { toSupportedLocale } from './locale';
import { determineEmojiMode } from './mode';
import { cleanExtraEmojis } from './normalize';
import { emojifyElement, emojifyText } from './render';
import type { CustomEmojiMapArg, EmojiAppState } from './types';
import { stringHasAnyEmoji } from './utils';
interface UseEmojifyOptions {
text: string;
extraEmojis?: CustomEmojiMapArg;
deep?: boolean;
}
export function useEmojify({
text,
extraEmojis,
deep = true,
}: UseEmojifyOptions) {
const [emojifiedText, setEmojifiedText] = useState<string | null>(null);
const appState = useEmojiAppState();
const extra = useMemo(() => cleanExtraEmojis(extraEmojis), [extraEmojis]);
const emojify = useCallback(
async (input: string) => {
let result: string | null = null;
if (deep) {
const wrapper = document.createElement('div');
wrapper.innerHTML = input;
if (await emojifyElement(wrapper, appState, extra ?? {})) {
result = wrapper.innerHTML;
}
} else {
result = await emojifyText(text, appState, extra ?? {});
}
if (result) {
setEmojifiedText(result);
} else {
setEmojifiedText(input);
}
},
[appState, deep, extra, text],
);
useLayoutEffect(() => {
if (isModernEmojiEnabled() && !!text.trim() && stringHasAnyEmoji(text)) {
void emojify(text);
} else {
// If no emoji or we don't want to render, fall back.
setEmojifiedText(text);
}
}, [emojify, text]);
return emojifiedText;
}
const modeSelector = createAppSelector(
[(state) => state.meta.get('emoji_style') as string],
(emoji_style) => determineEmojiMode(emoji_style),
);
export function useEmojiAppState(): EmojiAppState {
const locale = useAppSelector((state) =>
toSupportedLocale(state.meta.get('locale') as string),
);
const mode = useAppSelector(modeSelector);
return {
currentLocale: locale,
locales: [locale],
mode,
darkTheme: document.body.classList.contains('theme-default'),
};
}

View File

@@ -10,6 +10,8 @@ let worker: Worker | null = null;
const log = emojiLogger('index');
const WORKER_TIMEOUT = 1_000; // 1 second
export function initializeEmoji() {
log('initializing emojis');
if (!worker && 'Worker' in window) {
@@ -29,7 +31,7 @@ export function initializeEmoji() {
log('worker is not ready after timeout');
worker = null;
void fallbackLoad();
}, 500);
}, WORKER_TIMEOUT);
thisWorker.addEventListener('message', (event: MessageEvent<string>) => {
const { data: message } = event;
if (message === 'ready') {

View File

@@ -1,6 +1,7 @@
// Credit to Nolan Lawson for the original implementation.
// See: https://github.com/nolanlawson/emoji-picker-element/blob/master/src/picker/utils/testColorEmojiSupported.js
import { createAppSelector, useAppSelector } from '@/mastodon/store';
import { isDevelopment } from '@/mastodon/utils/environment';
import {
@@ -8,7 +9,27 @@ import {
EMOJI_MODE_NATIVE_WITH_FLAGS,
EMOJI_MODE_TWEMOJI,
} from './constants';
import type { EmojiMode } from './types';
import { toSupportedLocale } from './locale';
import type { EmojiAppState, EmojiMode } from './types';
const modeSelector = createAppSelector(
[(state) => state.meta.get('emoji_style') as string],
(emoji_style) => determineEmojiMode(emoji_style),
);
export function useEmojiAppState(): EmojiAppState {
const locale = useAppSelector((state) =>
toSupportedLocale(state.meta.get('locale') as string),
);
const mode = useAppSelector(modeSelector);
return {
currentLocale: locale,
locales: [locale],
mode,
darkTheme: document.body.classList.contains('theme-default'),
};
}
type Feature = Uint8ClampedArray;

View File

@@ -1,101 +1,12 @@
import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
import { EMOJI_MODE_TWEMOJI } from './constants';
import * as db from './database';
import * as loader from './loader';
import {
emojifyElement,
emojifyText,
testCacheClear,
loadEmojiDataToState,
stringToEmojiState,
tokenizeText,
} from './render';
import type { EmojiAppState } from './types';
function mockDatabase() {
return {
searchCustomEmojisByShortcodes: vi
.spyOn(db, 'searchCustomEmojisByShortcodes')
.mockResolvedValue([customEmojiFactory()]),
searchEmojisByHexcodes: vi
.spyOn(db, 'searchEmojisByHexcodes')
.mockResolvedValue([
unicodeEmojiFactory({
hexcode: '1F60A',
label: 'smiling face with smiling eyes',
unicode: '😊',
}),
unicodeEmojiFactory({
hexcode: '1F1EA-1F1FA',
label: 'flag-eu',
unicode: '🇪🇺',
}),
]),
};
}
const expectedSmileImage =
'<img draggable="false" class="emojione" alt="😊" title="smiling face with smiling eyes" src="/emoji/1f60a.svg">';
const expectedFlagImage =
'<img draggable="false" class="emojione" alt="🇪🇺" title="flag-eu" src="/emoji/1f1ea-1f1fa.svg">';
function testAppState(state: Partial<EmojiAppState> = {}) {
return {
locales: ['en'],
mode: EMOJI_MODE_TWEMOJI,
currentLocale: 'en',
darkTheme: false,
...state,
} satisfies EmojiAppState;
}
describe('emojifyElement', () => {
function testElement(text = '<p>Hello 😊🇪🇺!</p><p>:custom:</p>') {
const testElement = document.createElement('div');
testElement.innerHTML = text;
return testElement;
}
afterEach(() => {
testCacheClear();
vi.restoreAllMocks();
});
test('caches element rendering results', async () => {
const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } =
mockDatabase();
await emojifyElement(testElement(), testAppState());
await emojifyElement(testElement(), testAppState());
await emojifyElement(testElement(), testAppState());
expect(searchEmojisByHexcodes).toHaveBeenCalledExactlyOnceWith(
['1F1EA-1F1FA', '1F60A'],
'en',
);
expect(searchCustomEmojisByShortcodes).toHaveBeenCalledExactlyOnceWith([
':custom:',
]);
});
test('returns null when no emoji are found', async () => {
mockDatabase();
const actual = await emojifyElement(
testElement('<p>here is just text :)</p>'),
testAppState(),
);
expect(actual).toBeNull();
});
});
describe('emojifyText', () => {
test('returns original input when no emoji are in string', async () => {
const actual = await emojifyText('nothing here', testAppState());
expect(actual).toBe('nothing here');
});
test('renders Unicode emojis to twemojis', async () => {
mockDatabase();
const actual = await emojifyText('Hello 😊🇪🇺!', testAppState());
expect(actual).toBe(`Hello ${expectedSmileImage}${expectedFlagImage}!`);
});
});
describe('tokenizeText', () => {
test('returns an array of text to be a single token', () => {
@@ -162,3 +73,106 @@ describe('tokenizeText', () => {
]);
});
});
describe('stringToEmojiState', () => {
test('returns unicode emoji state for valid unicode emoji', () => {
expect(stringToEmojiState('😊')).toEqual({
type: 'unicode',
code: '1F60A',
});
});
test('returns custom emoji state for valid custom emoji', () => {
expect(stringToEmojiState(':smile:')).toEqual({
type: 'custom',
code: 'smile',
data: undefined,
});
});
test('returns custom emoji state with data when provided', () => {
const customEmoji = {
smile: customEmojiFactory({
shortcode: 'smile',
url: 'https://example.com/smile.png',
static_url: 'https://example.com/smile_static.png',
}),
};
expect(stringToEmojiState(':smile:', customEmoji)).toEqual({
type: 'custom',
code: 'smile',
data: customEmoji.smile,
});
});
test('returns null for invalid emoji strings', () => {
expect(stringToEmojiState('notanemoji')).toBeNull();
expect(stringToEmojiState(':invalid-emoji:')).toBeNull();
});
});
describe('loadEmojiDataToState', () => {
beforeEach(() => {
vi.clearAllMocks();
});
test('loads unicode data into state', async () => {
const dbCall = vi
.spyOn(db, 'loadEmojiByHexcode')
.mockResolvedValue(unicodeEmojiFactory());
const unicodeState = { type: 'unicode', code: '1F60A' } as const;
const result = await loadEmojiDataToState(unicodeState, 'en');
expect(dbCall).toHaveBeenCalledWith('1F60A', 'en');
expect(result).toEqual({
type: 'unicode',
code: '1F60A',
data: unicodeEmojiFactory(),
});
});
test('loads custom emoji data into state', async () => {
const dbCall = vi
.spyOn(db, 'loadCustomEmojiByShortcode')
.mockResolvedValueOnce(customEmojiFactory());
const customState = { type: 'custom', code: 'smile' } as const;
const result = await loadEmojiDataToState(customState, 'en');
expect(dbCall).toHaveBeenCalledWith('smile');
expect(result).toEqual({
type: 'custom',
code: 'smile',
data: customEmojiFactory(),
});
});
test('returns null if unicode emoji not found in database', async () => {
vi.spyOn(db, 'loadEmojiByHexcode').mockResolvedValueOnce(undefined);
const unicodeState = { type: 'unicode', code: '1F60A' } as const;
const result = await loadEmojiDataToState(unicodeState, 'en');
expect(result).toBeNull();
});
test('returns null if custom emoji not found in database', async () => {
vi.spyOn(db, 'loadCustomEmojiByShortcode').mockResolvedValueOnce(undefined);
const customState = { type: 'custom', code: 'smile' } as const;
const result = await loadEmojiDataToState(customState, 'en');
expect(result).toBeNull();
});
test('retries loading emoji data once if initial load fails', async () => {
const dbCall = vi
.spyOn(db, 'loadEmojiByHexcode')
.mockRejectedValue(new db.LocaleNotLoadedError('en'));
vi.spyOn(loader, 'importEmojiData').mockResolvedValueOnce();
const consoleCall = vi
.spyOn(console, 'warn')
.mockImplementationOnce(() => null);
const unicodeState = { type: 'unicode', code: '1F60A' } as const;
const result = await loadEmojiDataToState(unicodeState, 'en');
expect(dbCall).toHaveBeenCalledTimes(2);
expect(loader.importEmojiData).toHaveBeenCalledWith('en');
expect(consoleCall).toHaveBeenCalled();
expect(result).toBeNull();
});
});

View File

@@ -1,7 +1,3 @@
import { autoPlayGif } from '@/mastodon/initial_state';
import { createLimitedCache } from '@/mastodon/utils/cache';
import * as perf from '@/mastodon/utils/performance';
import {
EMOJI_MODE_NATIVE,
EMOJI_MODE_NATIVE_WITH_FLAGS,
@@ -12,33 +8,69 @@ import {
loadCustomEmojiByShortcode,
loadEmojiByHexcode,
LocaleNotLoadedError,
searchCustomEmojisByShortcodes,
searchEmojisByHexcodes,
} from './database';
import { importEmojiData } from './loader';
import { emojiToUnicodeHex, unicodeHexToUrl } from './normalize';
import { emojiToUnicodeHex } from './normalize';
import type {
EmojiAppState,
EmojiLoadedState,
EmojiMode,
EmojiState,
EmojiStateCustom,
EmojiStateMap,
EmojiStateUnicode,
ExtraCustomEmojiMap,
LocaleOrCustom,
} from './types';
import {
anyEmojiRegex,
emojiLogger,
isCustomEmoji,
isUnicodeEmoji,
stringHasAnyEmoji,
stringHasUnicodeFlags,
} from './utils';
const log = emojiLogger('render');
type TokenizedText = (string | EmojiState)[];
/**
* Tokenizes text into strings and emoji states.
* @param text Text to tokenize.
* @returns Array of strings and emoji states.
*/
export function tokenizeText(text: string): TokenizedText {
if (!text.trim()) {
return [text];
}
const tokens = [];
let lastIndex = 0;
for (const match of text.matchAll(anyEmojiRegex())) {
if (match.index > lastIndex) {
tokens.push(text.slice(lastIndex, match.index));
}
const code = match[0];
if (code.startsWith(':') && code.endsWith(':')) {
// Custom emoji
tokens.push({
type: EMOJI_TYPE_CUSTOM,
code,
} satisfies EmojiStateCustom);
} else {
// Unicode emoji
tokens.push({
type: EMOJI_TYPE_UNICODE,
code: code,
} satisfies EmojiStateUnicode);
}
lastIndex = match.index + code.length;
}
if (lastIndex < text.length) {
tokens.push(text.slice(lastIndex));
}
return tokens;
}
/**
* Parses emoji string to extract emoji state.
* @param code Hex code or custom shortcode.
@@ -132,305 +164,19 @@ export function isStateLoaded(state: EmojiState): state is EmojiLoadedState {
}
/**
* Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions.
* Determines if the given token should be rendered as an image based on the emoji mode.
* @param state Emoji state to parse.
* @param mode Rendering mode.
* @returns Whether to render as an image.
*/
export async function emojifyElement<Element extends HTMLElement>(
element: Element,
appState: EmojiAppState,
extraEmojis: ExtraCustomEmojiMap = {},
): Promise<Element | null> {
const cacheKey = createCacheKey(element, appState, extraEmojis);
const cached = getCached(cacheKey);
if (cached !== undefined) {
log('Cache hit on %s', element.outerHTML);
if (cached === null) {
return null;
}
element.innerHTML = cached;
return element;
}
if (!stringHasAnyEmoji(element.innerHTML)) {
updateCache(cacheKey, null);
return null;
}
perf.start('emojifyElement()');
const queue: (HTMLElement | Text)[] = [element];
while (queue.length > 0) {
const current = queue.shift();
if (
!current ||
current instanceof HTMLScriptElement ||
current instanceof HTMLStyleElement
) {
continue;
}
if (
current.textContent &&
(current instanceof Text || !current.hasChildNodes())
) {
const renderedContent = await textToElementArray(
current.textContent,
appState,
extraEmojis,
);
if (renderedContent) {
if (!(current instanceof Text)) {
current.textContent = null; // Clear the text content if it's not a Text node.
}
current.replaceWith(renderedToHTML(renderedContent));
}
continue;
}
for (const child of current.childNodes) {
if (child instanceof HTMLElement || child instanceof Text) {
queue.push(child);
}
}
}
updateCache(cacheKey, element.innerHTML);
perf.stop('emojifyElement()');
return element;
}
export async function emojifyText(
text: string,
appState: EmojiAppState,
extraEmojis: ExtraCustomEmojiMap = {},
): Promise<string | null> {
const cacheKey = createCacheKey(text, appState, extraEmojis);
const cached = getCached(cacheKey);
if (cached !== undefined) {
log('Cache hit on %s', text);
return cached ?? text;
}
if (!stringHasAnyEmoji(text)) {
updateCache(cacheKey, null);
return text;
}
const eleArray = await textToElementArray(text, appState, extraEmojis);
if (!eleArray) {
updateCache(cacheKey, null);
return text;
}
const rendered = renderedToHTML(eleArray, document.createElement('div'));
updateCache(cacheKey, rendered.innerHTML);
return rendered.innerHTML;
}
// Private functions
const {
set: updateCache,
get: getCached,
clear: cacheClear,
} = createLimitedCache<string | null>({ log: log.extend('cache') });
function createCacheKey(
input: HTMLElement | string,
appState: EmojiAppState,
extraEmojis: ExtraCustomEmojiMap,
) {
return JSON.stringify([
input instanceof HTMLElement ? input.outerHTML : input,
appState,
extraEmojis,
]);
}
type EmojifiedTextArray = (string | HTMLImageElement)[];
async function textToElementArray(
text: string,
appState: EmojiAppState,
extraEmojis: ExtraCustomEmojiMap = {},
): Promise<EmojifiedTextArray | null> {
// Exit if no text to convert.
if (!text.trim()) {
return null;
}
const tokens = tokenizeText(text);
// If only one token and it's a string, exit early.
if (tokens.length === 1 && typeof tokens[0] === 'string') {
return null;
}
// Get all emoji from the state map, loading any missing ones.
await loadMissingEmojiIntoCache(tokens, appState, extraEmojis);
const renderedFragments: EmojifiedTextArray = [];
for (const token of tokens) {
if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) {
let state: EmojiState | undefined;
if (token.type === EMOJI_TYPE_CUSTOM) {
const extraEmojiData = extraEmojis[token.code];
if (extraEmojiData) {
state = {
type: EMOJI_TYPE_CUSTOM,
data: extraEmojiData,
code: token.code,
};
} else {
state = emojiForLocale(token.code, EMOJI_TYPE_CUSTOM);
}
} else {
state = emojiForLocale(
emojiToUnicodeHex(token.code),
appState.currentLocale,
);
}
// If the state is valid, create an image element. Otherwise, just append as text.
if (state && typeof state !== 'string' && isStateLoaded(state)) {
const image = stateToImage(state, appState);
renderedFragments.push(image);
continue;
}
}
const text = typeof token === 'string' ? token : token.code;
renderedFragments.push(text);
}
return renderedFragments;
}
type TokenizedText = (string | EmojiState)[];
export function tokenizeText(text: string): TokenizedText {
if (!text.trim()) {
return [text];
}
const tokens = [];
let lastIndex = 0;
for (const match of text.matchAll(anyEmojiRegex())) {
if (match.index > lastIndex) {
tokens.push(text.slice(lastIndex, match.index));
}
const code = match[0];
if (code.startsWith(':') && code.endsWith(':')) {
// Custom emoji
tokens.push({
type: EMOJI_TYPE_CUSTOM,
code,
} satisfies EmojiStateCustom);
} else {
// Unicode emoji
tokens.push({
type: EMOJI_TYPE_UNICODE,
code: code,
} satisfies EmojiStateUnicode);
}
lastIndex = match.index + code.length;
}
if (lastIndex < text.length) {
tokens.push(text.slice(lastIndex));
}
return tokens;
}
const localeCacheMap = new Map<LocaleOrCustom, EmojiStateMap>([
[
EMOJI_TYPE_CUSTOM,
createLimitedCache<EmojiState>({ log: log.extend('custom') }),
],
]);
function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap {
return (
localeCacheMap.get(locale) ??
createLimitedCache<EmojiState>({ log: log.extend(locale) })
);
}
function emojiForLocale(
code: string,
locale: LocaleOrCustom,
): EmojiState | undefined {
const cache = cacheForLocale(locale);
return cache.get(code);
}
async function loadMissingEmojiIntoCache(
tokens: TokenizedText,
{ mode, currentLocale }: EmojiAppState,
extraEmojis: ExtraCustomEmojiMap,
) {
const missingUnicodeEmoji = new Set<string>();
const missingCustomEmoji = new Set<string>();
// Iterate over tokens and check if they are in the cache already.
for (const token of tokens) {
if (typeof token === 'string') {
continue; // Skip plain strings.
}
// If this is a custom emoji, check it separately.
if (token.type === EMOJI_TYPE_CUSTOM) {
const code = token.code;
if (code in extraEmojis) {
continue; // We don't care about extra emoji.
}
const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM);
if (!emojiState) {
missingCustomEmoji.add(code);
}
// Otherwise this is a unicode emoji, so check it against all locales.
} else if (shouldRenderImage(token, mode)) {
const code = emojiToUnicodeHex(token.code);
if (missingUnicodeEmoji.has(code)) {
continue; // Already marked as missing.
}
const emojiState = emojiForLocale(code, currentLocale);
if (!emojiState) {
// If it's missing in one locale, we consider it missing for all.
missingUnicodeEmoji.add(code);
}
}
}
if (missingUnicodeEmoji.size > 0) {
const missingEmojis = Array.from(missingUnicodeEmoji).toSorted();
const emojis = await searchEmojisByHexcodes(missingEmojis, currentLocale);
const cache = cacheForLocale(currentLocale);
for (const emoji of emojis) {
cache.set(emoji.hexcode, {
type: EMOJI_TYPE_UNICODE,
data: emoji,
code: emoji.hexcode,
});
}
localeCacheMap.set(currentLocale, cache);
}
if (missingCustomEmoji.size > 0) {
const missingEmojis = Array.from(missingCustomEmoji).toSorted();
const emojis = await searchCustomEmojisByShortcodes(missingEmojis);
const cache = cacheForLocale(EMOJI_TYPE_CUSTOM);
for (const emoji of emojis) {
cache.set(emoji.shortcode, {
type: EMOJI_TYPE_CUSTOM,
data: emoji,
code: emoji.shortcode,
});
}
localeCacheMap.set(EMOJI_TYPE_CUSTOM, cache);
}
}
export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean {
if (token.type === EMOJI_TYPE_UNICODE) {
export function shouldRenderImage(state: EmojiState, mode: EmojiMode): boolean {
if (state.type === EMOJI_TYPE_UNICODE) {
// If the mode is native or native with flags for non-flag emoji
// we can just append the text node directly.
if (
mode === EMOJI_MODE_NATIVE ||
(mode === EMOJI_MODE_NATIVE_WITH_FLAGS &&
!stringHasUnicodeFlags(token.code))
!stringHasUnicodeFlags(state.code))
) {
return false;
}
@@ -438,52 +184,3 @@ export function shouldRenderImage(token: EmojiState, mode: EmojiMode): boolean {
return true;
}
function stateToImage(state: EmojiLoadedState, appState: EmojiAppState) {
const image = document.createElement('img');
image.draggable = false;
image.classList.add('emojione');
if (state.type === EMOJI_TYPE_UNICODE) {
image.alt = state.data.unicode;
image.title = state.data.label;
image.src = unicodeHexToUrl(state.data.hexcode, appState.darkTheme);
} else {
// Custom emoji
const shortCode = `:${state.data.shortcode}:`;
image.classList.add('custom-emoji');
image.alt = shortCode;
image.title = shortCode;
image.src = autoPlayGif ? state.data.url : state.data.static_url;
image.dataset.original = state.data.url;
image.dataset.static = state.data.static_url;
}
return image;
}
function renderedToHTML(renderedArray: EmojifiedTextArray): DocumentFragment;
function renderedToHTML<ParentType extends ParentNode>(
renderedArray: EmojifiedTextArray,
parent: ParentType,
): ParentType;
function renderedToHTML(
renderedArray: EmojifiedTextArray,
parent: ParentNode | null = null,
) {
const fragment = parent ?? document.createDocumentFragment();
for (const fragmentItem of renderedArray) {
if (typeof fragmentItem === 'string') {
fragment.appendChild(document.createTextNode(fragmentItem));
} else if (fragmentItem instanceof HTMLImageElement) {
fragment.appendChild(fragmentItem);
}
}
return fragment;
}
// Testing helpers
export const testCacheClear = () => {
cacheClear();
localeCacheMap.clear();
};

View File

@@ -4,7 +4,6 @@ import type { FlatCompactEmoji, Locale } from 'emojibase';
import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
import type { LimitedCache } from '@/mastodon/utils/cache';
import type {
EMOJI_MODE_NATIVE,
@@ -48,12 +47,11 @@ export interface EmojiStateCustom {
data?: CustomEmojiRenderFields;
}
export type EmojiState = EmojiStateUnicode | EmojiStateCustom;
export type EmojiLoadedState =
| Required<EmojiStateUnicode>
| Required<EmojiStateCustom>;
export type EmojiStateMap = LimitedCache<string, EmojiState>;
export type CustomEmojiMapArg =
| ExtraCustomEmojiMap
| ImmutableList<CustomEmoji>
@@ -64,9 +62,3 @@ export type ExtraCustomEmojiMap = Record<
string,
Pick<CustomEmojiData, 'shortcode' | 'static_url' | 'url'>
>;
export interface TwemojiBorderInfo {
hexCode: string;
hasLightBorder: boolean;
hasDarkBorder: boolean;
}

View File

@@ -1,35 +1,31 @@
import {
stringHasAnyEmoji,
stringHasCustomEmoji,
stringHasUnicodeEmoji,
stringHasUnicodeFlags,
} from './utils';
import { isCustomEmoji, isUnicodeEmoji, stringHasUnicodeFlags } from './utils';
describe('stringHasUnicodeEmoji', () => {
describe('isUnicodeEmoji', () => {
test.concurrent.for([
['only text', false],
['text with non-emoji symbols ™©', false],
['text with emoji 😀', true],
['multiple emojis 😀😃😄', true],
['emoji with skin tone 👍🏽', true],
['emoji with ZWJ 👩‍❤️‍👨', true],
['emoji with variation selector ✊️', true],
['emoji with keycap 1⃣', true],
['emoji with flags 🇺🇸', true],
['emoji with regional indicators 🇦🇺', true],
['emoji with gender 👩‍⚕️', true],
['emoji with family 👨‍👩‍👧‍👦', true],
['emoji with zero width joiner 👩‍🔬', true],
['emoji with non-BMP codepoint 🧑‍🚀', true],
['emoji with combining marks 👨‍👩‍👧‍👦', true],
['emoji with enclosing keycap #️⃣', true],
['emoji with no visible glyph \u200D', false],
] as const)(
'stringHasUnicodeEmoji has emojis in "%s": %o',
([text, expected], { expect }) => {
expect(stringHasUnicodeEmoji(text)).toBe(expected);
},
);
['😊', true],
['🇿🇼', true],
['🏴‍☠️', true],
['🏳️‍🌈', true],
['foo', false],
[':smile:', false],
['😊foo', false],
] as const)('isUnicodeEmoji("%s") is %o', ([input, expected], { expect }) => {
expect(isUnicodeEmoji(input)).toBe(expected);
});
});
describe('isCustomEmoji', () => {
test.concurrent.for([
[':smile:', true],
[':smile_123:', true],
[':SMILE:', true],
['😊', false],
['foo', false],
[':smile', false],
['smile:', false],
] as const)('isCustomEmoji("%s") is %o', ([input, expected], { expect }) => {
expect(isCustomEmoji(input)).toBe(expected);
});
});
describe('stringHasUnicodeFlags', () => {
@@ -51,27 +47,3 @@ describe('stringHasUnicodeFlags', () => {
},
);
});
describe('stringHasCustomEmoji', () => {
test('string with custom emoji returns true', () => {
expect(stringHasCustomEmoji(':custom: :test:')).toBeTruthy();
});
test('string without custom emoji returns false', () => {
expect(stringHasCustomEmoji('🏳️‍🌈 :🏳️‍🌈: text ™')).toBeFalsy();
});
});
describe('stringHasAnyEmoji', () => {
test('string without any emoji or characters', () => {
expect(stringHasAnyEmoji('normal text. 12356?!')).toBeFalsy();
});
test('string with non-emoji characters', () => {
expect(stringHasAnyEmoji('™©')).toBeFalsy();
});
test('has unicode emoji', () => {
expect(stringHasAnyEmoji('🏳️‍🌈🔥🇸🇹 👩‍🔬')).toBeTruthy();
});
test('has custom emoji', () => {
expect(stringHasAnyEmoji(':test: :custom:')).toBeTruthy();
});
});

View File

@@ -6,10 +6,6 @@ export function emojiLogger(segment: string) {
return debug(`emojis:${segment}`);
}
export function stringHasUnicodeEmoji(input: string): boolean {
return new RegExp(EMOJI_REGEX, supportedFlags()).test(input);
}
export function isUnicodeEmoji(input: string): boolean {
return (
input.length > 0 &&
@@ -34,19 +30,13 @@ export function stringHasUnicodeFlags(input: string): boolean {
// Constant as this is supported by all browsers.
const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i;
// Use the polyfill regex or the Unicode property escapes if supported.
const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}';
export function isCustomEmoji(input: string): boolean {
return new RegExp(`^${CUSTOM_EMOJI_REGEX.source}$`, 'i').test(input);
}
export function stringHasCustomEmoji(input: string) {
return CUSTOM_EMOJI_REGEX.test(input);
}
export function stringHasAnyEmoji(input: string) {
return stringHasUnicodeEmoji(input) || stringHasCustomEmoji(input);
}
export function anyEmojiRegex() {
return new RegExp(
`${EMOJI_REGEX}|${CUSTOM_EMOJI_REGEX.source}`,
@@ -64,5 +54,3 @@ function supportedFlags(flags = '') {
}
return flags;
}
const EMOJI_REGEX = emojiRegexPolyfill?.source ?? '\\p{RGI_Emoji}';

View File

@@ -1,7 +1,10 @@
import type { CompactEmoji } from 'emojibase';
import { http, HttpResponse } from 'msw';
import { action } from 'storybook/actions';
import { relationshipsFactory } from './factories';
import { toSupportedLocale } from '@/mastodon/features/emoji/locale';
import { customEmojiFactory, relationshipsFactory } from './factories';
export const mockHandlers = {
mute: http.post<{ id: string }>('/api/v1/accounts/:id/mute', ({ params }) => {
@@ -40,6 +43,24 @@ export const mockHandlers = {
);
},
),
emojiCustomData: http.get('/api/v1/custom_emojis', () => {
action('fetching custom emoji data')();
return HttpResponse.json([customEmojiFactory()]);
}),
emojiData: http.get<{ locale: string }>(
'/packs-dev/emoji/:locale.json',
async ({ params }) => {
const locale = toSupportedLocale(params.locale);
action('fetching emoji data')(locale);
const { default: data } = (await import(
`emojibase-data/${locale}/compact.json`
)) as {
default: CompactEmoji[];
};
return HttpResponse.json([data]);
},
),
};
export const unhandledRequestHandler = ({ url }: Request) => {