techhub/mastodon
2
0
Fork 0
Code Activity

Emoji Indexing and Search (#35253)

Browse Source
This commit is contained in:
Echo
2025-07-09 11:55:41 +02:00
committed by GitHub
parent 76c1446416
commit a1e8813522
12 changed files with 632 additions and 171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
app/javascript/mastodon/features/emoji/constants.ts Normal file
View File

@@ -0,0 +1,110 @@
// Utility codes
export const VARIATION_SELECTOR_CODE = 0xfe0f;
export const KEYCAP_CODE = 0x20e3;
// Gender codes
export const GENDER_FEMALE_CODE = 0x2640;
export const GENDER_MALE_CODE = 0x2642;
// Skin tone codes
export const SKIN_TONE_CODES = [
0x1f3fb, // Light skin tone
0x1f3fc, // Medium-light skin tone
0x1f3fd, // Medium skin tone
0x1f3fe, // Medium-dark skin tone
0x1f3ff, // Dark skin tone
] as const;
export const EMOJIS_WITH_DARK_BORDER = [
'🎱', // 1F3B1
'🐜', // 1F41C
'⚫', // 26AB
'🖤', // 1F5A4
'⬛', // 2B1B
'◼️', // 25FC-FE0F
'◾', // 25FE
'◼️', // 25FC-FE0F
'✒️', // 2712-FE0F
'▪️', // 25AA-FE0F
'💣', // 1F4A3
'🎳', // 1F3B3
'📷', // 1F4F7
'📸', // 1F4F8
'♣️', // 2663-FE0F
'🕶️', // 1F576-FE0F
'✴️', // 2734-FE0F
'🔌', // 1F50C
'💂‍♀️', // 1F482-200D-2640-FE0F
'📽️', // 1F4FD-FE0F
'🍳', // 1F373
'🦍', // 1F98D
'💂', // 1F482
'🔪', // 1F52A
'🕳️', // 1F573-FE0F
'🕹️', // 1F579-FE0F
'🕋', // 1F54B
'🖊️', // 1F58A-FE0F
'🖋️', // 1F58B-FE0F
'💂‍♂️', // 1F482-200D-2642-FE0F
'🎤', // 1F3A4
'🎓', // 1F393
'🎥', // 1F3A5
'🎼', // 1F3BC
'♠️', // 2660-FE0F
'🎩', // 1F3A9
'🦃', // 1F983
'📼', // 1F4FC
'📹', // 1F4F9
'🎮', // 1F3AE
'🐃', // 1F403
'🏴', // 1F3F4
'🐞', // 1F41E
'🕺', // 1F57A
'📱', // 1F4F1
'📲', // 1F4F2
'🚲', // 1F6B2
'🪮', // 1FAA6
'🐦‍⬛', // 1F426-200D-2B1B
];
export const EMOJIS_WITH_LIGHT_BORDER = [
'👽', // 1F47D
'⚾', // 26BE
'🐔', // 1F414
'☁️', // 2601-FE0F
'💨', // 1F4A8
'🕊️', // 1F54A-FE0F
'👀', // 1F440
'🍥', // 1F365
'👻', // 1F47B
'🐐', // 1F410
'❕', // 2755
'❔', // 2754
'⛸️', // 26F8-FE0F
'🌩️', // 1F329-FE0F
'🔊', // 1F50A
'🔇', // 1F507
'📃', // 1F4C3
'🌧️', // 1F327-FE0F
'🐏', // 1F40F
'🍚', // 1F35A
'🍙', // 1F359
'🐓', // 1F413
'🐑', // 1F411
'💀', // 1F480
'☠️', // 2620-FE0F
'🌨️', // 1F328-FE0F
'🔉', // 1F509
'🔈', // 1F508
'💬', // 1F4AC
'💭', // 1F4AD
'🏐', // 1F3D0
'🏳️', // 1F3F3-FE0F
'⚪', // 26AA
'⬜', // 2B1C
'◽', // 25FD
'◻️', // 25FB-FE0F
'▫️', // 25AB-FE0F
'🪽', // 1FAE8
'🪿', // 1FABF
];

102
app/javascript/mastodon/features/emoji/database.ts Normal file
View File

@@ -0,0 +1,102 @@
import { SUPPORTED_LOCALES } from 'emojibase';
import type { FlatCompactEmoji, Locale } from 'emojibase';
import type { DBSchema } from 'idb';
import { openDB } from 'idb';
import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
import type { LocaleOrCustom } from './locale';
import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
interface EmojiDB extends LocaleTables, DBSchema {
custom: {
key: string;
value: ApiCustomEmojiJSON;
indexes: {
category: string;
};
};
etags: {
key: LocaleOrCustom;
value: string;
};
}
interface LocaleTable {
key: string;
value: FlatCompactEmoji;
indexes: {
group: number;
label: string;
order: number;
tags: string[];
};
}
type LocaleTables = Record<Locale, LocaleTable>;
const SCHEMA_VERSION = 1;
const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
upgrade(database) {
const customTable = database.createObjectStore('custom', {
keyPath: 'shortcode',
autoIncrement: false,
});
customTable.createIndex('category', 'category');
database.createObjectStore('etags');
for (const locale of SUPPORTED_LOCALES) {
const localeTable = database.createObjectStore(locale, {
keyPath: 'hexcode',
autoIncrement: false,
});
localeTable.createIndex('group', 'group');
localeTable.createIndex('label', 'label');
localeTable.createIndex('order', 'order');
localeTable.createIndex('tags', 'tags', { multiEntry: true });
}
},
});
export async function putEmojiData(emojis: FlatCompactEmoji[], locale: Locale) {
const trx = db.transaction(locale, 'readwrite');
await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
await trx.done;
}
export async function putCustomEmojiData(emojis: ApiCustomEmojiJSON[]) {
const trx = db.transaction('custom', 'readwrite');
await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
await trx.done;
}
export function putLatestEtag(etag: string, localeString: string) {
const locale = toSupportedLocaleOrCustom(localeString);
return db.put('etags', etag, locale);
}
export function searchEmojiByHexcode(hexcode: string, localeString: string) {
const locale = toSupportedLocale(localeString);
return db.get(locale, hexcode);
}
export function searchEmojiByTag(tag: string, localeString: string) {
const locale = toSupportedLocale(localeString);
const range = IDBKeyRange.only(tag.toLowerCase());
return db.getAllFromIndex(locale, 'tags', range);
}
export function searchCustomEmojiByShortcode(shortcode: string) {
return db.get('custom', shortcode);
}
export async function loadLatestEtag(localeString: string) {
const locale = toSupportedLocaleOrCustom(localeString);
const rowCount = await db.count(locale);
if (!rowCount) {
return null; // No data for this locale, return null even if there is an etag.
}
const etag = await db.get('etags', locale);
return etag ?? null;
}

38
app/javascript/mastodon/features/emoji/index.ts Normal file
View File

@@ -0,0 +1,38 @@
import initialState from '@/mastodon/initial_state';
import { toSupportedLocale } from './locale';
const serverLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
const worker =
'Worker' in window
? new Worker(new URL('./worker', import.meta.url), {
type: 'module',
})
: null;
export async function initializeEmoji() {
if (worker) {
worker.addEventListener('message', (event: MessageEvent<string>) => {
const { data: message } = event;
if (message === 'ready') {
worker.postMessage(serverLocale);
worker.postMessage('custom');
}
});
} else {
const { importCustomEmojiData, importEmojiData } = await import('./loader');
await Promise.all([importCustomEmojiData(), importEmojiData(serverLocale)]);
}
}
export async function loadEmojiLocale(localeString: string) {
const locale = toSupportedLocale(localeString);
if (worker) {
worker.postMessage(locale);
} else {
const { importEmojiData } = await import('./loader');
await importEmojiData(locale);
}
}

77
app/javascript/mastodon/features/emoji/loader.ts Normal file
View File

@@ -0,0 +1,77 @@
import { flattenEmojiData } from 'emojibase';
import type { CompactEmoji, FlatCompactEmoji } from 'emojibase';
import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
import { isDevelopment } from '@/mastodon/utils/environment';
import {
putEmojiData,
putCustomEmojiData,
loadLatestEtag,
putLatestEtag,
} from './database';
import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
import type { LocaleOrCustom } from './locale';
export async function importEmojiData(localeString: string) {
const locale = toSupportedLocale(localeString);
const emojis = await fetchAndCheckEtag<CompactEmoji[]>(locale);
if (!emojis) {
return;
}
const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(emojis);
await putEmojiData(flattenedEmojis, locale);
}
export async function importCustomEmojiData() {
const emojis = await fetchAndCheckEtag<ApiCustomEmojiJSON[]>('custom');
if (!emojis) {
return;
}
await putCustomEmojiData(emojis);
}
async function fetchAndCheckEtag<ResultType extends object[]>(
localeOrCustom: LocaleOrCustom,
): Promise<ResultType | null> {
const locale = toSupportedLocaleOrCustom(localeOrCustom);
let uri: string;
if (locale === 'custom') {
uri = '/api/v1/custom_emojis';
} else {
uri = `/packs${isDevelopment() ? '-dev' : ''}/emoji/${locale}.json`;
}
const oldEtag = await loadLatestEtag(locale);
const response = await fetch(uri, {
headers: {
'Content-Type': 'application/json',
'If-None-Match': oldEtag ?? '', // Send the old ETag to check for modifications
},
});
// If not modified, return null
if (response.status === 304) {
return null;
}
if (!response.ok) {
throw new Error(
`Failed to fetch emoji data for ${localeOrCustom}: ${response.statusText}`,
);
}
const data = (await response.json()) as ResultType;
if (!Array.isArray(data)) {
throw new Error(
`Unexpected data format for ${localeOrCustom}: expected an array`,
);
}
// Store the ETag for future requests
const etag = response.headers.get('ETag');
if (etag) {
await putLatestEtag(etag, localeOrCustom);
}
return data;
}

61
app/javascript/mastodon/features/emoji/locale.test.ts
View File

@@ -1,52 +1,6 @@
import { flattenEmojiData, SUPPORTED_LOCALES } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase';
import emojiEnData from 'emojibase-data/en/compact.json';
import emojiFrData from 'emojibase-data/fr/compact.json';
import { toSupportedLocale, unicodeToLocaleLabel } from './locale'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
describe('unicodeToLocaleLabel', () => {
const emojiTestCases = [
'1F3CB-1F3FF-200D-2640-FE0F', // 🏋🏿‍♀️ Woman weightlifter, dark skin
'1F468-1F3FB', // 👨🏻 Man, light skin
'1F469-1F3FB-200D-2695-FE0F', // 👩🏻‍⚕️ Woman health worker, light skin
'1F468-1F3FD-200D-1F692', // 👨🏽‍🚒 Man firefighter, medium skin
'1F469-1F3FE', // 👩🏾 Woman, medium-dark skin
'1F469-1F3FF-200D-1F4BB', // 👩🏿‍💻 Woman technologist, dark skin
'1F478-1F3FF', // 👸🏿 Princess with dark skin tone
'1F935-1F3FC-200D-2640-FE0F', // 🤵🏼‍♀️ Woman in tuxedo, medium-light skin
'1F9D1-1F3FC', // 🧑🏼 Person, medium-light skin
'1F9D4-1F3FE', // 🧔🏾 Person with beard, medium-dark skin
];
const flattenedEnData = flattenEmojiData(emojiEnData);
const flattenedFrData = flattenEmojiData(emojiFrData);
const emojiTestEnLabels = new Map(
emojiTestCases.map((code) => [
code,
flattenedEnData.find((emoji) => emoji.hexcode === code)?.label,
]),
);
const emojiTestFrLabels = new Map(
emojiTestCases.map((code) => [
code,
flattenedFrData.find((emoji) => emoji.hexcode === code)?.label,
]),
);
test.for(
emojiTestCases.flatMap((code) => [
[code, 'en', emojiTestEnLabels.get(code)],
[code, 'fr', emojiTestFrLabels.get(code)],
]) satisfies [string, string, string | undefined][],
)(
'returns correct label for %s for %s locale',
async ([unicodeHex, locale, expectedLabel]) => {
const label = await unicodeToLocaleLabel(unicodeHex, locale);
expect(label).toBe(expectedLabel);
},
);
});
describe('toSupportedLocale', () => { describe('toSupportedLocale', () => {
test('returns the same locale if it is supported', () => { test('returns the same locale if it is supported', () => {
@@ -62,3 +16,14 @@ describe('toSupportedLocale', () => {
} }
}); });
}); });
describe('toSupportedLocaleOrCustom', () => {
test('returns custom for "custom" locale', () => {
expect(toSupportedLocaleOrCustom('custom')).toBe('custom');
});
test('returns supported locale for valid locales', () => {
for (const locale of SUPPORTED_LOCALES) {
expect(toSupportedLocaleOrCustom(locale)).toBe(locale);
}
});
});

56
app/javascript/mastodon/features/emoji/locale.ts
View File

@@ -1,51 +1,23 @@
import type { CompactEmoji, Locale } from 'emojibase'; import type { Locale } from 'emojibase';
import { flattenEmojiData, SUPPORTED_LOCALES } from 'emojibase'; import { SUPPORTED_LOCALES } from 'emojibase';
// Simple cache. This will be replaced with an IndexedDB cache in the future. export type LocaleOrCustom = Locale | 'custom';
const localeCache = new Map<Locale, Map<string, CompactEmoji>>();
export async function unicodeToLocaleLabel( export function toSupportedLocale(localeBase: string): Locale {
unicodeHex: string, const locale = localeBase.toLowerCase();
localeString: string,
) {
const locale = toSupportedLocale(localeString);
let hexMap = localeCache.get(locale);
if (!hexMap) {
hexMap = await loadLocaleLabels(locale);
localeCache.set(locale, hexMap);
}
const label = hexMap.get(unicodeHex)?.label;
if (!label) {
throw new Error(
`Label for unicode hex ${unicodeHex} not found in locale ${locale}`,
);
}
return label;
}
async function loadLocaleLabels(
locale: Locale,
): Promise<Map<string, CompactEmoji>> {
const { default: localeEmoji } = ((await import(
`emojibase-data/${locale}/compact.json`
)) ?? { default: [] }) as { default: CompactEmoji[] };
if (!Array.isArray(localeEmoji)) {
throw new Error(`Locale data for ${locale} not found`);
}
const hexMapEntries = flattenEmojiData(localeEmoji).map(
(emoji) => [emoji.hexcode, emoji] satisfies [string, CompactEmoji],
);
return new Map(hexMapEntries);
}
export function toSupportedLocale(locale: string): Locale {
if (isSupportedLocale(locale)) { if (isSupportedLocale(locale)) {
return locale; return locale;
} }
return 'en'; // Default to English if unsupported return 'en'; // Default to English if unsupported
} }
function isSupportedLocale(locale: string): locale is Locale { export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom {
return SUPPORTED_LOCALES.includes(locale as Locale); if (locale.toLowerCase() === 'custom') {
return 'custom';
}
return toSupportedLocale(locale);
}
function isSupportedLocale(locale: string): locale is Locale {
return SUPPORTED_LOCALES.includes(locale.toLowerCase() as Locale);
} }

135
app/javascript/mastodon/features/emoji/normalize.test.ts
View File

@@ -1,9 +1,17 @@
import { readdir } from 'fs/promises'; import { readdir } from 'fs/promises';
import { basename, resolve } from 'path'; import { basename, resolve } from 'path';
import unicodeEmojis from 'emojibase-data/en/data.json'; import { flattenEmojiData } from 'emojibase';
import unicodeRawEmojis from 'emojibase-data/en/data.json';
import { twemojiToUnicodeInfo, unicodeToTwemojiHex } from './normalize'; import {
twemojiHasBorder,
twemojiToUnicodeInfo,
unicodeToTwemojiHex,
CODES_WITH_DARK_BORDER,
CODES_WITH_LIGHT_BORDER,
emojiToUnicodeHex,
} from './normalize';
const emojiSVGFiles = await readdir( const emojiSVGFiles = await readdir(
// This assumes tests are run from project root // This assumes tests are run from project root
@@ -13,60 +21,81 @@ const emojiSVGFiles = await readdir(
}, },
); );
const svgFileNames = emojiSVGFiles const svgFileNames = emojiSVGFiles
.filter( .filter((file) => file.isFile() && file.name.endsWith('.svg'))
(file) =>
file.isFile() &&
file.name.endsWith('.svg') &&
!file.name.endsWith('_border.svg'),
)
.map((file) => basename(file.name, '.svg').toUpperCase()); .map((file) => basename(file.name, '.svg').toUpperCase());
const svgFileNamesWithoutBorder = svgFileNames.filter(
(fileName) => !fileName.endsWith('_BORDER'),
);
describe('normalizeEmoji', () => { const unicodeEmojis = flattenEmojiData(unicodeRawEmojis);
describe('unicodeToSVGName', () => {
test.concurrent.for(
unicodeEmojis
// Our version of Twemoji only supports up to version 15.1
.filter((emoji) => emoji.version < 16)
.map((emoji) => [emoji.hexcode, emoji.label] as [string, string]),
)('verifying an emoji exists for %s (%s)', ([hexcode], { expect }) => {
const result = unicodeToTwemojiHex(hexcode);
expect(svgFileNames).toContain(result);
});
});
describe('twemojiToUnicodeInfo', () => { describe('emojiToUnicodeHex', () => {
const unicodeMap = new Map( test.concurrent.for([
unicodeEmojis.flatMap((emoji) => { ['🎱', '1F3B1'],
const base: [string, string][] = [[emoji.hexcode, emoji.label]]; ['🐜', '1F41C'],
if (emoji.skins) { ['⚫', '26AB'],
base.push( ['🖤', '1F5A4'],
...emoji.skins.map( ['💀', '1F480'],
(skin) => [skin.hexcode, skin.label] as [string, string], ['💂‍♂️', '1F482-200D-2642-FE0F'],
), ] as const)(
); 'emojiToUnicodeHex converts %s to %s',
} ([emoji, hexcode], { expect }) => {
return base; expect(emojiToUnicodeHex(emoji)).toBe(hexcode);
}), },
); );
});
test.concurrent.for(svgFileNames)( describe('unicodeToTwemojiHex', () => {
'verifying SVG file %s maps to Unicode emoji', test.concurrent.for(
(svgFileName, { expect }) => { unicodeEmojis
assert(!!svgFileName); // Our version of Twemoji only supports up to version 15.1
const result = twemojiToUnicodeInfo(svgFileName); .filter((emoji) => emoji.version < 16)
const hexcode = .map((emoji) => [emoji.hexcode, emoji.label] as [string, string]),
typeof result === 'string' ? result : result.unqualified; )('verifying an emoji exists for %s (%s)', ([hexcode], { expect }) => {
if (!hexcode) { const result = unicodeToTwemojiHex(hexcode);
// No hexcode means this is a special case like the Shibuya 109 emoji expect(svgFileNamesWithoutBorder).toContain(result);
expect(result).toHaveProperty('label');
return;
}
assert(!!hexcode);
expect(
unicodeMap.has(hexcode),
`${hexcode} (${svgFileName}) not found`,
).toBeTruthy();
},
);
}); });
}); });
describe('twemojiHasBorder', () => {
test.concurrent.for(
svgFileNames
.filter((file) => file.endsWith('_BORDER'))
.map((file) => {
const hexCode = file.replace('_BORDER', '');
return [
hexCode,
CODES_WITH_LIGHT_BORDER.includes(hexCode),
CODES_WITH_DARK_BORDER.includes(hexCode),
] as const;
}),
)('twemojiHasBorder for %s', ([hexCode, isLight, isDark], { expect }) => {
const result = twemojiHasBorder(hexCode);
expect(result).toHaveProperty('hexCode', hexCode);
expect(result).toHaveProperty('hasLightBorder', isLight);
expect(result).toHaveProperty('hasDarkBorder', isDark);
});
});
describe('twemojiToUnicodeInfo', () => {
const unicodeCodeSet = new Set(unicodeEmojis.map((emoji) => emoji.hexcode));
test.concurrent.for(svgFileNamesWithoutBorder)(
'verifying SVG file %s maps to Unicode emoji',
(svgFileName, { expect }) => {
assert(!!svgFileName);
const result = twemojiToUnicodeInfo(svgFileName);
const hexcode = typeof result === 'string' ? result : result.unqualified;
if (!hexcode) {
// No hexcode means this is a special case like the Shibuya 109 emoji
expect(result).toHaveProperty('label');
return;
}
assert(!!hexcode);
expect(
unicodeCodeSet.has(hexcode),
`${hexcode} (${svgFileName}) not found`,
).toBeTruthy();
},
);
});

78
app/javascript/mastodon/features/emoji/normalize.ts
View File

@@ -1,19 +1,12 @@
// Utility codes import {
const VARIATION_SELECTOR_CODE = 0xfe0f; VARIATION_SELECTOR_CODE,
const KEYCAP_CODE = 0x20e3; KEYCAP_CODE,
GENDER_FEMALE_CODE,
// Gender codes GENDER_MALE_CODE,
const GENDER_FEMALE_CODE = 0x2640; SKIN_TONE_CODES,
const GENDER_MALE_CODE = 0x2642; EMOJIS_WITH_DARK_BORDER,
EMOJIS_WITH_LIGHT_BORDER,
// Skin tone codes } from './constants';
const SKIN_TONE_CODES = [
0x1f3fb, // Light skin tone
0x1f3fc, // Medium-light skin tone
0x1f3fd, // Medium skin tone
0x1f3fe, // Medium-dark skin tone
0x1f3ff, // Dark skin tone
] as const;
// Misc codes that have special handling // Misc codes that have special handling
const SKIER_CODE = 0x26f7; const SKIER_CODE = 0x26f7;
@@ -24,6 +17,17 @@ const LEVITATING_PERSON_CODE = 0x1f574;
const SPEECH_BUBBLE_CODE = 0x1f5e8; const SPEECH_BUBBLE_CODE = 0x1f5e8;
const MS_CLAUS_CODE = 0x1f936; const MS_CLAUS_CODE = 0x1f936;
export function emojiToUnicodeHex(emoji: string): string {
const codes: number[] = [];
for (const char of emoji) {
const code = char.codePointAt(0);
if (code !== undefined) {
codes.push(code);
}
}
return hexNumbersToString(codes);
}
export function unicodeToTwemojiHex(unicodeHex: string): string { export function unicodeToTwemojiHex(unicodeHex: string): string {
const codes = hexStringToNumbers(unicodeHex); const codes = hexStringToNumbers(unicodeHex);
const normalizedCodes: number[] = []; const normalizedCodes: number[] = [];
@@ -50,6 +54,35 @@ export function unicodeToTwemojiHex(unicodeHex: string): string {
return hexNumbersToString(normalizedCodes, 0); return hexNumbersToString(normalizedCodes, 0);
} }
interface TwemojiBorderInfo {
hexCode: string;
hasLightBorder: boolean;
hasDarkBorder: boolean;
}
export const CODES_WITH_DARK_BORDER =
EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex);
export const CODES_WITH_LIGHT_BORDER =
EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex);
export function twemojiHasBorder(twemojiHex: string): TwemojiBorderInfo {
const normalizedHex = twemojiHex.toUpperCase();
let hasLightBorder = false;
let hasDarkBorder = false;
if (CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) {
hasLightBorder = true;
}
if (CODES_WITH_DARK_BORDER.includes(normalizedHex)) {
hasDarkBorder = true;
}
return {
hexCode: normalizedHex,
hasLightBorder,
hasDarkBorder,
};
}
interface TwemojiSpecificEmoji { interface TwemojiSpecificEmoji {
unqualified?: string; unqualified?: string;
gender?: number; gender?: number;
@@ -84,11 +117,16 @@ export function twemojiToUnicodeInfo(
let gender: undefined | number; let gender: undefined | number;
let skin: undefined | number; let skin: undefined | number;
for (const code of codes) { for (const code of codes) {
if (code in GENDER_CODES_MAP) { if (!gender && code in GENDER_CODES_MAP) {
gender = GENDER_CODES_MAP[code]; gender = GENDER_CODES_MAP[code];
} else if (code in SKIN_TONE_CODES) { } else if (!skin && code in SKIN_TONE_CODES) {
skin = code; skin = code;
} }
// Exit if we have both skin and gender
if (skin && gender) {
break;
}
} }
let mappedCodes: unknown[] = codes; let mappedCodes: unknown[] = codes;
@@ -103,8 +141,8 @@ export function twemojiToUnicodeInfo(
// For key emoji, insert the variation selector // For key emoji, insert the variation selector
mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE]; mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE];
} else if ( } else if (
codes.at(0) === SKIER_CODE || (codes.at(0) === SKIER_CODE || codes.at(0) === LEVITATING_PERSON_CODE) &&
codes.at(0) === LEVITATING_PERSON_CODE codes.length > 1
) { ) {
// Twemoji offers more gender and skin options for the skier and levitating person emoji. // Twemoji offers more gender and skin options for the skier and levitating person emoji.
return { return {

13
app/javascript/mastodon/features/emoji/worker.ts Normal file
View File

@@ -0,0 +1,13 @@
import { importEmojiData, importCustomEmojiData } from './loader';
addEventListener('message', handleMessage);
self.postMessage('ready'); // After the worker is ready, notify the main thread
function handleMessage(event: MessageEvent<string>) {
const { data: locale } = event;
if (locale !== 'custom') {
void importEmojiData(locale);
} else {
void importCustomEmojiData();
}
}

2
package.json
View File

@@ -73,6 +73,7 @@
"history": "^4.10.1", "history": "^4.10.1",
"hoist-non-react-statics": "^3.3.2", "hoist-non-react-statics": "^3.3.2",
"http-link-header": "^1.1.1", "http-link-header": "^1.1.1",
"idb": "^8.0.3",
"immutable": "^4.3.0", "immutable": "^4.3.0",
"intl-messageformat": "^10.7.16", "intl-messageformat": "^10.7.16",
"js-yaml": "^4.1.0", "js-yaml": "^4.1.0",
@@ -117,6 +118,7 @@
"vite-plugin-pwa": "^1.0.0", "vite-plugin-pwa": "^1.0.0",
"vite-plugin-rails": "^0.5.0", "vite-plugin-rails": "^0.5.0",
"vite-plugin-ruby": "^5.1.1", "vite-plugin-ruby": "^5.1.1",
"vite-plugin-static-copy": "^3.1.0",
"vite-plugin-svgr": "^4.3.0", "vite-plugin-svgr": "^4.3.0",
"vite-tsconfig-paths": "^5.1.4", "vite-tsconfig-paths": "^5.1.4",
"wicg-inert": "^3.1.2", "wicg-inert": "^3.1.2",

25
vite.config.mts
View File

@@ -1,14 +1,15 @@
import path from 'node:path'; import path from 'node:path';
import { optimizeLodashImports } from '@optimize-lodash/rollup-plugin'; import { optimizeLodashImports } from '@optimize-lodash/rollup-plugin';
import legacy from '@vitejs/plugin-legacy';
import react from '@vitejs/plugin-react'; import react from '@vitejs/plugin-react';
import { PluginOption } from 'vite'; import { PluginOption } from 'vite';
import svgr from 'vite-plugin-svgr';
import { visualizer } from 'rollup-plugin-visualizer'; import { visualizer } from 'rollup-plugin-visualizer';
import RailsPlugin from 'vite-plugin-rails';
import { VitePWA } from 'vite-plugin-pwa'; import { VitePWA } from 'vite-plugin-pwa';
import RailsPlugin from 'vite-plugin-rails';
import { viteStaticCopy } from 'vite-plugin-static-copy';
import svgr from 'vite-plugin-svgr';
import tsconfigPaths from 'vite-tsconfig-paths'; import tsconfigPaths from 'vite-tsconfig-paths';
import legacy from '@vitejs/plugin-legacy';
import { defineConfig, UserConfigFnPromise, UserConfig } from 'vite'; import { defineConfig, UserConfigFnPromise, UserConfig } from 'vite';
import postcssPresetEnv from 'postcss-preset-env'; import postcssPresetEnv from 'postcss-preset-env';
@@ -78,6 +79,9 @@ export const config: UserConfigFnPromise = async ({ mode, command }) => {
}, },
}, },
}, },
worker: {
format: 'es',
},
plugins: [ plugins: [
tsconfigPaths(), tsconfigPaths(),
RailsPlugin({ RailsPlugin({
@@ -92,6 +96,21 @@ export const config: UserConfigFnPromise = async ({ mode, command }) => {
plugins: ['formatjs', 'transform-react-remove-prop-types'], plugins: ['formatjs', 'transform-react-remove-prop-types'],
}, },
}), }),
viteStaticCopy({
targets: [
{
src: path.resolve(
__dirname,
'node_modules/emojibase-data/**/compact.json',
),
dest: 'emoji',
rename(_name, ext, dir) {
const locale = path.basename(path.dirname(dir));
return `${locale}.${ext}`;
},
},
],
}),
MastodonServiceWorkerLocales(), MastodonServiceWorkerLocales(),
MastodonEmojiCompressed(), MastodonEmojiCompressed(),
legacy({ legacy({

106
yarn.lock
View File

@@ -2686,6 +2686,7 @@ __metadata:
hoist-non-react-statics: "npm:^3.3.2" hoist-non-react-statics: "npm:^3.3.2"
http-link-header: "npm:^1.1.1" http-link-header: "npm:^1.1.1"
husky: "npm:^9.0.11" husky: "npm:^9.0.11"
idb: "npm:^8.0.3"
immutable: "npm:^4.3.0" immutable: "npm:^4.3.0"
intl-messageformat: "npm:^10.7.16" intl-messageformat: "npm:^10.7.16"
js-yaml: "npm:^4.1.0" js-yaml: "npm:^4.1.0"
@@ -2742,6 +2743,7 @@ __metadata:
vite-plugin-pwa: "npm:^1.0.0" vite-plugin-pwa: "npm:^1.0.0"
vite-plugin-rails: "npm:^0.5.0" vite-plugin-rails: "npm:^0.5.0"
vite-plugin-ruby: "npm:^5.1.1" vite-plugin-ruby: "npm:^5.1.1"
vite-plugin-static-copy: "npm:^3.1.0"
vite-plugin-svgr: "npm:^4.2.0" vite-plugin-svgr: "npm:^4.2.0"
vite-tsconfig-paths: "npm:^5.1.4" vite-tsconfig-paths: "npm:^5.1.4"
vitest: "npm:^3.2.1" vitest: "npm:^3.2.1"
@@ -5083,6 +5085,16 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"anymatch@npm:~3.1.2":
version: 3.1.3
resolution: "anymatch@npm:3.1.3"
dependencies:
normalize-path: "npm:^3.0.0"
picomatch: "npm:^2.0.4"
checksum: 10c0/57b06ae984bc32a0d22592c87384cd88fe4511b1dd7581497831c56d41939c8a001b28e7b853e1450f2bf61992dfcaa8ae2d0d161a0a90c4fb631ef07098fbac
languageName: node
linkType: hard
"are-docs-informative@npm:^0.0.2": "are-docs-informative@npm:^0.0.2":
version: 0.0.2 version: 0.0.2
resolution: "are-docs-informative@npm:0.0.2" resolution: "are-docs-informative@npm:0.0.2"
@@ -5471,6 +5483,13 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"binary-extensions@npm:^2.0.0":
version: 2.3.0
resolution: "binary-extensions@npm:2.3.0"
checksum: 10c0/75a59cafc10fb12a11d510e77110c6c7ae3f4ca22463d52487709ca7f18f69d886aa387557cc9864fbdb10153d0bdb4caacabf11541f55e89ed6e18d12ece2b5
languageName: node
linkType: hard
"bintrees@npm:1.0.2": "bintrees@npm:1.0.2":
version: 1.0.2 version: 1.0.2
resolution: "bintrees@npm:1.0.2" resolution: "bintrees@npm:1.0.2"
@@ -5531,7 +5550,7 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"braces@npm:^3.0.3": "braces@npm:^3.0.3, braces@npm:~3.0.2":
version: 3.0.3 version: 3.0.3
resolution: "braces@npm:3.0.3" resolution: "braces@npm:3.0.3"
dependencies: dependencies:
@@ -5745,6 +5764,25 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"chokidar@npm:^3.5.3":
version: 3.6.0
resolution: "chokidar@npm:3.6.0"
dependencies:
anymatch: "npm:~3.1.2"
braces: "npm:~3.0.2"
fsevents: "npm:~2.3.2"
glob-parent: "npm:~5.1.2"
is-binary-path: "npm:~2.1.0"
is-glob: "npm:~4.0.1"
normalize-path: "npm:~3.0.0"
readdirp: "npm:~3.6.0"
dependenciesMeta:
fsevents:
optional: true
checksum: 10c0/8361dcd013f2ddbe260eacb1f3cb2f2c6f2b0ad118708a343a5ed8158941a39cb8fb1d272e0f389712e74ee90ce8ba864eece9e0e62b9705cb468a2f6d917462
languageName: node
linkType: hard
"chokidar@npm:^4.0.0": "chokidar@npm:^4.0.0":
version: 4.0.0 version: 4.0.0
resolution: "chokidar@npm:4.0.0" resolution: "chokidar@npm:4.0.0"
@@ -7563,6 +7601,17 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"fs-extra@npm:^11.3.0":
version: 11.3.0
resolution: "fs-extra@npm:11.3.0"
dependencies:
graceful-fs: "npm:^4.2.0"
jsonfile: "npm:^6.0.1"
universalify: "npm:^2.0.0"
checksum: 10c0/5f95e996186ff45463059feb115a22fb048bdaf7e487ecee8a8646c78ed8fdca63630e3077d4c16ce677051f5e60d3355a06f3cd61f3ca43f48cc58822a44d0a
languageName: node
linkType: hard
"fs-extra@npm:^9.0.1": "fs-extra@npm:^9.0.1":
version: 9.1.0 version: 9.1.0
resolution: "fs-extra@npm:9.1.0" resolution: "fs-extra@npm:9.1.0"
@@ -7749,7 +7798,7 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"glob-parent@npm:^5.1.2": "glob-parent@npm:^5.1.2, glob-parent@npm:~5.1.2":
version: 5.1.2 version: 5.1.2
resolution: "glob-parent@npm:5.1.2" resolution: "glob-parent@npm:5.1.2"
dependencies: dependencies:
@@ -8116,6 +8165,13 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"idb@npm:^8.0.3":
version: 8.0.3
resolution: "idb@npm:8.0.3"
checksum: 10c0/421cd9a3281b7564528857031cc33fd9e95753f8191e483054cb25d1ceea7303a0d1462f4f69f5b41606f0f066156999e067478abf2460dfcf9cab80dae2a2b2
languageName: node
linkType: hard
"ieee754@npm:^1.2.1": "ieee754@npm:^1.2.1":
version: 1.2.1 version: 1.2.1
resolution: "ieee754@npm:1.2.1" resolution: "ieee754@npm:1.2.1"
@@ -8319,6 +8375,15 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"is-binary-path@npm:~2.1.0":
version: 2.1.0
resolution: "is-binary-path@npm:2.1.0"
dependencies:
binary-extensions: "npm:^2.0.0"
checksum: 10c0/a16eaee59ae2b315ba36fad5c5dcaf8e49c3e27318f8ab8fa3cdb8772bf559c8d1ba750a589c2ccb096113bb64497084361a25960899cb6172a6925ab6123d38
languageName: node
linkType: hard
"is-boolean-object@npm:^1.2.1": "is-boolean-object@npm:^1.2.1":
version: 1.2.2 version: 1.2.2
resolution: "is-boolean-object@npm:1.2.2" resolution: "is-boolean-object@npm:1.2.2"
@@ -8432,7 +8497,7 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3": "is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1":
version: 4.0.3 version: 4.0.3
resolution: "is-glob@npm:4.0.3" resolution: "is-glob@npm:4.0.3"
dependencies: dependencies:
@@ -9712,7 +9777,7 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"normalize-path@npm:^3.0.0": "normalize-path@npm:^3.0.0, normalize-path@npm:~3.0.0":
version: 3.0.0 version: 3.0.0
resolution: "normalize-path@npm:3.0.0" resolution: "normalize-path@npm:3.0.0"
checksum: 10c0/e008c8142bcc335b5e38cf0d63cfd39d6cf2d97480af9abdbe9a439221fd4d749763bab492a8ee708ce7a194bb00c9da6d0a115018672310850489137b3da046 checksum: 10c0/e008c8142bcc335b5e38cf0d63cfd39d6cf2d97480af9abdbe9a439221fd4d749763bab492a8ee708ce7a194bb00c9da6d0a115018672310850489137b3da046
@@ -9927,6 +9992,13 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"p-map@npm:^7.0.3":
version: 7.0.3
resolution: "p-map@npm:7.0.3"
checksum: 10c0/46091610da2b38ce47bcd1d8b4835a6fa4e832848a6682cf1652bc93915770f4617afc844c10a77d1b3e56d2472bb2d5622353fa3ead01a7f42b04fc8e744a5c
languageName: node
linkType: hard
"package-json-from-dist@npm:^1.0.0": "package-json-from-dist@npm:^1.0.0":
version: 1.0.0 version: 1.0.0
resolution: "package-json-from-dist@npm:1.0.0" resolution: "package-json-from-dist@npm:1.0.0"
@@ -10165,7 +10237,7 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"picomatch@npm:^2.2.2, picomatch@npm:^2.3.1": "picomatch@npm:^2.0.4, picomatch@npm:^2.2.1, picomatch@npm:^2.2.2, picomatch@npm:^2.3.1":
version: 2.3.1 version: 2.3.1
resolution: "picomatch@npm:2.3.1" resolution: "picomatch@npm:2.3.1"
checksum: 10c0/26c02b8d06f03206fc2ab8d16f19960f2ff9e81a658f831ecb656d8f17d9edc799e8364b1f4a7873e89d9702dff96204be0fa26fe4181f6843f040f819dac4be checksum: 10c0/26c02b8d06f03206fc2ab8d16f19960f2ff9e81a658f831ecb656d8f17d9edc799e8364b1f4a7873e89d9702dff96204be0fa26fe4181f6843f040f819dac4be
@@ -11394,6 +11466,15 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"readdirp@npm:~3.6.0":
version: 3.6.0
resolution: "readdirp@npm:3.6.0"
dependencies:
picomatch: "npm:^2.2.1"
checksum: 10c0/6fa848cf63d1b82ab4e985f4cf72bd55b7dcfd8e0a376905804e48c3634b7e749170940ba77b32804d5fe93b3cc521aa95a8d7e7d725f830da6d93f3669ce66b
languageName: node
linkType: hard
"real-require@npm:^0.2.0": "real-require@npm:^0.2.0":
version: 0.2.0 version: 0.2.0
resolution: "real-require@npm:0.2.0" resolution: "real-require@npm:0.2.0"
@@ -13822,6 +13903,21 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"vite-plugin-static-copy@npm:^3.1.0":
version: 3.1.0
resolution: "vite-plugin-static-copy@npm:3.1.0"
dependencies:
chokidar: "npm:^3.5.3"
fs-extra: "npm:^11.3.0"
p-map: "npm:^7.0.3"
picocolors: "npm:^1.1.1"
tinyglobby: "npm:^0.2.14"
peerDependencies:
vite: ^5.0.0 || ^6.0.0 || ^7.0.0
checksum: 10c0/dce43f12ecc71417f1afd530d15b316774fe0441c2502e48e2bfafcd07fd4ae90a5782621f932d8d12a8c8213bed6746e80d5452e2fb216ece2bcf7e80309f82
languageName: node
linkType: hard
"vite-plugin-stimulus-hmr@npm:^3.0.0": "vite-plugin-stimulus-hmr@npm:^3.0.0":
version: 3.0.0 version: 3.0.0
resolution: "vite-plugin-stimulus-hmr@npm:3.0.0" resolution: "vite-plugin-stimulus-hmr@npm:3.0.0"