From a1e88135225753544197d8f1dd57fbb58cead43f Mon Sep 17 00:00:00 2001 From: Echo Date: Wed, 9 Jul 2025 11:55:41 +0200 Subject: [PATCH] Emoji Indexing and Search (#35253) --- .../mastodon/features/emoji/constants.ts | 110 ++++++++++++++ .../mastodon/features/emoji/database.ts | 102 +++++++++++++ .../mastodon/features/emoji/index.ts | 38 +++++ .../mastodon/features/emoji/loader.ts | 77 ++++++++++ .../mastodon/features/emoji/locale.test.ts | 61 ++------ .../mastodon/features/emoji/locale.ts | 56 ++------ .../mastodon/features/emoji/normalize.test.ts | 135 +++++++++++------- .../mastodon/features/emoji/normalize.ts | 78 +++++++--- .../mastodon/features/emoji/worker.ts | 13 ++ package.json | 2 + vite.config.mts | 25 +++- yarn.lock | 106 +++++++++++++- 12 files changed, 632 insertions(+), 171 deletions(-) create mode 100644 app/javascript/mastodon/features/emoji/constants.ts create mode 100644 app/javascript/mastodon/features/emoji/database.ts create mode 100644 app/javascript/mastodon/features/emoji/index.ts create mode 100644 app/javascript/mastodon/features/emoji/loader.ts create mode 100644 app/javascript/mastodon/features/emoji/worker.ts diff --git a/app/javascript/mastodon/features/emoji/constants.ts b/app/javascript/mastodon/features/emoji/constants.ts new file mode 100644 index 000000000..d38f17f21 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/constants.ts @@ -0,0 +1,110 @@ +// Utility codes +export const VARIATION_SELECTOR_CODE = 0xfe0f; +export const KEYCAP_CODE = 0x20e3; + +// Gender codes +export const GENDER_FEMALE_CODE = 0x2640; +export const GENDER_MALE_CODE = 0x2642; + +// Skin tone codes +export const SKIN_TONE_CODES = [ + 0x1f3fb, // Light skin tone + 0x1f3fc, // Medium-light skin tone + 0x1f3fd, // Medium skin tone + 0x1f3fe, // Medium-dark skin tone + 0x1f3ff, // Dark skin tone +] as const; + +export const EMOJIS_WITH_DARK_BORDER = [ + '🎱', // 1F3B1 + '🐜', // 1F41C + 'âšĢ', // 26AB + '🖤', // 1F5A4 + 'âŦ›', // 2B1B + 'â—ŧī¸', // 25FC-FE0F + '◾', // 25FE + 'â—ŧī¸', // 25FC-FE0F + '✒ī¸', // 2712-FE0F + 'â–Ēī¸', // 25AA-FE0F + 'đŸ’Ŗ', // 1F4A3 + 'đŸŽŗ', // 1F3B3 + '📷', // 1F4F7 + '📸', // 1F4F8 + 'â™Ŗī¸', // 2663-FE0F + 'đŸ•ļī¸', // 1F576-FE0F + '✴ī¸', // 2734-FE0F + '🔌', // 1F50C + '💂‍♀ī¸', // 1F482-200D-2640-FE0F + 'đŸ“Ŋī¸', // 1F4FD-FE0F + 'đŸŗ', // 1F373 + 'đŸĻ', // 1F98D + '💂', // 1F482 + 'đŸ”Ē', // 1F52A + 'đŸ•ŗī¸', // 1F573-FE0F + '🕹ī¸', // 1F579-FE0F + '🕋', // 1F54B + '🖊ī¸', // 1F58A-FE0F + '🖋ī¸', // 1F58B-FE0F + '💂‍♂ī¸', // 1F482-200D-2642-FE0F + '🎤', // 1F3A4 + '🎓', // 1F393 + 'đŸŽĨ', // 1F3A5 + 'đŸŽŧ', // 1F3BC + '♠ī¸', // 2660-FE0F + '🎩', // 1F3A9 + 'đŸĻƒ', // 1F983 + 'đŸ“ŧ', // 1F4FC + '📹', // 1F4F9 + '🎮', // 1F3AE + '🐃', // 1F403 + '🏴', // 1F3F4 + '🐞', // 1F41E + 'đŸ•ē', // 1F57A + '📱', // 1F4F1 + '📲', // 1F4F2 + '🚲', // 1F6B2 + 'đŸĒŽ', // 1FAA6 + 'đŸĻ‍âŦ›', // 1F426-200D-2B1B +]; + +export const EMOJIS_WITH_LIGHT_BORDER = [ + 'đŸ‘Ŋ', // 1F47D + '⚾', // 26BE + '🐔', // 1F414 + '☁ī¸', // 2601-FE0F + '💨', // 1F4A8 + '🕊ī¸', // 1F54A-FE0F + '👀', // 1F440 + 'đŸĨ', // 1F365 + 'đŸ‘ģ', // 1F47B + '🐐', // 1F410 + '❕', // 2755 + '❔', // 2754 + '⛸ī¸', // 26F8-FE0F + '🌩ī¸', // 1F329-FE0F + '🔊', // 1F50A + '🔇', // 1F507 + '📃', // 1F4C3 + '🌧ī¸', // 1F327-FE0F + '🐏', // 1F40F + '🍚', // 1F35A + '🍙', // 1F359 + '🐓', // 1F413 + '🐑', // 1F411 + '💀', // 1F480 + '☠ī¸', // 2620-FE0F + '🌨ī¸', // 1F328-FE0F + '🔉', // 1F509 + '🔈', // 1F508 + 'đŸ’Ŧ', // 1F4AC + '💭', // 1F4AD + '🏐', // 1F3D0 + 'đŸŗī¸', // 1F3F3-FE0F + 'âšĒ', // 26AA + 'âŦœ', // 2B1C + 'â—Ŋ', // 25FD + 'â—ģī¸', // 25FB-FE0F + 'â–Ģī¸', // 25AB-FE0F + 'đŸĒŊ', // 1FAE8 + 'đŸĒŋ', // 1FABF +]; diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts new file mode 100644 index 000000000..618f01085 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -0,0 +1,102 @@ +import { SUPPORTED_LOCALES } from 'emojibase'; +import type { FlatCompactEmoji, Locale } from 'emojibase'; +import type { DBSchema } from 'idb'; +import { openDB } from 'idb'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; + +import type { LocaleOrCustom } from './locale'; +import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; + +interface EmojiDB extends LocaleTables, DBSchema { + custom: { + key: string; + value: ApiCustomEmojiJSON; + indexes: { + category: string; + }; + }; + etags: { + key: LocaleOrCustom; + value: string; + }; +} + +interface LocaleTable { + key: string; + value: FlatCompactEmoji; + indexes: { + group: number; + label: string; + order: number; + tags: string[]; + }; +} +type LocaleTables = Record; + +const SCHEMA_VERSION = 1; + +const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { + upgrade(database) { + const customTable = database.createObjectStore('custom', { + keyPath: 'shortcode', + autoIncrement: false, + }); + customTable.createIndex('category', 'category'); + + database.createObjectStore('etags'); + + for (const locale of SUPPORTED_LOCALES) { + const localeTable = database.createObjectStore(locale, { + keyPath: 'hexcode', + autoIncrement: false, + }); + localeTable.createIndex('group', 'group'); + localeTable.createIndex('label', 'label'); + localeTable.createIndex('order', 'order'); + localeTable.createIndex('tags', 'tags', { multiEntry: true }); + } + }, +}); + +export async function putEmojiData(emojis: FlatCompactEmoji[], locale: Locale) { + const trx = db.transaction(locale, 'readwrite'); + await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); + await trx.done; +} + +export async function putCustomEmojiData(emojis: ApiCustomEmojiJSON[]) { + const trx = db.transaction('custom', 'readwrite'); + await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); + await trx.done; +} + +export function putLatestEtag(etag: string, localeString: string) { + const locale = toSupportedLocaleOrCustom(localeString); + return db.put('etags', etag, locale); +} + +export function searchEmojiByHexcode(hexcode: string, localeString: string) { + const locale = toSupportedLocale(localeString); + return db.get(locale, hexcode); +} + +export function searchEmojiByTag(tag: string, localeString: string) { + const locale = toSupportedLocale(localeString); + const range = IDBKeyRange.only(tag.toLowerCase()); + return db.getAllFromIndex(locale, 'tags', range); +} + +export function searchCustomEmojiByShortcode(shortcode: string) { + return db.get('custom', shortcode); +} + +export async function loadLatestEtag(localeString: string) { + const locale = toSupportedLocaleOrCustom(localeString); + const rowCount = await db.count(locale); + if (!rowCount) { + return null; // No data for this locale, return null even if there is an etag. + } + const etag = await db.get('etags', locale); + return etag ?? null; +} diff --git a/app/javascript/mastodon/features/emoji/index.ts b/app/javascript/mastodon/features/emoji/index.ts new file mode 100644 index 000000000..6975465b5 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/index.ts @@ -0,0 +1,38 @@ +import initialState from '@/mastodon/initial_state'; + +import { toSupportedLocale } from './locale'; + +const serverLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); + +const worker = + 'Worker' in window + ? new Worker(new URL('./worker', import.meta.url), { + type: 'module', + }) + : null; + +export async function initializeEmoji() { + if (worker) { + worker.addEventListener('message', (event: MessageEvent) => { + const { data: message } = event; + if (message === 'ready') { + worker.postMessage(serverLocale); + worker.postMessage('custom'); + } + }); + } else { + const { importCustomEmojiData, importEmojiData } = await import('./loader'); + await Promise.all([importCustomEmojiData(), importEmojiData(serverLocale)]); + } +} + +export async function loadEmojiLocale(localeString: string) { + const locale = toSupportedLocale(localeString); + + if (worker) { + worker.postMessage(locale); + } else { + const { importEmojiData } = await import('./loader'); + await importEmojiData(locale); + } +} diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts new file mode 100644 index 000000000..f9c697135 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -0,0 +1,77 @@ +import { flattenEmojiData } from 'emojibase'; +import type { CompactEmoji, FlatCompactEmoji } from 'emojibase'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; +import { isDevelopment } from '@/mastodon/utils/environment'; + +import { + putEmojiData, + putCustomEmojiData, + loadLatestEtag, + putLatestEtag, +} from './database'; +import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; +import type { LocaleOrCustom } from './locale'; + +export async function importEmojiData(localeString: string) { + const locale = toSupportedLocale(localeString); + const emojis = await fetchAndCheckEtag(locale); + if (!emojis) { + return; + } + const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(emojis); + await putEmojiData(flattenedEmojis, locale); +} + +export async function importCustomEmojiData() { + const emojis = await fetchAndCheckEtag('custom'); + if (!emojis) { + return; + } + await putCustomEmojiData(emojis); +} + +async function fetchAndCheckEtag( + localeOrCustom: LocaleOrCustom, +): Promise { + const locale = toSupportedLocaleOrCustom(localeOrCustom); + + let uri: string; + if (locale === 'custom') { + uri = '/api/v1/custom_emojis'; + } else { + uri = `/packs${isDevelopment() ? '-dev' : ''}/emoji/${locale}.json`; + } + + const oldEtag = await loadLatestEtag(locale); + const response = await fetch(uri, { + headers: { + 'Content-Type': 'application/json', + 'If-None-Match': oldEtag ?? '', // Send the old ETag to check for modifications + }, + }); + // If not modified, return null + if (response.status === 304) { + return null; + } + if (!response.ok) { + throw new Error( + `Failed to fetch emoji data for ${localeOrCustom}: ${response.statusText}`, + ); + } + + const data = (await response.json()) as ResultType; + if (!Array.isArray(data)) { + throw new Error( + `Unexpected data format for ${localeOrCustom}: expected an array`, + ); + } + + // Store the ETag for future requests + const etag = response.headers.get('ETag'); + if (etag) { + await putLatestEtag(etag, localeOrCustom); + } + + return data; +} diff --git a/app/javascript/mastodon/features/emoji/locale.test.ts b/app/javascript/mastodon/features/emoji/locale.test.ts index 0e098b2d4..5a474e942 100644 --- a/app/javascript/mastodon/features/emoji/locale.test.ts +++ b/app/javascript/mastodon/features/emoji/locale.test.ts @@ -1,52 +1,6 @@ -import { flattenEmojiData, SUPPORTED_LOCALES } from 'emojibase'; -import emojiEnData from 'emojibase-data/en/compact.json'; -import emojiFrData from 'emojibase-data/fr/compact.json'; +import { SUPPORTED_LOCALES } from 'emojibase'; -import { toSupportedLocale, unicodeToLocaleLabel } from './locale'; - -describe('unicodeToLocaleLabel', () => { - const emojiTestCases = [ - '1F3CB-1F3FF-200D-2640-FE0F', // 🏋đŸŋ‍♀ī¸ Woman weightlifter, dark skin - '1F468-1F3FB', // 👨đŸģ Man, light skin - '1F469-1F3FB-200D-2695-FE0F', // 👩đŸģ‍⚕ī¸ Woman health worker, light skin - '1F468-1F3FD-200D-1F692', // 👨đŸŊ‍🚒 Man firefighter, medium skin - '1F469-1F3FE', // 👩🏾 Woman, medium-dark skin - '1F469-1F3FF-200D-1F4BB', // 👩đŸŋ‍đŸ’ģ Woman technologist, dark skin - '1F478-1F3FF', // 👸đŸŋ Princess with dark skin tone - '1F935-1F3FC-200D-2640-FE0F', // đŸ¤ĩđŸŧ‍♀ī¸ Woman in tuxedo, medium-light skin - '1F9D1-1F3FC', // 🧑đŸŧ Person, medium-light skin - '1F9D4-1F3FE', // 🧔🏾 Person with beard, medium-dark skin - ]; - - const flattenedEnData = flattenEmojiData(emojiEnData); - const flattenedFrData = flattenEmojiData(emojiFrData); - - const emojiTestEnLabels = new Map( - emojiTestCases.map((code) => [ - code, - flattenedEnData.find((emoji) => emoji.hexcode === code)?.label, - ]), - ); - const emojiTestFrLabels = new Map( - emojiTestCases.map((code) => [ - code, - flattenedFrData.find((emoji) => emoji.hexcode === code)?.label, - ]), - ); - - test.for( - emojiTestCases.flatMap((code) => [ - [code, 'en', emojiTestEnLabels.get(code)], - [code, 'fr', emojiTestFrLabels.get(code)], - ]) satisfies [string, string, string | undefined][], - )( - 'returns correct label for %s for %s locale', - async ([unicodeHex, locale, expectedLabel]) => { - const label = await unicodeToLocaleLabel(unicodeHex, locale); - expect(label).toBe(expectedLabel); - }, - ); -}); +import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; describe('toSupportedLocale', () => { test('returns the same locale if it is supported', () => { @@ -62,3 +16,14 @@ describe('toSupportedLocale', () => { } }); }); + +describe('toSupportedLocaleOrCustom', () => { + test('returns custom for "custom" locale', () => { + expect(toSupportedLocaleOrCustom('custom')).toBe('custom'); + }); + test('returns supported locale for valid locales', () => { + for (const locale of SUPPORTED_LOCALES) { + expect(toSupportedLocaleOrCustom(locale)).toBe(locale); + } + }); +}); diff --git a/app/javascript/mastodon/features/emoji/locale.ts b/app/javascript/mastodon/features/emoji/locale.ts index aac6c376b..561c94afb 100644 --- a/app/javascript/mastodon/features/emoji/locale.ts +++ b/app/javascript/mastodon/features/emoji/locale.ts @@ -1,51 +1,23 @@ -import type { CompactEmoji, Locale } from 'emojibase'; -import { flattenEmojiData, SUPPORTED_LOCALES } from 'emojibase'; +import type { Locale } from 'emojibase'; +import { SUPPORTED_LOCALES } from 'emojibase'; -// Simple cache. This will be replaced with an IndexedDB cache in the future. -const localeCache = new Map>(); +export type LocaleOrCustom = Locale | 'custom'; -export async function unicodeToLocaleLabel( - unicodeHex: string, - localeString: string, -) { - const locale = toSupportedLocale(localeString); - let hexMap = localeCache.get(locale); - if (!hexMap) { - hexMap = await loadLocaleLabels(locale); - localeCache.set(locale, hexMap); - } - - const label = hexMap.get(unicodeHex)?.label; - if (!label) { - throw new Error( - `Label for unicode hex ${unicodeHex} not found in locale ${locale}`, - ); - } - return label; -} - -async function loadLocaleLabels( - locale: Locale, -): Promise> { - const { default: localeEmoji } = ((await import( - `emojibase-data/${locale}/compact.json` - )) ?? { default: [] }) as { default: CompactEmoji[] }; - if (!Array.isArray(localeEmoji)) { - throw new Error(`Locale data for ${locale} not found`); - } - const hexMapEntries = flattenEmojiData(localeEmoji).map( - (emoji) => [emoji.hexcode, emoji] satisfies [string, CompactEmoji], - ); - return new Map(hexMapEntries); -} - -export function toSupportedLocale(locale: string): Locale { +export function toSupportedLocale(localeBase: string): Locale { + const locale = localeBase.toLowerCase(); if (isSupportedLocale(locale)) { return locale; } return 'en'; // Default to English if unsupported } -function isSupportedLocale(locale: string): locale is Locale { - return SUPPORTED_LOCALES.includes(locale as Locale); +export function toSupportedLocaleOrCustom(locale: string): LocaleOrCustom { + if (locale.toLowerCase() === 'custom') { + return 'custom'; + } + return toSupportedLocale(locale); +} + +function isSupportedLocale(locale: string): locale is Locale { + return SUPPORTED_LOCALES.includes(locale.toLowerCase() as Locale); } diff --git a/app/javascript/mastodon/features/emoji/normalize.test.ts b/app/javascript/mastodon/features/emoji/normalize.test.ts index 29255d529..ee9cd8948 100644 --- a/app/javascript/mastodon/features/emoji/normalize.test.ts +++ b/app/javascript/mastodon/features/emoji/normalize.test.ts @@ -1,9 +1,17 @@ import { readdir } from 'fs/promises'; import { basename, resolve } from 'path'; -import unicodeEmojis from 'emojibase-data/en/data.json'; +import { flattenEmojiData } from 'emojibase'; +import unicodeRawEmojis from 'emojibase-data/en/data.json'; -import { twemojiToUnicodeInfo, unicodeToTwemojiHex } from './normalize'; +import { + twemojiHasBorder, + twemojiToUnicodeInfo, + unicodeToTwemojiHex, + CODES_WITH_DARK_BORDER, + CODES_WITH_LIGHT_BORDER, + emojiToUnicodeHex, +} from './normalize'; const emojiSVGFiles = await readdir( // This assumes tests are run from project root @@ -13,60 +21,81 @@ const emojiSVGFiles = await readdir( }, ); const svgFileNames = emojiSVGFiles - .filter( - (file) => - file.isFile() && - file.name.endsWith('.svg') && - !file.name.endsWith('_border.svg'), - ) + .filter((file) => file.isFile() && file.name.endsWith('.svg')) .map((file) => basename(file.name, '.svg').toUpperCase()); +const svgFileNamesWithoutBorder = svgFileNames.filter( + (fileName) => !fileName.endsWith('_BORDER'), +); -describe('normalizeEmoji', () => { - describe('unicodeToSVGName', () => { - test.concurrent.for( - unicodeEmojis - // Our version of Twemoji only supports up to version 15.1 - .filter((emoji) => emoji.version < 16) - .map((emoji) => [emoji.hexcode, emoji.label] as [string, string]), - )('verifying an emoji exists for %s (%s)', ([hexcode], { expect }) => { - const result = unicodeToTwemojiHex(hexcode); - expect(svgFileNames).toContain(result); - }); - }); +const unicodeEmojis = flattenEmojiData(unicodeRawEmojis); - describe('twemojiToUnicodeInfo', () => { - const unicodeMap = new Map( - unicodeEmojis.flatMap((emoji) => { - const base: [string, string][] = [[emoji.hexcode, emoji.label]]; - if (emoji.skins) { - base.push( - ...emoji.skins.map( - (skin) => [skin.hexcode, skin.label] as [string, string], - ), - ); - } - return base; - }), - ); +describe('emojiToUnicodeHex', () => { + test.concurrent.for([ + ['🎱', '1F3B1'], + ['🐜', '1F41C'], + ['âšĢ', '26AB'], + ['🖤', '1F5A4'], + ['💀', '1F480'], + ['💂‍♂ī¸', '1F482-200D-2642-FE0F'], + ] as const)( + 'emojiToUnicodeHex converts %s to %s', + ([emoji, hexcode], { expect }) => { + expect(emojiToUnicodeHex(emoji)).toBe(hexcode); + }, + ); +}); - test.concurrent.for(svgFileNames)( - 'verifying SVG file %s maps to Unicode emoji', - (svgFileName, { expect }) => { - assert(!!svgFileName); - const result = twemojiToUnicodeInfo(svgFileName); - const hexcode = - typeof result === 'string' ? result : result.unqualified; - if (!hexcode) { - // No hexcode means this is a special case like the Shibuya 109 emoji - expect(result).toHaveProperty('label'); - return; - } - assert(!!hexcode); - expect( - unicodeMap.has(hexcode), - `${hexcode} (${svgFileName}) not found`, - ).toBeTruthy(); - }, - ); +describe('unicodeToTwemojiHex', () => { + test.concurrent.for( + unicodeEmojis + // Our version of Twemoji only supports up to version 15.1 + .filter((emoji) => emoji.version < 16) + .map((emoji) => [emoji.hexcode, emoji.label] as [string, string]), + )('verifying an emoji exists for %s (%s)', ([hexcode], { expect }) => { + const result = unicodeToTwemojiHex(hexcode); + expect(svgFileNamesWithoutBorder).toContain(result); }); }); + +describe('twemojiHasBorder', () => { + test.concurrent.for( + svgFileNames + .filter((file) => file.endsWith('_BORDER')) + .map((file) => { + const hexCode = file.replace('_BORDER', ''); + return [ + hexCode, + CODES_WITH_LIGHT_BORDER.includes(hexCode), + CODES_WITH_DARK_BORDER.includes(hexCode), + ] as const; + }), + )('twemojiHasBorder for %s', ([hexCode, isLight, isDark], { expect }) => { + const result = twemojiHasBorder(hexCode); + expect(result).toHaveProperty('hexCode', hexCode); + expect(result).toHaveProperty('hasLightBorder', isLight); + expect(result).toHaveProperty('hasDarkBorder', isDark); + }); +}); + +describe('twemojiToUnicodeInfo', () => { + const unicodeCodeSet = new Set(unicodeEmojis.map((emoji) => emoji.hexcode)); + + test.concurrent.for(svgFileNamesWithoutBorder)( + 'verifying SVG file %s maps to Unicode emoji', + (svgFileName, { expect }) => { + assert(!!svgFileName); + const result = twemojiToUnicodeInfo(svgFileName); + const hexcode = typeof result === 'string' ? result : result.unqualified; + if (!hexcode) { + // No hexcode means this is a special case like the Shibuya 109 emoji + expect(result).toHaveProperty('label'); + return; + } + assert(!!hexcode); + expect( + unicodeCodeSet.has(hexcode), + `${hexcode} (${svgFileName}) not found`, + ).toBeTruthy(); + }, + ); +}); diff --git a/app/javascript/mastodon/features/emoji/normalize.ts b/app/javascript/mastodon/features/emoji/normalize.ts index 024cd5362..94dc33a6e 100644 --- a/app/javascript/mastodon/features/emoji/normalize.ts +++ b/app/javascript/mastodon/features/emoji/normalize.ts @@ -1,19 +1,12 @@ -// Utility codes -const VARIATION_SELECTOR_CODE = 0xfe0f; -const KEYCAP_CODE = 0x20e3; - -// Gender codes -const GENDER_FEMALE_CODE = 0x2640; -const GENDER_MALE_CODE = 0x2642; - -// Skin tone codes -const SKIN_TONE_CODES = [ - 0x1f3fb, // Light skin tone - 0x1f3fc, // Medium-light skin tone - 0x1f3fd, // Medium skin tone - 0x1f3fe, // Medium-dark skin tone - 0x1f3ff, // Dark skin tone -] as const; +import { + VARIATION_SELECTOR_CODE, + KEYCAP_CODE, + GENDER_FEMALE_CODE, + GENDER_MALE_CODE, + SKIN_TONE_CODES, + EMOJIS_WITH_DARK_BORDER, + EMOJIS_WITH_LIGHT_BORDER, +} from './constants'; // Misc codes that have special handling const SKIER_CODE = 0x26f7; @@ -24,6 +17,17 @@ const LEVITATING_PERSON_CODE = 0x1f574; const SPEECH_BUBBLE_CODE = 0x1f5e8; const MS_CLAUS_CODE = 0x1f936; +export function emojiToUnicodeHex(emoji: string): string { + const codes: number[] = []; + for (const char of emoji) { + const code = char.codePointAt(0); + if (code !== undefined) { + codes.push(code); + } + } + return hexNumbersToString(codes); +} + export function unicodeToTwemojiHex(unicodeHex: string): string { const codes = hexStringToNumbers(unicodeHex); const normalizedCodes: number[] = []; @@ -50,6 +54,35 @@ export function unicodeToTwemojiHex(unicodeHex: string): string { return hexNumbersToString(normalizedCodes, 0); } +interface TwemojiBorderInfo { + hexCode: string; + hasLightBorder: boolean; + hasDarkBorder: boolean; +} + +export const CODES_WITH_DARK_BORDER = + EMOJIS_WITH_DARK_BORDER.map(emojiToUnicodeHex); + +export const CODES_WITH_LIGHT_BORDER = + EMOJIS_WITH_LIGHT_BORDER.map(emojiToUnicodeHex); + +export function twemojiHasBorder(twemojiHex: string): TwemojiBorderInfo { + const normalizedHex = twemojiHex.toUpperCase(); + let hasLightBorder = false; + let hasDarkBorder = false; + if (CODES_WITH_LIGHT_BORDER.includes(normalizedHex)) { + hasLightBorder = true; + } + if (CODES_WITH_DARK_BORDER.includes(normalizedHex)) { + hasDarkBorder = true; + } + return { + hexCode: normalizedHex, + hasLightBorder, + hasDarkBorder, + }; +} + interface TwemojiSpecificEmoji { unqualified?: string; gender?: number; @@ -84,11 +117,16 @@ export function twemojiToUnicodeInfo( let gender: undefined | number; let skin: undefined | number; for (const code of codes) { - if (code in GENDER_CODES_MAP) { + if (!gender && code in GENDER_CODES_MAP) { gender = GENDER_CODES_MAP[code]; - } else if (code in SKIN_TONE_CODES) { + } else if (!skin && code in SKIN_TONE_CODES) { skin = code; } + + // Exit if we have both skin and gender + if (skin && gender) { + break; + } } let mappedCodes: unknown[] = codes; @@ -103,8 +141,8 @@ export function twemojiToUnicodeInfo( // For key emoji, insert the variation selector mappedCodes = [codes[0], VARIATION_SELECTOR_CODE, KEYCAP_CODE]; } else if ( - codes.at(0) === SKIER_CODE || - codes.at(0) === LEVITATING_PERSON_CODE + (codes.at(0) === SKIER_CODE || codes.at(0) === LEVITATING_PERSON_CODE) && + codes.length > 1 ) { // Twemoji offers more gender and skin options for the skier and levitating person emoji. return { diff --git a/app/javascript/mastodon/features/emoji/worker.ts b/app/javascript/mastodon/features/emoji/worker.ts new file mode 100644 index 000000000..1c48a0777 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/worker.ts @@ -0,0 +1,13 @@ +import { importEmojiData, importCustomEmojiData } from './loader'; + +addEventListener('message', handleMessage); +self.postMessage('ready'); // After the worker is ready, notify the main thread + +function handleMessage(event: MessageEvent) { + const { data: locale } = event; + if (locale !== 'custom') { + void importEmojiData(locale); + } else { + void importCustomEmojiData(); + } +} diff --git a/package.json b/package.json index 267042233..ace593ab0 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "history": "^4.10.1", "hoist-non-react-statics": "^3.3.2", "http-link-header": "^1.1.1", + "idb": "^8.0.3", "immutable": "^4.3.0", "intl-messageformat": "^10.7.16", "js-yaml": "^4.1.0", @@ -117,6 +118,7 @@ "vite-plugin-pwa": "^1.0.0", "vite-plugin-rails": "^0.5.0", "vite-plugin-ruby": "^5.1.1", + "vite-plugin-static-copy": "^3.1.0", "vite-plugin-svgr": "^4.3.0", "vite-tsconfig-paths": "^5.1.4", "wicg-inert": "^3.1.2", diff --git a/vite.config.mts b/vite.config.mts index 484211eaa..6429f97c5 100644 --- a/vite.config.mts +++ b/vite.config.mts @@ -1,14 +1,15 @@ import path from 'node:path'; import { optimizeLodashImports } from '@optimize-lodash/rollup-plugin'; +import legacy from '@vitejs/plugin-legacy'; import react from '@vitejs/plugin-react'; import { PluginOption } from 'vite'; -import svgr from 'vite-plugin-svgr'; import { visualizer } from 'rollup-plugin-visualizer'; -import RailsPlugin from 'vite-plugin-rails'; import { VitePWA } from 'vite-plugin-pwa'; +import RailsPlugin from 'vite-plugin-rails'; +import { viteStaticCopy } from 'vite-plugin-static-copy'; +import svgr from 'vite-plugin-svgr'; import tsconfigPaths from 'vite-tsconfig-paths'; -import legacy from '@vitejs/plugin-legacy'; import { defineConfig, UserConfigFnPromise, UserConfig } from 'vite'; import postcssPresetEnv from 'postcss-preset-env'; @@ -78,6 +79,9 @@ export const config: UserConfigFnPromise = async ({ mode, command }) => { }, }, }, + worker: { + format: 'es', + }, plugins: [ tsconfigPaths(), RailsPlugin({ @@ -92,6 +96,21 @@ export const config: UserConfigFnPromise = async ({ mode, command }) => { plugins: ['formatjs', 'transform-react-remove-prop-types'], }, }), + viteStaticCopy({ + targets: [ + { + src: path.resolve( + __dirname, + 'node_modules/emojibase-data/**/compact.json', + ), + dest: 'emoji', + rename(_name, ext, dir) { + const locale = path.basename(path.dirname(dir)); + return `${locale}.${ext}`; + }, + }, + ], + }), MastodonServiceWorkerLocales(), MastodonEmojiCompressed(), legacy({ diff --git a/yarn.lock b/yarn.lock index a1aea84b5..a0814a27a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2686,6 +2686,7 @@ __metadata: hoist-non-react-statics: "npm:^3.3.2" http-link-header: "npm:^1.1.1" husky: "npm:^9.0.11" + idb: "npm:^8.0.3" immutable: "npm:^4.3.0" intl-messageformat: "npm:^10.7.16" js-yaml: "npm:^4.1.0" @@ -2742,6 +2743,7 @@ __metadata: vite-plugin-pwa: "npm:^1.0.0" vite-plugin-rails: "npm:^0.5.0" vite-plugin-ruby: "npm:^5.1.1" + vite-plugin-static-copy: "npm:^3.1.0" vite-plugin-svgr: "npm:^4.2.0" vite-tsconfig-paths: "npm:^5.1.4" vitest: "npm:^3.2.1" @@ -5083,6 +5085,16 @@ __metadata: languageName: node linkType: hard +"anymatch@npm:~3.1.2": + version: 3.1.3 + resolution: "anymatch@npm:3.1.3" + dependencies: + normalize-path: "npm:^3.0.0" + picomatch: "npm:^2.0.4" + checksum: 10c0/57b06ae984bc32a0d22592c87384cd88fe4511b1dd7581497831c56d41939c8a001b28e7b853e1450f2bf61992dfcaa8ae2d0d161a0a90c4fb631ef07098fbac + languageName: node + linkType: hard + "are-docs-informative@npm:^0.0.2": version: 0.0.2 resolution: "are-docs-informative@npm:0.0.2" @@ -5471,6 +5483,13 @@ __metadata: languageName: node linkType: hard +"binary-extensions@npm:^2.0.0": + version: 2.3.0 + resolution: "binary-extensions@npm:2.3.0" + checksum: 10c0/75a59cafc10fb12a11d510e77110c6c7ae3f4ca22463d52487709ca7f18f69d886aa387557cc9864fbdb10153d0bdb4caacabf11541f55e89ed6e18d12ece2b5 + languageName: node + linkType: hard + "bintrees@npm:1.0.2": version: 1.0.2 resolution: "bintrees@npm:1.0.2" @@ -5531,7 +5550,7 @@ __metadata: languageName: node linkType: hard -"braces@npm:^3.0.3": +"braces@npm:^3.0.3, braces@npm:~3.0.2": version: 3.0.3 resolution: "braces@npm:3.0.3" dependencies: @@ -5745,6 +5764,25 @@ __metadata: languageName: node linkType: hard +"chokidar@npm:^3.5.3": + version: 3.6.0 + resolution: "chokidar@npm:3.6.0" + dependencies: + anymatch: "npm:~3.1.2" + braces: "npm:~3.0.2" + fsevents: "npm:~2.3.2" + glob-parent: "npm:~5.1.2" + is-binary-path: "npm:~2.1.0" + is-glob: "npm:~4.0.1" + normalize-path: "npm:~3.0.0" + readdirp: "npm:~3.6.0" + dependenciesMeta: + fsevents: + optional: true + checksum: 10c0/8361dcd013f2ddbe260eacb1f3cb2f2c6f2b0ad118708a343a5ed8158941a39cb8fb1d272e0f389712e74ee90ce8ba864eece9e0e62b9705cb468a2f6d917462 + languageName: node + linkType: hard + "chokidar@npm:^4.0.0": version: 4.0.0 resolution: "chokidar@npm:4.0.0" @@ -7563,6 +7601,17 @@ __metadata: languageName: node linkType: hard +"fs-extra@npm:^11.3.0": + version: 11.3.0 + resolution: "fs-extra@npm:11.3.0" + dependencies: + graceful-fs: "npm:^4.2.0" + jsonfile: "npm:^6.0.1" + universalify: "npm:^2.0.0" + checksum: 10c0/5f95e996186ff45463059feb115a22fb048bdaf7e487ecee8a8646c78ed8fdca63630e3077d4c16ce677051f5e60d3355a06f3cd61f3ca43f48cc58822a44d0a + languageName: node + linkType: hard + "fs-extra@npm:^9.0.1": version: 9.1.0 resolution: "fs-extra@npm:9.1.0" @@ -7749,7 +7798,7 @@ __metadata: languageName: node linkType: hard -"glob-parent@npm:^5.1.2": +"glob-parent@npm:^5.1.2, glob-parent@npm:~5.1.2": version: 5.1.2 resolution: "glob-parent@npm:5.1.2" dependencies: @@ -8116,6 +8165,13 @@ __metadata: languageName: node linkType: hard +"idb@npm:^8.0.3": + version: 8.0.3 + resolution: "idb@npm:8.0.3" + checksum: 10c0/421cd9a3281b7564528857031cc33fd9e95753f8191e483054cb25d1ceea7303a0d1462f4f69f5b41606f0f066156999e067478abf2460dfcf9cab80dae2a2b2 + languageName: node + linkType: hard + "ieee754@npm:^1.2.1": version: 1.2.1 resolution: "ieee754@npm:1.2.1" @@ -8319,6 +8375,15 @@ __metadata: languageName: node linkType: hard +"is-binary-path@npm:~2.1.0": + version: 2.1.0 + resolution: "is-binary-path@npm:2.1.0" + dependencies: + binary-extensions: "npm:^2.0.0" + checksum: 10c0/a16eaee59ae2b315ba36fad5c5dcaf8e49c3e27318f8ab8fa3cdb8772bf559c8d1ba750a589c2ccb096113bb64497084361a25960899cb6172a6925ab6123d38 + languageName: node + linkType: hard + "is-boolean-object@npm:^1.2.1": version: 1.2.2 resolution: "is-boolean-object@npm:1.2.2" @@ -8432,7 +8497,7 @@ __metadata: languageName: node linkType: hard -"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3": +"is-glob@npm:^4.0.0, is-glob@npm:^4.0.1, is-glob@npm:^4.0.3, is-glob@npm:~4.0.1": version: 4.0.3 resolution: "is-glob@npm:4.0.3" dependencies: @@ -9712,7 +9777,7 @@ __metadata: languageName: node linkType: hard -"normalize-path@npm:^3.0.0": +"normalize-path@npm:^3.0.0, normalize-path@npm:~3.0.0": version: 3.0.0 resolution: "normalize-path@npm:3.0.0" checksum: 10c0/e008c8142bcc335b5e38cf0d63cfd39d6cf2d97480af9abdbe9a439221fd4d749763bab492a8ee708ce7a194bb00c9da6d0a115018672310850489137b3da046 @@ -9927,6 +9992,13 @@ __metadata: languageName: node linkType: hard +"p-map@npm:^7.0.3": + version: 7.0.3 + resolution: "p-map@npm:7.0.3" + checksum: 10c0/46091610da2b38ce47bcd1d8b4835a6fa4e832848a6682cf1652bc93915770f4617afc844c10a77d1b3e56d2472bb2d5622353fa3ead01a7f42b04fc8e744a5c + languageName: node + linkType: hard + "package-json-from-dist@npm:^1.0.0": version: 1.0.0 resolution: "package-json-from-dist@npm:1.0.0" @@ -10165,7 +10237,7 @@ __metadata: languageName: node linkType: hard -"picomatch@npm:^2.2.2, picomatch@npm:^2.3.1": +"picomatch@npm:^2.0.4, picomatch@npm:^2.2.1, picomatch@npm:^2.2.2, picomatch@npm:^2.3.1": version: 2.3.1 resolution: "picomatch@npm:2.3.1" checksum: 10c0/26c02b8d06f03206fc2ab8d16f19960f2ff9e81a658f831ecb656d8f17d9edc799e8364b1f4a7873e89d9702dff96204be0fa26fe4181f6843f040f819dac4be @@ -11394,6 +11466,15 @@ __metadata: languageName: node linkType: hard +"readdirp@npm:~3.6.0": + version: 3.6.0 + resolution: "readdirp@npm:3.6.0" + dependencies: + picomatch: "npm:^2.2.1" + checksum: 10c0/6fa848cf63d1b82ab4e985f4cf72bd55b7dcfd8e0a376905804e48c3634b7e749170940ba77b32804d5fe93b3cc521aa95a8d7e7d725f830da6d93f3669ce66b + languageName: node + linkType: hard + "real-require@npm:^0.2.0": version: 0.2.0 resolution: "real-require@npm:0.2.0" @@ -13822,6 +13903,21 @@ __metadata: languageName: node linkType: hard +"vite-plugin-static-copy@npm:^3.1.0": + version: 3.1.0 + resolution: "vite-plugin-static-copy@npm:3.1.0" + dependencies: + chokidar: "npm:^3.5.3" + fs-extra: "npm:^11.3.0" + p-map: "npm:^7.0.3" + picocolors: "npm:^1.1.1" + tinyglobby: "npm:^0.2.14" + peerDependencies: + vite: ^5.0.0 || ^6.0.0 || ^7.0.0 + checksum: 10c0/dce43f12ecc71417f1afd530d15b316774fe0441c2502e48e2bfafcd07fd4ae90a5782621f932d8d12a8c8213bed6746e80d5452e2fb216ece2bcf7e80309f82 + languageName: node + linkType: hard + "vite-plugin-stimulus-hmr@npm:^3.0.0": version: 3.0.0 resolution: "vite-plugin-stimulus-hmr@npm:3.0.0"