The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
refactor(i18n): one runtime key space for translation catalogs (#24325)
Félix Malfait1e4cd272ea80
diff --git a/packages/twenty-sdk/src/cli/utilities/translations/__tests__/compile-catalog-to-message-ids.spec.ts b/packages/twenty-sdk/src/cli/utilities/translations/__tests__/compile-catalog-to-message-ids.spec.tsnew file mode 100644index 00000000000..cd64a3297ca--- /dev/null+++ b/packages/twenty-sdk/src/cli/utilities/translations/__tests__/compile-catalog-to-message-ids.spec.ts@@ -0,0 +1,56 @@+import { describe, expect, it, vi } from 'vitest';++import { generateMessageId } from 'twenty-shared/i18n';++import { compileCatalogToMessageIds } from '@/cli/utilities/translations/compile-catalog-to-message-ids';+import { CONTEXT_SEPARATOR } from '@/sdk/front-component/translations/message/context-separator.constant';+import { getTranslationCatalogKey } from '@/sdk/front-component/translations/message';++describe('compileCatalogToMessageIds', () => {+ it('keys translations by message id, honoring the context', () => {+ const compiled = compileCatalogToMessageIds({+ catalog: {+ Export: 'Exporter',+ [getTranslationCatalogKey('Export', 'view.name')]: 'Export de vue',+ },+ });++ expect(compiled).toEqual({+ [generateMessageId('Export')]: 'Exporter',+ [generateMessageId('Export', 'view.name')]: 'Export de vue',+ });+ });++ it('skips empty and non-string translations', () => {+ const compiled = compileCatalogToMessageIds({+ catalog: {+ Export: '',+ Import: 42,+ Delete: 'Supprimer',+ },+ });++ expect(compiled).toEqual({ [generateMessageId('Delete')]: 'Supprimer' });+ });++ it('reports a collision and keeps the later entry', () => {+ const onCollision = vi.fn();++ // A context-less key and an empty-context key hash identically, so two+ // distinct catalog keys can claim one message id.+ const compiled = compileCatalogToMessageIds({+ catalog: {+ Export: 'first',+ [`${CONTEXT_SEPARATOR}Export`]: 'second',+ },+ onCollision,+ });++ expect(onCollision).toHaveBeenCalledWith({+ messageId: generateMessageId('Export'),+ keptKey: `${CONTEXT_SEPARATOR}Export`,+ droppedKey: 'Export',+ });+ expect(compiled).toEqual({ [generateMessageId('Export')]: 'second' });+ });+});diff --git a/packages/twenty-sdk/src/cli/utilities/translations/compile-application-translations.ts b/packages/twenty-sdk/src/cli/utilities/translations/compile-application-translations.tsindex 73f61f99910..912225989cc 100644--- a/packages/twenty-sdk/src/cli/utilities/translations/compile-application-translations.ts+++ b/packages/twenty-sdk/src/cli/utilities/translations/compile-application-translations.ts@@ -1,10 +1,9 @@import { readdir } from 'node:fs/promises';import path from 'path';-import { parseTranslationCatalogKey } from '@/sdk/front-component/translations/message';+import { compileCatalogToMessageIds } from '@/cli/utilities/translations/compile-catalog-to-message-ids';import { pathExists, readJson } from '@/cli/utilities/file/fs-utils';import { LOCALES_DIR } from '@/cli/utilities/translations/constants';-import { generateMessageId } from 'twenty-shared/i18n';import { type TranslationsManifest } from 'twenty-shared/application';import {APP_LOCALES,@@ -45,31 +44,17 @@ export const compileApplicationTranslations = async (}const sourceToTranslation =- (await readJson<Record<string, string>>(+ (await readJson<Record<string, unknown>>(path.join(localesDir, localeFile),)) ?? {};- const compiled: Record<string, string> = {};- const keyByMessageId = new Map<string, string>();-- for (const [key, translation] of Object.entries(sourceToTranslation)) {- if (typeof translation !== 'string' || translation.length === 0) {- continue;- }-- const { message, context } = parseTranslationCatalogKey(key);- const messageId = generateMessageId(message, context);- const collidingKey = keyByMessageId.get(messageId);-- if (collidingKey !== undefined && collidingKey !== key) {+ const compiled = compileCatalogToMessageIds({+ catalog: sourceToTranslation,+ onCollision: ({ messageId, keptKey, droppedKey }) =>console.warn(- `Message id collision in "${localeFile}": "${key}" and "${collidingKey}" share id "${messageId}". Keeping "${key}".`,- );- }-- keyByMessageId.set(messageId, key);- compiled[messageId] = translation;- }+ `Message id collision in "${localeFile}": "${keptKey}" and "${droppedKey}" share id "${messageId}". Keeping "${keptKey}".`,+ ),+ });if (Object.keys(compiled).length > 0) {translations[locale] = compiled;diff --git a/packages/twenty-sdk/src/cli/utilities/translations/compile-catalog-to-message-ids.ts b/packages/twenty-sdk/src/cli/utilities/translations/compile-catalog-to-message-ids.tsnew file mode 100644index 00000000000..5276df4fbb6--- /dev/null+++ b/packages/twenty-sdk/src/cli/utilities/translations/compile-catalog-to-message-ids.ts@@ -0,0 +1,41 @@+import { generateMessageId } from 'twenty-shared/i18n';+import { isDefined } from 'twenty-shared/utils';++import { parseTranslationCatalogKey } from '@/sdk/front-component/translations/message';++// Authoring files are keyed readable (context + message) for translators;+// runtime lookups are by message id, so every consumer compiles through here.+export const compileCatalogToMessageIds = ({+ catalog,+ onCollision,+}: {+ // Parsed straight from on-disk JSON, so values are only claimed strings.+ catalog: Record<string, unknown>;+ onCollision?: (args: {+ messageId: string;+ keptKey: string;+ droppedKey: string;+ }) => void;+}): Record<string, string> => {+ const compiled: Record<string, string> = {};+ const keyByMessageId = new Map<string, string>();++ for (const [key, translation] of Object.entries(catalog)) {+ if (typeof translation !== 'string' || translation.length === 0) {+ continue;+ }++ const { message, context } = parseTranslationCatalogKey(key);+ const messageId = generateMessageId(message, context);+ const collidingKey = keyByMessageId.get(messageId);++ if (isDefined(collidingKey) && collidingKey !== key) {+ onCollision?.({ messageId, keptKey: key, droppedKey: collidingKey });+ }++ keyByMessageId.set(messageId, key);+ compiled[messageId] = translation;+ }++ return compiled;+};diff --git a/packages/twenty-sdk/src/cli/utilities/translations/load-front-component-translation-catalogs.ts b/packages/twenty-sdk/src/cli/utilities/translations/load-front-component-translation-catalogs.tsindex 3fdefb8ca2a..debd3382791 100644--- a/packages/twenty-sdk/src/cli/utilities/translations/load-front-component-translation-catalogs.ts+++ b/packages/twenty-sdk/src/cli/utilities/translations/load-front-component-translation-catalogs.ts@@ -9,6 +9,7 @@ import {import { type TranslationCatalogsByLocale } from '@/sdk/front-component/translations/message';import { pathExists, readJson } from '@/cli/utilities/file/fs-utils';+import { compileCatalogToMessageIds } from '@/cli/utilities/translations/compile-catalog-to-message-ids';import { LOCALES_DIR } from '@/cli/utilities/translations/constants';const isSupportedLocale = (locale: string): locale is AppLocale =>@@ -37,20 +38,14 @@ export const loadFrontComponentTranslationCatalogs = async (}const catalog =- (await readJson<Record<string, string>>(+ (await readJson<Record<string, unknown>>(path.join(localesDir, localeFile),)) ?? {};- const nonEmptyEntries: Record<string, string> = {};+ const compiled = compileCatalogToMessageIds({ catalog });- for (const [key, value] of Object.entries(catalog)) {- if (typeof value === 'string' && value.length > 0) {- nonEmptyEntries[key] = value;- }- }-- if (Object.keys(nonEmptyEntries).length > 0) {- catalogs[locale] = nonEmptyEntries;+ if (Object.keys(compiled).length > 0) {+ catalogs[locale] = compiled;}}diff --git a/packages/twenty-sdk/src/sdk/front-component/translations/__tests__/resolveTranslation.spec.ts b/packages/twenty-sdk/src/sdk/front-component/translations/__tests__/resolveTranslation.spec.tsindex bc3a3ec9029..946c77185a8 100644--- a/packages/twenty-sdk/src/sdk/front-component/translations/__tests__/resolveTranslation.spec.ts+++ b/packages/twenty-sdk/src/sdk/front-component/translations/__tests__/resolveTranslation.spec.ts@@ -1,7 +1,7 @@import { afterEach, describe, expect, it } from 'vitest';import { setFrontComponentTranslations } from '@/sdk/front-component/translations/front-component-translations';-import { getTranslationCatalogKey } from '@/sdk/front-component/translations/message';+import { generateMessageId } from 'twenty-shared/i18n';import { resolveTranslation } from '@/sdk/front-component/translations/resolveTranslation';afterEach(() => {@@ -10,13 +10,17 @@ afterEach(() => {describe('resolveTranslation', () => {it('returns the translation for the active locale', () => {- setFrontComponentTranslations({ 'fr-FR': { Save: 'Enregistrer' } });+ setFrontComponentTranslations({+ 'fr-FR': { [generateMessageId('Save')]: 'Enregistrer' },+ });expect(resolveTranslation('Save', undefined, 'fr-FR')).toBe('Enregistrer');});it('falls back to the source message when the locale is missing', () => {- setFrontComponentTranslations({ 'fr-FR': { Save: 'Enregistrer' } });+ setFrontComponentTranslations({+ 'fr-FR': { [generateMessageId('Save')]: 'Enregistrer' },+ });expect(resolveTranslation('Save', undefined, 'de-DE')).toBe('Save');});@@ -30,8 +34,8 @@ describe('resolveTranslation', () => {it('resolves context-disambiguated messages independently', () => {setFrontComponentTranslations({'fr-FR': {- [getTranslationCatalogKey('Open', 'door')]: 'Ouvrir',- [getTranslationCatalogKey('Open', 'window')]: 'Lever',+ [generateMessageId('Open', 'door')]: 'Ouvrir',+ [generateMessageId('Open', 'window')]: 'Lever',},});@@ -53,7 +57,10 @@ describe('resolveTranslation', () => {it('interpolates values into the resolved translation', () => {setFrontComponentTranslations({- 'fr-FR': { 'Saved {count} cards': 'Cartes enregistrées : {count}' },+ 'fr-FR': {+ [generateMessageId('Saved {count} cards')]:+ 'Cartes enregistrées : {count}',+ },});expect(diff --git a/packages/twenty-sdk/src/sdk/front-component/translations/resolveTranslation.ts b/packages/twenty-sdk/src/sdk/front-component/translations/resolveTranslation.tsindex f5b979adaba..2fa2b332762 100644--- a/packages/twenty-sdk/src/sdk/front-component/translations/resolveTranslation.ts+++ b/packages/twenty-sdk/src/sdk/front-component/translations/resolveTranslation.ts@@ -1,8 +1,8 @@+import { generateMessageId } from 'twenty-shared/i18n';import { type AppLocale } from 'twenty-shared/translations';import { getFrontComponentTranslations } from './front-component-translations';import {- getTranslationCatalogKey,interpolateMessage,normalizeMessageDescriptor,type MessageDescriptor,@@ -17,7 +17,7 @@ export const resolveTranslation = (const { message, context } = normalizeMessageDescriptor(descriptor);const catalog = getFrontComponentTranslations()[locale];- const translation = catalog?.[getTranslationCatalogKey(message, context)];+ const translation = catalog?.[generateMessageId(message, context)];return interpolateMessage(translation ?? message, values);};diff --git a/packages/twenty-server/src/engine/metadata-modules/command-menu-item/utils/is-object-metadata-command-menu-item-payload.util.ts b/packages/twenty-server/src/engine/metadata-modules/command-menu-item/utils/is-object-metadata-command-menu-item-payload.util.tsindex 4cdc67618fa..aa56c359128 100644--- a/packages/twenty-server/src/engine/metadata-modules/command-menu-item/utils/is-object-metadata-command-menu-item-payload.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/command-menu-item/utils/is-object-metadata-command-menu-item-payload.util.ts@@ -2,8 +2,6 @@ import { isDefined } from 'twenty-shared/utils';import { type ObjectMetadataCommandMenuItemPayload } from 'src/engine/metadata-modules/command-menu-item/dtos/types/object-metadata-command-menu-item-payload.type';-// Accepts unknown so callers holding an event payload do not have to assert a-// type in order to ask what it is.export const isObjectMetadataCommandMenuItemPayload = (payload?: unknown,): payload is ObjectMetadataCommandMenuItemPayload =>diff --git a/packages/twenty-server/src/engine/metadata-modules/utils/resolve-effective-entity-property.util.ts b/packages/twenty-server/src/engine/metadata-modules/utils/resolve-effective-entity-property.util.tsindex 2769ed428f1..95998b3c70b 100644--- a/packages/twenty-server/src/engine/metadata-modules/utils/resolve-effective-entity-property.util.ts+++ b/packages/twenty-server/src/engine/metadata-modules/utils/resolve-effective-entity-property.util.ts@@ -13,8 +13,6 @@ import { ALL_TRANSLATABLE_PROPERTIES_BY_METADATA_NAME } from 'src/engine/metadatimport { type EffectiveEntityI18nContext } from 'src/engine/metadata-modules/utils/effective-entity-i18n-context.type';import { type MetadataPresentationOverrides } from 'src/engine/metadata-modules/utils/metadata-presentation-overrides.type';-// Reads off an overrides bag whose exact shape depends on a metadata name only-// known at runtime, so the two lookups it needs are guarded rather than typed.const readOverrideProperty = (overrides: unknown, property: string): unknown =>isDefined(overrides) && typeof overrides === 'object'? (overrides as Record<string, unknown>)[property]@@ -36,10 +34,6 @@ const readOverrideTranslation = ({return typeof translation === 'string' ? translation : undefined;};-// The resolution itself never depends on the metadata name being a literal: it-// asks the registry whether the property is translatable and reads the rest by-// string. Both entry points share this so neither has to assert its way past-// the other's signature.const resolveEffectiveProperty = ({metadataName,baseValue,diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-publisher.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-publisher.tsindex 43231229e5f..812b95a0753 100644--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-publisher.ts+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/metadata-event-publisher.ts@@ -60,8 +60,6 @@ export class MetadataEventPublisher {return record?.userWorkspaceId ?? undefined;}- // Publish-time work is per-workspace lookups only. Anything viewer-dependent- // (overrides, translations) belongs at delivery, where the subscriber is known.private async enrichMetadataEventBatch(metadataEventBatch: MetadataEventBatch,): Promise<MetadataEventBatch> {diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/services/metadata-event-resolution.service.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/services/metadata-event-resolution.service.tsindex f5fde9d2cc0..e7f89bc3f96 100644--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/services/metadata-event-resolution.service.ts+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/services/metadata-event-resolution.service.ts@@ -14,8 +14,6 @@ import { type EventStreamMetadataEvent } from 'src/engine/subscriptions/types/evconst RECORD_KEYS = ['before', 'after'] as const;-// Event records are untyped bags, so the application a record belongs to has to-// be read defensively -- in the same way at both call sites.const readRecordApplicationId = (record: EventStreamMetadataEvent['properties'][(typeof RECORD_KEYS)[number]],): string | undefined =>@@ -51,7 +49,6 @@ export class MetadataEventResolutionService {isTranslatableMetadataName(metadataEvent.metadataName),);- // Most batches have nothing to resolve; they return without a cache read.const carriesOverrides = metadataEvents.some((metadataEvent) =>RECORD_KEYS.some((key) =>recordCarriesOverrides(metadataEvent.properties[key]),@@ -90,8 +87,6 @@ export class MetadataEventResolutionService {: undefined,});- // Only NAVIGATION command menu items interpolate against another entity, so- // the object maps are fetched only when such an event is actually present.const flatObjectMetadataMaps = metadataEvents.some((metadataEvent) => metadataEvent.metadataName === 'commandMenuItem',)diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/interpolate-navigation-command-menu-item-event.util.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/interpolate-navigation-command-menu-item-event.util.tsindex e18b6010291..06098e9c6bc 100644--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/interpolate-navigation-command-menu-item-event.util.ts+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/interpolate-navigation-command-menu-item-event.util.ts@@ -12,8 +12,6 @@ import { findFlatEntityByIdInFlatEntityMaps } from 'src/engine/metadata-modules/import { type FlatObjectMetadata } from 'src/engine/metadata-modules/flat-object-metadata/types/flat-object-metadata.type';import { type EffectiveEntityI18nContext } from 'src/engine/metadata-modules/utils/effective-entity-i18n-context.type';-// Described by what this reads rather than by the full flat entity, so callers-// and specs can pass the event payload without casting it into one.type InterpolatableCommandMenuItemRecord = Record<string, unknown> & {engineComponentKey?: unknown;payload?: unknown;diff --git a/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/resolve-metadata-event-record.util.ts b/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/resolve-metadata-event-record.util.tsindex 23f19bf9fd2..45aac325406 100644--- a/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/resolve-metadata-event-record.util.ts+++ b/packages/twenty-server/src/engine/subscriptions/metadata-event/utils/resolve-metadata-event-record.util.ts@@ -6,7 +6,6 @@ import { type EffectiveEntityI18nContext } from 'src/engine/metadata-modules/utiimport { resolveEffectiveEntityPropertyByName } from 'src/engine/metadata-modules/utils/resolve-effective-entity-property.util';import { isTranslatableMetadataName } from 'src/engine/subscriptions/metadata-event/utils/is-translatable-metadata-name.util';-// Per-locale overrides are consumed by the resolver, not copied onto the record.const TRANSLATIONS_OVERRIDE_KEY = 'translations';export const resolveMetadataEventRecord = ({diff --git a/packages/twenty-shared/jest.config.mjs b/packages/twenty-shared/jest.config.mjsindex 465ce750c3d..e7d5d687ad4 100644--- a/packages/twenty-shared/jest.config.mjs+++ b/packages/twenty-shared/jest.config.mjs@@ -3,6 +3,7 @@ const jestConfig = {displayName: 'twenty-shared',preset: '../../jest.preset.js',testEnvironment: 'jsdom',+ setupFiles: ['<rootDir>/jest.setup.ts'],transformIgnorePatterns: ['../../node_modules/'],transform: {'^.+\\.[tj]sx?$': [diff --git a/packages/twenty-shared/jest.setup.ts b/packages/twenty-shared/jest.setup.tsnew file mode 100644index 00000000000..abc789f8937--- /dev/null+++ b/packages/twenty-shared/jest.setup.ts@@ -0,0 +1,4 @@+// jsdom lacks TextEncoder/TextDecoder, which @noble/hashes needs.+import { TextDecoder, TextEncoder } from 'node:util';++Object.assign(globalThis, { TextDecoder, TextEncoder });diff --git a/packages/twenty-shared/package.json b/packages/twenty-shared/package.jsonindex 96872169c89..25b6386f49e 100644--- a/packages/twenty-shared/package.json+++ b/packages/twenty-shared/package.json@@ -44,6 +44,7 @@},"dependencies": {"@dagrejs/dagre": "^1.1.8",+ "@noble/hashes": "^1.8.0","@sniptt/guards": "^0.2.0","addressparser": "1.0.1","ai": "6.0.97",diff --git a/packages/twenty-shared/src/i18n/generate-message-id.ts b/packages/twenty-shared/src/i18n/generate-message-id.tsindex 3f5f6e21361..55571a5c915 100644--- a/packages/twenty-shared/src/i18n/generate-message-id.ts+++ b/packages/twenty-shared/src/i18n/generate-message-id.ts@@ -1,20 +1,40 @@-import { createHash } from 'node:crypto';--// Lingui's generated-id scheme. Catalogs are emitted with `printLinguiId: true`,-// so a source string persisted in metadata can be hashed back into its catalog-// key at read time.-//-// The server and the application SDK must produce byte-identical ids: the SDK-// writes app catalogs keyed by this hash at build time and the server reads them-// at request time. Drift between the two silently untranslates every installed-// application, which is why this lives here rather than in either package.-//-// This pulls `node:crypto`, so `twenty-shared/i18n` is a Node-only subpath.-// Nothing in twenty-front imports it.+import { sha256 } from '@noble/hashes/sha2';+import { utf8ToBytes } from '@noble/hashes/utils';++import { isDefined } from '../utils/validation/isDefined';++// Lingui's generated-id scheme (catalogs are emitted with `printLinguiId: true`).+// Server, SDK and front-component runtime must produce byte-identical ids --+// drift silently untranslates -- so there is exactly one implementation, in+// pure JS so the sandboxed front-component worker can call it too.const UNIT_SEPARATOR = String.fromCharCode(0x1f);-export const generateMessageId = (message: string, context = ''): string =>- createHash('sha256')- .update(message + UNIT_SEPARATOR + (context || ''))- .digest('base64')- .slice(0, 6);+// The same few thousand source strings recur across requests; the cap bounds+// a workspace that mints unusual labels.+const MAX_CACHED_MESSAGE_IDS = 50_000;++const messageIdByCacheKey = new Map<string, string>();++const toBase64 = (bytes: Uint8Array): string =>+ typeof Buffer !== 'undefined'+ ? Buffer.from(bytes).toString('base64')+ : btoa(String.fromCharCode(...bytes));++export const generateMessageId = (message: string, context = ''): string => {+ const cacheKey = message + UNIT_SEPARATOR + (context || '');+ const cachedMessageId = messageIdByCacheKey.get(cacheKey);++ if (isDefined(cachedMessageId)) {+ return cachedMessageId;+ }++ const messageId = toBase64(sha256(utf8ToBytes(cacheKey))).slice(0, 6);++ if (messageIdByCacheKey.size >= MAX_CACHED_MESSAGE_IDS) {+ messageIdByCacheKey.clear();+ }++ messageIdByCacheKey.set(cacheKey, messageId);++ return messageId;+};diff --git a/yarn.lock b/yarn.lockindex 27016fa8179..928ee69124d 100644--- a/yarn.lock+++ b/yarn.lock@@ -50260,6 +50260,7 @@ __metadata:"@babel/preset-typescript": "npm:^7.24.6""@dagrejs/dagre": "npm:^1.1.8""@lingui/core": "npm:^5.9.5"+ "@noble/hashes": "npm:^1.8.0""@prettier/sync": "npm:^0.5.2""@sniptt/guards": "npm:^0.2.0""@swc/core": "npm:^1.15.11"