The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Cover every translation pipeline with the normalizer (#25848)
Félix Malfaitdfa3a39efe94
diff --git a/.github/workflows/docs-i18n-pull.yaml b/.github/workflows/docs-i18n-pull.yamlindex 00821920f65..576cffb16bd 100644--- a/.github/workflows/docs-i18n-pull.yaml+++ b/.github/workflows/docs-i18n-pull.yaml@@ -83,7 +83,7 @@ jobs:- name: Normalize translations in Crowdinif: github.event_name != 'pull_request' && (inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')- run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=2 --apply --rules=escaped-inline-code-tags || echo "::warning::translation normalization failed (non-blocking)"+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=2 --format=mdx --apply || echo "::warning::translation normalization failed (non-blocking)"env:CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}diff --git a/.github/workflows/i18n-pull.yaml b/.github/workflows/i18n-pull.yamlindex b9c59ed868e..10ed88bc5be 100644--- a/.github/workflows/i18n-pull.yaml+++ b/.github/workflows/i18n-pull.yaml@@ -55,6 +55,14 @@ jobs:git add .git stash+ # Repair machine-translation damage in Crowdin before downloading, so the+ # catalogs that reach the repo are already correct rather than a cycle behind.+ - name: Normalize translations in Crowdin+ if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --format=po --apply || echo "::warning::translation normalization failed (non-blocking)"+ env:+ CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}+- name: Pull translations from Crowdinif: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2@@ -87,14 +95,6 @@ jobs:- name: Fix file permissionsrun: sudo chown -R runner:docker .- # Fix encoding issues (escaped Unicode like \u62db -> 招) and translations carrying a- # leaked response envelope, then push the fixes back to Crowdin so they- # survive the next sync.- - name: Normalize translations in Crowdin- run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode,corrupted-model-output,invented-markup,invented-argument || echo "::warning::translation normalization failed (non-blocking)"- env:- CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}-- name: Normalize downloaded catalogs to lingui formatrun: |npx nx run twenty-server:lingui:extractdiff --git a/.github/workflows/website-i18n-pull.yaml b/.github/workflows/website-i18n-pull.yamlindex da4d1e76444..3a0ee7ad073 100644--- a/.github/workflows/website-i18n-pull.yaml+++ b/.github/workflows/website-i18n-pull.yaml@@ -55,6 +55,14 @@ jobs:git add .git stash+ # Repair machine-translation damage in Crowdin before downloading, so the+ # catalogs that reach the repo are already correct rather than a cycle behind.+ - name: Normalize translations in Crowdin+ if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=4 --format=po --apply || echo "::warning::translation normalization failed (non-blocking)"+ env:+ CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}+- name: Pull website translations from Crowdinif: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2diff --git a/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx b/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsxindex fdd802311fb..b746b84e01a 100644--- a/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx+++ b/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx@@ -79,7 +79,7 @@ export const CommandMenuItemEditRecordSelectionDropdown = ({const TriggerIcon = isNoneSelected ? IconSquareX : IconSquareCheck;const triggerLabel = isNoneSelected? t`No record selected`- : t`Record(s) selected`;+ : t`Records selected`;return (<Dropdown@@ -119,7 +119,7 @@ export const CommandMenuItemEditRecordSelectionDropdown = ({/><MenuItemSelectLeftIcon={IconSquareCheck}- text={t`Record(s) selected`}+ text={t`Records selected`}selected={!isNoneSelected}onClick={() => handleSelectMode('selection')}/>diff --git a/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.ts b/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.tsnew file mode 100644index 00000000000..4e1844ee095--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.ts@@ -0,0 +1,28 @@+import { NORMALIZATION_RULES } from '../normalization-rules.constant';++const namesFor = (format: 'po' | 'mdx') =>+ NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format)).map(+ (rule) => rule.name,+ );++describe('NORMALIZATION_RULES', () => {+ it('gives every rule at least one format to run against', () => {+ for (const rule of NORMALIZATION_RULES) {+ expect(rule.formats.length).toBeGreaterThan(0);+ }+ });++ it('names every rule uniquely', () => {+ const names = NORMALIZATION_RULES.map((rule) => rule.name);++ expect(new Set(names).size).toBe(names.length);+ });++ it('keeps the ICU rules away from MDX, where braces are JSX', () => {+ expect(namesFor('mdx')).toEqual(['escaped-inline-code-tags']);+ });++ it('runs every rule against PO catalogs', () => {+ expect(namesFor('po')).toHaveLength(NORMALIZATION_RULES.length);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.tsindex c6ff5cd0d62..a51f74945a0 100644--- a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts+++ b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts@@ -1,7 +1,10 @@import { NORMALIZATION_RULES } from './constants/normalization-rules.constant';import { type CrowdinContext } from './types/crowdin-context.type';import { type CrowdinTranslation } from './types/crowdin-translation.type';-import { type NormalizationRule } from './types/normalization-rule.type';+import {+ type CatalogFormat,+ type NormalizationRule,+} from './types/normalization-rule.type';import { addTranslation } from './utils/add-translation.util';import { deleteTranslation } from './utils/delete-translation.util';import { evaluateRules } from './utils/evaluate-rules.util';@@ -43,20 +46,46 @@ function parseProjectIdOrThrow(): number {return projectId;}+const CATALOG_FORMATS: CatalogFormat[] = ['po', 'mdx'];++// Selecting by format rather than by name means a new rule protects every+// project holding that format, instead of only the workflow someone remembered+// to edit. --rules stays for narrowing a run to one rule by hand.function selectRulesOrThrow(): NormalizationRule[] {+ const requestedFormat = getArgumentValue('format');const requestedNames = (getArgumentValue('rules') ?? '').split(',').map((name) => name.trim()).filter(Boolean);+ if ((requestedFormat === undefined) === (requestedNames.length === 0)) {+ throw new Error(+ `Pass exactly one of --format=${CATALOG_FORMATS.join('|')} or --rules=<name,...>`,+ );+ }++ if (requestedFormat !== undefined) {+ const format = CATALOG_FORMATS.find(+ (candidate) => candidate === requestedFormat,+ );++ if (format === undefined) {+ throw new Error(+ `Unknown --format=${requestedFormat}. Available: ${CATALOG_FORMATS.join(', ')}`,+ );+ }++ return NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format));+ }+const availableNames = NORMALIZATION_RULES.map((rule) => rule.name);const unknownNames = requestedNames.filter((name) => !availableNames.includes(name),);- if (requestedNames.length === 0 || unknownNames.length > 0) {+ if (unknownNames.length > 0) {throw new Error(- `Missing or unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,+ `Unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,);}diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.tsindex 876799b5161..eb896febb97 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts@@ -6,26 +6,48 @@ describe('ESCAPED_UNICODE_RULE', () => {it('restores characters that leaked as literal \\uXXXX sequences', () => {const translationText = '\\u62db\\u8058';- expect(detect(translationText)).toBe(true);- expect(fix(translationText)).toBe('招聘');+ expect(detect(translationText, 'Recruiting')).toBe(true);+ expect(fix(translationText, 'Recruiting')).toBe('招聘');});it('restores surrogate pairs', () => {- expect(fix('ok \\ud83d\\ude00')).toBe('ok 😀');+ expect(fix('ok \\ud83d\\ude00', 'ok 😀')).toBe('ok 😀');});- it('applies to any string, since the corruption can appear anywhere', () => {+ it('keeps an escape the source writes itself, such as a code sample', () => {+ const sourceText = 'Escape the accent as \\u00e9 in the config';+ const translationText = "Echappez l'accent en \\u00e9 dans la configuration";++ expect(detect(translationText, sourceText)).toBe(false);+ expect(fix(translationText, sourceText)).toBe(translationText);+ });++ it('decodes only the escapes the source does not carry', () => {+ const sourceText = 'Use \\u00e9 here';++ expect(fix('Utilisez \\u00e9 ici \\u62db', sourceText)).toBe(+ 'Utilisez \\u00e9 ici 招',+ );+ });++ it('stands down when the source is unknown, rather than decoding blind', () => {+ expect(detect('\\u62db\\u8058', undefined)).toBe(false);+ expect(fix('\\u62db\\u8058', undefined)).toBe('\\u62db\\u8058');+ });++ it('scans every string rather than filtering by source', () => {expect(ESCAPED_UNICODE_RULE.sourceFilter).toBeUndefined();- expect(fix('\\u00e9')).toBe('é');+ expect(ESCAPED_UNICODE_RULE.needsSourceText).toBe(true);});it('leaves text without escaped sequences untouched', () => {- expect(detect('招聘')).toBe(false);+ expect(detect('招聘', 'Recruiting')).toBe(false);});it('is idempotent', () => {- const once = fix('\\u62db');+ const once = fix('\\u62db', 'Recruiting');- expect(fix(once)).toBe(once);+ expect(detect(once, 'Recruiting')).toBe(false);+ expect(fix(once, 'Recruiting')).toBe(once);});});diff --git a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.tsindex 5b661c1ba70..c002138149e 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts@@ -124,6 +124,7 @@ function salvage(text: string, sourceText?: string): string {export const CORRUPTED_MODEL_OUTPUT_RULE: NormalizationRule = {name: 'corrupted-model-output',+ formats: ['po'],needsSourceText: true,detect: isCorrupted,fix: salvage,diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.tsindex 39e99e36aad..b14e2176a3f 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts@@ -33,6 +33,7 @@ function unescapeTagsInInlineCode(text: string): string {export const ESCAPED_INLINE_CODE_TAGS_RULE: NormalizationRule = {name: 'escaped-inline-code-tags',+ formats: ['po', 'mdx'],detect: hasEscapedTagInInlineCode,fix: unescapeTagsInInlineCode,sourceFilter: sourceHasTagInInlineCode,diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.tsindex 5bd0a441713..10021116de3 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts@@ -1,15 +1,33 @@import { type NormalizationRule } from '../types/normalization-rule.type';-const ESCAPED_UNICODE_REGEX = /\\u[0-9a-fA-F]{4}/;+const ESCAPED_UNICODE_REGEX = /\\u([0-9a-fA-F]{4})/g;-function unescapeUnicode(text: string): string {- return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex) =>- String.fromCharCode(parseInt(hex, 16)),+function escapesIn(text: string): string[] {+ return [...text.matchAll(ESCAPED_UNICODE_REGEX)].map(([escape]) => escape);+}++// A source that writes the escape itself - a code sample, a string about+// escaping - means the translation is right to carry it through. Only an escape+// the translator introduced is a decoding failure, and without the source there+// is no way to tell the two apart, so the rule stands down as its peers do.+function hasEscapedUnicode(text: string, sourceText?: string): boolean {+ if (sourceText === undefined) return false;++ return escapesIn(text).some((escape) => !sourceText.includes(escape));+}++function unescapeUnicode(text: string, sourceText?: string): string {+ if (sourceText === undefined) return text;++ return text.replace(ESCAPED_UNICODE_REGEX, (escape, hex) =>+ sourceText.includes(escape) ? escape : String.fromCharCode(parseInt(hex, 16)),);}export const ESCAPED_UNICODE_RULE: NormalizationRule = {name: 'escaped-unicode',- detect: (text) => ESCAPED_UNICODE_REGEX.test(text),+ formats: ['po'],+ needsSourceText: true,+ detect: hasEscapedUnicode,fix: unescapeUnicode,};diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.tsindex 0f1c0a5eb02..54e421db582 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts@@ -80,6 +80,7 @@ function repairInventedArgument(text: string, sourceText?: string): string {export const INVENTED_ARGUMENT_RULE: NormalizationRule = {name: 'invented-argument',+ formats: ['po'],needsSourceText: true,detect: hasInventedArgument,fix: repairInventedArgument,diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.tsindex f411e13e73b..8bfab2defb8 100644--- a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts@@ -25,6 +25,7 @@ function removeInventedMarkup(text: string): string {export const INVENTED_MARKUP_RULE: NormalizationRule = {name: 'invented-markup',+ formats: ['po'],needsSourceText: true,detect: hasInventedMarkup,fix: removeInventedMarkup,diff --git a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.tsindex acee21fa150..37d5f067563 100644--- a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts+++ b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts@@ -1,5 +1,11 @@+// Catalogs Crowdin holds for Twenty: Lingui PO for the app and website, MDX for+// the docs. ICU braces mean something different in each, so rules declare which+// they are safe to run against rather than being opted in per workflow.+export type CatalogFormat = 'po' | 'mdx';+export type NormalizationRule = {name: string;+ formats: CatalogFormat[];detect: (text: string, sourceText?: string) => boolean;// Returning an empty string means the translation could not be salvaged and// should be deleted rather than replaced.