Skip to content

The open alternative to Salesforce, designed for AI.

Cover every translation pipeline with the normalizer (#25848)

Félix Malfaitdfa3a39efe94
diff --git a/.github/workflows/docs-i18n-pull.yaml b/.github/workflows/docs-i18n-pull.yaml
index 00821920f65..576cffb16bd 100644
--- a/.github/workflows/docs-i18n-pull.yaml
+++ b/.github/workflows/docs-i18n-pull.yaml
@@ -83,7 +83,7 @@ jobs:
- name: Normalize translations in Crowdin
if: github.event_name != 'pull_request' && (inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
- run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=2 --apply --rules=escaped-inline-code-tags || echo "::warning::translation normalization failed (non-blocking)"
+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=2 --format=mdx --apply || echo "::warning::translation normalization failed (non-blocking)"
env:
CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
diff --git a/.github/workflows/i18n-pull.yaml b/.github/workflows/i18n-pull.yaml
index b9c59ed868e..10ed88bc5be 100644
--- a/.github/workflows/i18n-pull.yaml
+++ b/.github/workflows/i18n-pull.yaml
@@ -55,6 +55,14 @@ jobs:
git add .
git stash
+ # Repair machine-translation damage in Crowdin before downloading, so the
+ # catalogs that reach the repo are already correct rather than a cycle behind.
+ - name: Normalize translations in Crowdin
+ if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --format=po --apply || echo "::warning::translation normalization failed (non-blocking)"
+ env:
+ CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+
- name: Pull translations from Crowdin
if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2
@@ -87,14 +95,6 @@ jobs:
- name: Fix file permissions
run: sudo chown -R runner:docker .
- # Fix encoding issues (escaped Unicode like \u62db -> 招) and translations carrying a
- # leaked response envelope, then push the fixes back to Crowdin so they
- # survive the next sync.
- - name: Normalize translations in Crowdin
- run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode,corrupted-model-output,invented-markup,invented-argument || echo "::warning::translation normalization failed (non-blocking)"
- env:
- CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
-
- name: Normalize downloaded catalogs to lingui format
run: |
npx nx run twenty-server:lingui:extract
diff --git a/.github/workflows/website-i18n-pull.yaml b/.github/workflows/website-i18n-pull.yaml
index da4d1e76444..3a0ee7ad073 100644
--- a/.github/workflows/website-i18n-pull.yaml
+++ b/.github/workflows/website-i18n-pull.yaml
@@ -55,6 +55,14 @@ jobs:
git add .
git stash
+ # Repair machine-translation damage in Crowdin before downloading, so the
+ # catalogs that reach the repo are already correct rather than a cycle behind.
+ - name: Normalize translations in Crowdin
+ if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=4 --format=po --apply || echo "::warning::translation normalization failed (non-blocking)"
+ env:
+ CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+
- name: Pull website translations from Crowdin
if: inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
uses: crowdin/github-action@8868a33591d21088edfc398968173a3b98d51706 # v2
diff --git a/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx b/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx
index fdd802311fb..b746b84e01a 100644
--- a/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx
+++ b/packages/twenty-front/src/modules/command-menu-item/edit/components/CommandMenuItemEditRecordSelectionDropdown.tsx
@@ -79,7 +79,7 @@ export const CommandMenuItemEditRecordSelectionDropdown = ({
const TriggerIcon = isNoneSelected ? IconSquareX : IconSquareCheck;
const triggerLabel = isNoneSelected
? t`No record selected`
- : t`Record(s) selected`;
+ : t`Records selected`;
return (
<Dropdown
@@ -119,7 +119,7 @@ export const CommandMenuItemEditRecordSelectionDropdown = ({
/>
<MenuItemSelect
LeftIcon={IconSquareCheck}
- text={t`Record(s) selected`}
+ text={t`Records selected`}
selected={!isNoneSelected}
onClick={() => handleSelectMode('selection')}
/>
diff --git a/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.ts b/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.ts
new file mode 100644
index 00000000000..4e1844ee095
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/constants/__tests__/normalization-rules.constant.spec.ts
@@ -0,0 +1,28 @@
+import { NORMALIZATION_RULES } from '../normalization-rules.constant';
+
+const namesFor = (format: 'po' | 'mdx') =>
+ NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format)).map(
+ (rule) => rule.name,
+ );
+
+describe('NORMALIZATION_RULES', () => {
+ it('gives every rule at least one format to run against', () => {
+ for (const rule of NORMALIZATION_RULES) {
+ expect(rule.formats.length).toBeGreaterThan(0);
+ }
+ });
+
+ it('names every rule uniquely', () => {
+ const names = NORMALIZATION_RULES.map((rule) => rule.name);
+
+ expect(new Set(names).size).toBe(names.length);
+ });
+
+ it('keeps the ICU rules away from MDX, where braces are JSX', () => {
+ expect(namesFor('mdx')).toEqual(['escaped-inline-code-tags']);
+ });
+
+ it('runs every rule against PO catalogs', () => {
+ expect(namesFor('po')).toHaveLength(NORMALIZATION_RULES.length);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
index c6ff5cd0d62..a51f74945a0 100644
--- a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
+++ b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
@@ -1,7 +1,10 @@
import { NORMALIZATION_RULES } from './constants/normalization-rules.constant';
import { type CrowdinContext } from './types/crowdin-context.type';
import { type CrowdinTranslation } from './types/crowdin-translation.type';
-import { type NormalizationRule } from './types/normalization-rule.type';
+import {
+ type CatalogFormat,
+ type NormalizationRule,
+} from './types/normalization-rule.type';
import { addTranslation } from './utils/add-translation.util';
import { deleteTranslation } from './utils/delete-translation.util';
import { evaluateRules } from './utils/evaluate-rules.util';
@@ -43,20 +46,46 @@ function parseProjectIdOrThrow(): number {
return projectId;
}
+const CATALOG_FORMATS: CatalogFormat[] = ['po', 'mdx'];
+
+// Selecting by format rather than by name means a new rule protects every
+// project holding that format, instead of only the workflow someone remembered
+// to edit. --rules stays for narrowing a run to one rule by hand.
function selectRulesOrThrow(): NormalizationRule[] {
+ const requestedFormat = getArgumentValue('format');
const requestedNames = (getArgumentValue('rules') ?? '')
.split(',')
.map((name) => name.trim())
.filter(Boolean);
+ if ((requestedFormat === undefined) === (requestedNames.length === 0)) {
+ throw new Error(
+ `Pass exactly one of --format=${CATALOG_FORMATS.join('|')} or --rules=<name,...>`,
+ );
+ }
+
+ if (requestedFormat !== undefined) {
+ const format = CATALOG_FORMATS.find(
+ (candidate) => candidate === requestedFormat,
+ );
+
+ if (format === undefined) {
+ throw new Error(
+ `Unknown --format=${requestedFormat}. Available: ${CATALOG_FORMATS.join(', ')}`,
+ );
+ }
+
+ return NORMALIZATION_RULES.filter((rule) => rule.formats.includes(format));
+ }
+
const availableNames = NORMALIZATION_RULES.map((rule) => rule.name);
const unknownNames = requestedNames.filter(
(name) => !availableNames.includes(name),
);
- if (requestedNames.length === 0 || unknownNames.length > 0) {
+ if (unknownNames.length > 0) {
throw new Error(
- `Missing or unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,
+ `Unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,
);
}
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts
index 876799b5161..eb896febb97 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts
@@ -6,26 +6,48 @@ describe('ESCAPED_UNICODE_RULE', () => {
it('restores characters that leaked as literal \\uXXXX sequences', () => {
const translationText = '\\u62db\\u8058';
- expect(detect(translationText)).toBe(true);
- expect(fix(translationText)).toBe('招聘');
+ expect(detect(translationText, 'Recruiting')).toBe(true);
+ expect(fix(translationText, 'Recruiting')).toBe('招聘');
});
it('restores surrogate pairs', () => {
- expect(fix('ok \\ud83d\\ude00')).toBe('ok 😀');
+ expect(fix('ok \\ud83d\\ude00', 'ok 😀')).toBe('ok 😀');
});
- it('applies to any string, since the corruption can appear anywhere', () => {
+ it('keeps an escape the source writes itself, such as a code sample', () => {
+ const sourceText = 'Escape the accent as \\u00e9 in the config';
+ const translationText = "Echappez l'accent en \\u00e9 dans la configuration";
+
+ expect(detect(translationText, sourceText)).toBe(false);
+ expect(fix(translationText, sourceText)).toBe(translationText);
+ });
+
+ it('decodes only the escapes the source does not carry', () => {
+ const sourceText = 'Use \\u00e9 here';
+
+ expect(fix('Utilisez \\u00e9 ici \\u62db', sourceText)).toBe(
+ 'Utilisez \\u00e9 ici 招',
+ );
+ });
+
+ it('stands down when the source is unknown, rather than decoding blind', () => {
+ expect(detect('\\u62db\\u8058', undefined)).toBe(false);
+ expect(fix('\\u62db\\u8058', undefined)).toBe('\\u62db\\u8058');
+ });
+
+ it('scans every string rather than filtering by source', () => {
expect(ESCAPED_UNICODE_RULE.sourceFilter).toBeUndefined();
- expect(fix('\\u00e9')).toBe('é');
+ expect(ESCAPED_UNICODE_RULE.needsSourceText).toBe(true);
});
it('leaves text without escaped sequences untouched', () => {
- expect(detect('招聘')).toBe(false);
+ expect(detect('招聘', 'Recruiting')).toBe(false);
});
it('is idempotent', () => {
- const once = fix('\\u62db');
+ const once = fix('\\u62db', 'Recruiting');
- expect(fix(once)).toBe(once);
+ expect(detect(once, 'Recruiting')).toBe(false);
+ expect(fix(once, 'Recruiting')).toBe(once);
});
});
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts
index 5b661c1ba70..c002138149e 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts
@@ -124,6 +124,7 @@ function salvage(text: string, sourceText?: string): string {
export const CORRUPTED_MODEL_OUTPUT_RULE: NormalizationRule = {
name: 'corrupted-model-output',
+ formats: ['po'],
needsSourceText: true,
detect: isCorrupted,
fix: salvage,
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts
index 39e99e36aad..b14e2176a3f 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts
@@ -33,6 +33,7 @@ function unescapeTagsInInlineCode(text: string): string {
export const ESCAPED_INLINE_CODE_TAGS_RULE: NormalizationRule = {
name: 'escaped-inline-code-tags',
+ formats: ['po', 'mdx'],
detect: hasEscapedTagInInlineCode,
fix: unescapeTagsInInlineCode,
sourceFilter: sourceHasTagInInlineCode,
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts
index 5bd0a441713..10021116de3 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts
@@ -1,15 +1,33 @@
import { type NormalizationRule } from '../types/normalization-rule.type';
-const ESCAPED_UNICODE_REGEX = /\\u[0-9a-fA-F]{4}/;
+const ESCAPED_UNICODE_REGEX = /\\u([0-9a-fA-F]{4})/g;
-function unescapeUnicode(text: string): string {
- return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex) =>
- String.fromCharCode(parseInt(hex, 16)),
+function escapesIn(text: string): string[] {
+ return [...text.matchAll(ESCAPED_UNICODE_REGEX)].map(([escape]) => escape);
+}
+
+// A source that writes the escape itself - a code sample, a string about
+// escaping - means the translation is right to carry it through. Only an escape
+// the translator introduced is a decoding failure, and without the source there
+// is no way to tell the two apart, so the rule stands down as its peers do.
+function hasEscapedUnicode(text: string, sourceText?: string): boolean {
+ if (sourceText === undefined) return false;
+
+ return escapesIn(text).some((escape) => !sourceText.includes(escape));
+}
+
+function unescapeUnicode(text: string, sourceText?: string): string {
+ if (sourceText === undefined) return text;
+
+ return text.replace(ESCAPED_UNICODE_REGEX, (escape, hex) =>
+ sourceText.includes(escape) ? escape : String.fromCharCode(parseInt(hex, 16)),
);
}
export const ESCAPED_UNICODE_RULE: NormalizationRule = {
name: 'escaped-unicode',
- detect: (text) => ESCAPED_UNICODE_REGEX.test(text),
+ formats: ['po'],
+ needsSourceText: true,
+ detect: hasEscapedUnicode,
fix: unescapeUnicode,
};
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts
index 0f1c0a5eb02..54e421db582 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts
@@ -80,6 +80,7 @@ function repairInventedArgument(text: string, sourceText?: string): string {
export const INVENTED_ARGUMENT_RULE: NormalizationRule = {
name: 'invented-argument',
+ formats: ['po'],
needsSourceText: true,
detect: hasInventedArgument,
fix: repairInventedArgument,
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts
index f411e13e73b..8bfab2defb8 100644
--- a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts
+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts
@@ -25,6 +25,7 @@ function removeInventedMarkup(text: string): string {
export const INVENTED_MARKUP_RULE: NormalizationRule = {
name: 'invented-markup',
+ formats: ['po'],
needsSourceText: true,
detect: hasInventedMarkup,
fix: removeInventedMarkup,
diff --git a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
index acee21fa150..37d5f067563 100644
--- a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
+++ b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
@@ -1,5 +1,11 @@
+// Catalogs Crowdin holds for Twenty: Lingui PO for the app and website, MDX for
+// the docs. ICU braces mean something different in each, so rules declare which
+// they are safe to run against rather than being opted in per workflow.
+export type CatalogFormat = 'po' | 'mdx';
+
export type NormalizationRule = {
name: string;
+ formats: CatalogFormat[];
detect: (text: string, sourceText?: string) => boolean;
// Returning an empty string means the translation could not be salvaged and
// should be deleted rather than replaced.