Skip to content

The open alternative to Salesforce, designed for AI.

Delete translations that came back as debris rather than words (#25853)

Félix Malfait273851a06ee3
diff --git a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
index 7c358e1dabf..25c95ca9773 100644
--- a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
+++ b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
@@ -3,6 +3,7 @@ import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tags
import { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule';
import { INVENTED_ARGUMENT_RULE } from '../rules/invented-argument.rule';
import { INVENTED_MARKUP_RULE } from '../rules/invented-markup.rule';
+import { WORDLESS_TRANSLATION_RULE } from '../rules/wordless-translation.rule';
import { type NormalizationRule } from '../types/normalization-rule.type';
export const NORMALIZATION_RULES: NormalizationRule[] = [
@@ -11,4 +12,5 @@ export const NORMALIZATION_RULES: NormalizationRule[] = [
CORRUPTED_MODEL_OUTPUT_RULE,
INVENTED_MARKUP_RULE,
INVENTED_ARGUMENT_RULE,
+ WORDLESS_TRANSLATION_RULE,
];
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.ts
new file mode 100644
index 00000000000..cc716c4afa2
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.ts
@@ -0,0 +1,72 @@
+import { WORDLESS_TRANSLATION_RULE } from '../wordless-translation.rule';
+
+const { detect, fix } = WORDLESS_TRANSLATION_RULE;
+
+describe('WORDLESS_TRANSLATION_RULE', () => {
+ it('drops a translation that is a lone backslash', () => {
+ expect(detect('\\', 'The values of this field must be unique')).toBe(true);
+ expect(fix('\\', 'The values of this field must be unique')).toBe('');
+ });
+
+ it('drops a translation that is an escaped quote pair', () => {
+ expect(detect("''", 'Count false')).toBe(true);
+ expect(fix("''", 'Count false')).toBe('');
+ });
+
+ it('drops punctuation and whitespace debris', () => {
+ expect(detect(' .,; ', 'Regenerate key')).toBe(true);
+ });
+
+ // \w is ASCII-only in JavaScript, so a naive check would delete every
+ // translation in a non-Latin script.
+ it.each([
+ ['他們的值必須是唯一的', 'Chinese'],
+ ['הערכים חייבים להיות ייחודיים', 'Hebrew'],
+ ['القيم يجب أن تكون فريدة', 'Arabic'],
+ ['Значения должны быть уникальными', 'Cyrillic'],
+ ['Արժեքներըպետք է եզակի լինեն', 'Armenian'],
+ ['値は一意である必要があります', 'Japanese'],
+ ])('keeps %s (%s)', (translationText) => {
+ expect(detect(translationText, 'The values must be unique')).toBe(false);
+ });
+
+ it('keeps a translation that is only digits, which are words enough', () => {
+ expect(detect('2026', 'Year')).toBe(false);
+ });
+
+ it('keeps a wordless translation when the source has no words either', () => {
+ expect(detect('—', '—')).toBe(false);
+ expect(detect(':', ':')).toBe(false);
+ });
+
+ it('keeps a translation that is only a placeholder, as its source is', () => {
+ expect(detect('{count}', '{count}')).toBe(false);
+ });
+
+ // Ignoring placeholder names would catch a translation left as a bare
+ // {count}, but a dry run over ~200k strings found none of those and two
+ // Finnish translations that legitimately render a connector word as a
+ // symbol. Deleting a good translation is worse than keeping a thin one, so
+ // the rule reads the whole string.
+ it('keeps a translation that renders a connector word as a symbol', () => {
+ expect(detect('{0} / {1}', '{0} of {1}')).toBe(false);
+ expect(detect(' /{intervalLabel}', ' per {intervalLabel}')).toBe(false);
+ });
+
+ it('stands down when the source is unknown, rather than deleting blind', () => {
+ expect(detect('\\', undefined)).toBe(false);
+ });
+
+ it('leaves a translation already dropped by an earlier rule alone', () => {
+ expect(detect('', 'Regenerate key')).toBe(false);
+ });
+
+ it('scans every string rather than filtering by source', () => {
+ expect(WORDLESS_TRANSLATION_RULE.sourceFilter).toBeUndefined();
+ expect(WORDLESS_TRANSLATION_RULE.needsSourceText).toBe(true);
+ });
+
+ it('runs against PO catalogs only', () => {
+ expect(WORDLESS_TRANSLATION_RULE.formats).toEqual(['po']);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.ts
new file mode 100644
index 00000000000..2b9e8dc53d0
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.ts
@@ -0,0 +1,38 @@
+import { type NormalizationRule } from '../types/normalization-rule.type';
+
+// \w is ASCII-only in JavaScript, which would read every Chinese, Arabic or
+// Hebrew translation as wordless. Only the Unicode letter and number categories
+// hold across the scripts we translate into.
+const WORD_CHARACTER_REGEX = /[\p{L}\p{N}]/u;
+
+function hasWordCharacter(text: string): boolean {
+ return WORD_CHARACTER_REGEX.test(text);
+}
+
+// A translation carrying no letter or digit at all cannot be a rendering of a
+// source that carries them - it is a stray backslash, an escaped quote pair,
+// punctuation debris. A source with no words of its own, such as a lone
+// placeholder or a separator, is left alone: there a wordless translation is
+// the correct one.
+function isWordlessTranslation(text: string, sourceText?: string): boolean {
+ // Rules run in sequence on each other's output, so an empty text means an
+ // earlier rule already dropped this translation: firing again would only
+ // report the same deletion twice.
+ if (sourceText === undefined || text === '') return false;
+
+ return hasWordCharacter(sourceText) && !hasWordCharacter(text);
+}
+
+// Debris holds nothing to rebuild the missing words from, so the translation is
+// dropped and the string falls back to English until Crowdin retranslates it.
+function dropTranslation(): string {
+ return '';
+}
+
+export const WORDLESS_TRANSLATION_RULE: NormalizationRule = {
+ name: 'wordless-translation',
+ formats: ['po'],
+ needsSourceText: true,
+ detect: isWordlessTranslation,
+ fix: dropTranslation,
+};