The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Delete translations that came back as debris rather than words (#25853)
Félix Malfait273851a06ee3
diff --git a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.tsindex 7c358e1dabf..25c95ca9773 100644--- a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts+++ b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts@@ -3,6 +3,7 @@ import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tagsimport { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule';import { INVENTED_ARGUMENT_RULE } from '../rules/invented-argument.rule';import { INVENTED_MARKUP_RULE } from '../rules/invented-markup.rule';+import { WORDLESS_TRANSLATION_RULE } from '../rules/wordless-translation.rule';import { type NormalizationRule } from '../types/normalization-rule.type';export const NORMALIZATION_RULES: NormalizationRule[] = [@@ -11,4 +12,5 @@ export const NORMALIZATION_RULES: NormalizationRule[] = [CORRUPTED_MODEL_OUTPUT_RULE,INVENTED_MARKUP_RULE,INVENTED_ARGUMENT_RULE,+ WORDLESS_TRANSLATION_RULE,];diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.tsnew file mode 100644index 00000000000..cc716c4afa2--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/wordless-translation.rule.spec.ts@@ -0,0 +1,72 @@+import { WORDLESS_TRANSLATION_RULE } from '../wordless-translation.rule';++const { detect, fix } = WORDLESS_TRANSLATION_RULE;++describe('WORDLESS_TRANSLATION_RULE', () => {+ it('drops a translation that is a lone backslash', () => {+ expect(detect('\\', 'The values of this field must be unique')).toBe(true);+ expect(fix('\\', 'The values of this field must be unique')).toBe('');+ });++ it('drops a translation that is an escaped quote pair', () => {+ expect(detect("''", 'Count false')).toBe(true);+ expect(fix("''", 'Count false')).toBe('');+ });++ it('drops punctuation and whitespace debris', () => {+ expect(detect(' .,; ', 'Regenerate key')).toBe(true);+ });++ // \w is ASCII-only in JavaScript, so a naive check would delete every+ // translation in a non-Latin script.+ it.each([+ ['他們的值必須是唯一的', 'Chinese'],+ ['הערכים חייבים להיות ייחודיים', 'Hebrew'],+ ['القيم يجب أن تكون فريدة', 'Arabic'],+ ['Значения должны быть уникальными', 'Cyrillic'],+ ['Արժեքներըպետք է եզակի լինեն', 'Armenian'],+ ['値は一意である必要があります', 'Japanese'],+ ])('keeps %s (%s)', (translationText) => {+ expect(detect(translationText, 'The values must be unique')).toBe(false);+ });++ it('keeps a translation that is only digits, which are words enough', () => {+ expect(detect('2026', 'Year')).toBe(false);+ });++ it('keeps a wordless translation when the source has no words either', () => {+ expect(detect('—', '—')).toBe(false);+ expect(detect(':', ':')).toBe(false);+ });++ it('keeps a translation that is only a placeholder, as its source is', () => {+ expect(detect('{count}', '{count}')).toBe(false);+ });++ // Ignoring placeholder names would catch a translation left as a bare+ // {count}, but a dry run over ~200k strings found none of those and two+ // Finnish translations that legitimately render a connector word as a+ // symbol. Deleting a good translation is worse than keeping a thin one, so+ // the rule reads the whole string.+ it('keeps a translation that renders a connector word as a symbol', () => {+ expect(detect('{0} / {1}', '{0} of {1}')).toBe(false);+ expect(detect(' /{intervalLabel}', ' per {intervalLabel}')).toBe(false);+ });++ it('stands down when the source is unknown, rather than deleting blind', () => {+ expect(detect('\\', undefined)).toBe(false);+ });++ it('leaves a translation already dropped by an earlier rule alone', () => {+ expect(detect('', 'Regenerate key')).toBe(false);+ });++ it('scans every string rather than filtering by source', () => {+ expect(WORDLESS_TRANSLATION_RULE.sourceFilter).toBeUndefined();+ expect(WORDLESS_TRANSLATION_RULE.needsSourceText).toBe(true);+ });++ it('runs against PO catalogs only', () => {+ expect(WORDLESS_TRANSLATION_RULE.formats).toEqual(['po']);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.tsnew file mode 100644index 00000000000..2b9e8dc53d0--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/wordless-translation.rule.ts@@ -0,0 +1,38 @@+import { type NormalizationRule } from '../types/normalization-rule.type';++// \w is ASCII-only in JavaScript, which would read every Chinese, Arabic or+// Hebrew translation as wordless. Only the Unicode letter and number categories+// hold across the scripts we translate into.+const WORD_CHARACTER_REGEX = /[\p{L}\p{N}]/u;++function hasWordCharacter(text: string): boolean {+ return WORD_CHARACTER_REGEX.test(text);+}++// A translation carrying no letter or digit at all cannot be a rendering of a+// source that carries them - it is a stray backslash, an escaped quote pair,+// punctuation debris. A source with no words of its own, such as a lone+// placeholder or a separator, is left alone: there a wordless translation is+// the correct one.+function isWordlessTranslation(text: string, sourceText?: string): boolean {+ // Rules run in sequence on each other's output, so an empty text means an+ // earlier rule already dropped this translation: firing again would only+ // report the same deletion twice.+ if (sourceText === undefined || text === '') return false;++ return hasWordCharacter(sourceText) && !hasWordCharacter(text);+}++// Debris holds nothing to rebuild the missing words from, so the translation is+// dropped and the string falls back to English until Crowdin retranslates it.+function dropTranslation(): string {+ return '';+}++export const WORDLESS_TRANSLATION_RULE: NormalizationRule = {+ name: 'wordless-translation',+ formats: ['po'],+ needsSourceText: true,+ detect: isWordlessTranslation,+ fix: dropTranslation,+};