Skip to content

The open alternative to Salesforce, designed for AI.

Repair machine-translation damage in Crowdin automatically (#25842)

Félix Malfait40040983de99
diff --git a/.github/workflows/i18n-pull.yaml b/.github/workflows/i18n-pull.yaml
index eccf004681a..ecac4bf51f0 100644
--- a/.github/workflows/i18n-pull.yaml
+++ b/.github/workflows/i18n-pull.yaml
@@ -87,9 +87,11 @@ jobs:
- name: Fix file permissions
run: sudo chown -R runner:docker .
- # Fix encoding issues (escaped Unicode like \u62db -> 招) and push fixes back to Crowdin
+ # Fix encoding issues (escaped Unicode like \u62db -> 招) and translations carrying a
+ # leaked response envelope, then push the fixes back to Crowdin so they
+ # survive the next sync.
- name: Normalize translations in Crowdin
- run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode || echo "::warning::translation normalization failed (non-blocking)"
+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode,corrupted-model-output,invented-markup,invented-argument || echo "::warning::translation normalization failed (non-blocking)"
env:
CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
diff --git a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
index 1a96b7f4a46..7c358e1dabf 100644
--- a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
+++ b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts
@@ -1,8 +1,14 @@
+import { CORRUPTED_MODEL_OUTPUT_RULE } from '../rules/corrupted-model-output.rule';
import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tags.rule';
import { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule';
+import { INVENTED_ARGUMENT_RULE } from '../rules/invented-argument.rule';
+import { INVENTED_MARKUP_RULE } from '../rules/invented-markup.rule';
import { type NormalizationRule } from '../types/normalization-rule.type';
export const NORMALIZATION_RULES: NormalizationRule[] = [
ESCAPED_UNICODE_RULE,
ESCAPED_INLINE_CODE_TAGS_RULE,
+ CORRUPTED_MODEL_OUTPUT_RULE,
+ INVENTED_MARKUP_RULE,
+ INVENTED_ARGUMENT_RULE,
];
diff --git a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
index 40f2a76ba06..c6ff5cd0d62 100644
--- a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
+++ b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts
@@ -163,11 +163,16 @@ async function repairOne(
finding: NormalizationFinding,
): Promise<boolean> {
try {
- await addTranslation(context, {
- stringId: finding.stringId,
- languageId: finding.languageId,
- text: finding.fixedText,
- });
+ // An empty repair means the translation could not be salvaged: deleting it
+ // without a replacement falls back to English until Crowdin retranslates.
+ if (finding.fixedText !== '') {
+ await addTranslation(context, {
+ stringId: finding.stringId,
+ languageId: finding.languageId,
+ text: finding.fixedText,
+ });
+ }
+
await deleteTranslation(context, { translationId: finding.translationId });
return true;
@@ -206,7 +211,9 @@ async function main() {
);
console.log(`Rules: ${rules.map((rule) => rule.name).join(', ')}`);
- const needsSourceStrings = rules.some((rule) => rule.sourceFilter);
+ const needsSourceStrings = rules.some(
+ (rule) => rule.sourceFilter !== undefined || rule.needsSourceText === true,
+ );
const sourceStrings = needsSourceStrings
? await fetchSourceStringsById(context)
: undefined;
@@ -221,7 +228,11 @@ async function main() {
);
if (finding.sourceText) console.log(` source: ${finding.sourceText}`);
console.log(` trans: ${finding.originalText}`);
- console.log(` fixed: ${finding.fixedText}`);
+ console.log(
+ finding.fixedText === ''
+ ? ' fixed: <deleted, falls back to English until retranslated>'
+ : ` fixed: ${finding.fixedText}`,
+ );
}
if (findings.length === 0) {
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts
new file mode 100644
index 00000000000..e040f5d6b5b
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/corrupted-model-output.rule.spec.ts
@@ -0,0 +1,99 @@
+import { CORRUPTED_MODEL_OUTPUT_RULE } from '../corrupted-model-output.rule';
+
+const { detect, fix } = CORRUPTED_MODEL_OUTPUT_RULE;
+
+describe('CORRUPTED_MODEL_OUTPUT_RULE', () => {
+ it('cuts a leaked response envelope off a plain translation', () => {
+ const sourceText = 'Group name';
+ const translationText =
+ 'Nome do grupo}]}``` }}} code block? Wait. We messed up.';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe('Nome do grupo');
+ });
+
+ it('cuts a trailing JSON array separator', () => {
+ expect(detect('レコードテーブルを編集},{', 'Edit Record Table')).toBe(true);
+ expect(fix('レコードテーブルを編集},{', 'Edit Record Table')).toBe(
+ 'レコードテーブルを編集',
+ );
+ });
+
+ it('keeps a plural block intact while dropping the envelope after it', () => {
+ const sourceText =
+ '{hiddenFieldCount, plural, one {# more populated field available} other {# more populated fields available}}';
+ const translationText =
+ '{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}}]}દાવાદ്?ฤศจassistant to=developer';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe(
+ '{hiddenFieldCount, plural, other {還有 # 個已填入欄位可用}}',
+ );
+ });
+
+ it('keeps a closing quote the source also ends with', () => {
+ const sourceText = 'Set fields created in the future as "visible"';
+ const translationText =
+ 'Aseta tulevaisuudessa luotavat kentät "näkyviksi"}]}``` }]} } } }';
+
+ expect(fix(translationText, sourceText)).toBe(
+ 'Aseta tulevaisuudessa luotavat kentät "näkyviksi"',
+ );
+ });
+
+ it('drops a trailing apostrophe the source does not carry', () => {
+ const sourceText = 'An error occurred';
+
+ expect(fix("'n Fout het voorgekom'}]}```}``` }```{", sourceText)).toBe(
+ "'n Fout het voorgekom",
+ );
+ });
+
+ it('deletes a translation whose salvageable prefix is still envelope', () => {
+ const sourceText = 'Trigger';
+ const translationText = 'הפעל","pluralForm":null}]}]}{';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe('');
+ });
+
+ it('deletes a translation with an unclosed brace and no clean cut', () => {
+ expect(fix('Nome do grupo {broken', 'Group name')).toBe('');
+ });
+
+ it('keeps markdown links when the source carries them too', () => {
+ const sourceText = 'See [Getting Started](https://twenty.com/docs) first.';
+ const translationText =
+ 'Consulte [Introdução](https://twenty.com/docs) primeiro.},{';
+
+ expect(fix(translationText, sourceText)).toBe(
+ 'Consulte [Introdução](https://twenty.com/docs) primeiro.',
+ );
+ });
+
+ it('does not flag a healthy translation', () => {
+ expect(detect('Nome do grupo', 'Group name')).toBe(false);
+ expect(
+ detect(
+ '{count, plural, other {# champs}}',
+ '{count, plural, one {# field} other {# fields}}',
+ ),
+ ).toBe(false);
+ });
+
+ it('does not flag when the source is unavailable', () => {
+ expect(detect('Nome do grupo}]}', undefined)).toBe(false);
+ });
+
+ it('does not flag when the source is itself unbalanced', () => {
+ expect(detect('Algo}', 'Something}')).toBe(false);
+ });
+
+ it('is idempotent', () => {
+ const sourceText = 'Group name';
+ const once = fix('Nome do grupo}]}``` }}}', sourceText);
+
+ expect(detect(once, sourceText)).toBe(false);
+ expect(fix(once, sourceText)).toBe(once);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts
new file mode 100644
index 00000000000..7dde649830d
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-argument.rule.spec.ts
@@ -0,0 +1,66 @@
+import { INVENTED_ARGUMENT_RULE } from '../invented-argument.rule';
+
+const { detect, fix } = INVENTED_ARGUMENT_RULE;
+
+describe('INVENTED_ARGUMENT_RULE', () => {
+ it('restores the source spelling of a recased argument', () => {
+ const sourceText =
+ 'Multiple records found for {conflictingFieldsValues}. Cannot determine which record to update.';
+ const translationText =
+ '{ConflictingFieldsValues} のために複数のレコードが見つかりました。';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe(
+ '{conflictingFieldsValues} のために複数のレコードが見つかりました。',
+ );
+ });
+
+ it('deletes a translation that pluralises an argument-less source', () => {
+ const sourceText = 'Record(s) selected';
+ const translationText =
+ '{count, plural, one {# registro selecionado} other {# registros selecionados}}';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe('');
+ });
+
+ it('deletes a translation that adds an argument nothing will supply', () => {
+ const sourceText = 'We keep your data for {dataRetentionDays} days.';
+ const translationText =
+ 'Чувамо податке {dataRetentionDays} {dayOrDays}.';
+
+ expect(fix(translationText, sourceText)).toBe('');
+ });
+
+ it('allows a translation to add plural cases its locale needs', () => {
+ const sourceText = '{count, plural, one {# record} other {# records}}';
+ const translationText =
+ '{count, plural, one {# запись} few {# записи} many {# записей} other {# записи}}';
+
+ expect(detect(translationText, sourceText)).toBe(false);
+ });
+
+ it('does not read plural case bodies as arguments', () => {
+ const sourceText = '{unitCount, plural, one {Day} other {Days}}';
+ const translationText = '{unitCount, plural, one {Dag} other {Dae}}';
+
+ expect(detect(translationText, sourceText)).toBe(false);
+ });
+
+ it('does not flag a matching translation or a missing source', () => {
+ expect(detect('Supprimer {name}', 'Delete {name}')).toBe(false);
+ expect(detect('Supprimer {objet}', undefined)).toBe(false);
+ });
+
+ it('ignores a dropped argument, which it cannot repair', () => {
+ expect(detect('Supprimer', 'Delete {name}')).toBe(false);
+ });
+
+ it('is idempotent', () => {
+ const sourceText = 'Found for {conflictingFieldsValues}.';
+ const once = fix('{ConflictingFieldsValues} で見つかりました。', sourceText);
+
+ expect(detect(once, sourceText)).toBe(false);
+ expect(fix(once, sourceText)).toBe(once);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts
new file mode 100644
index 00000000000..93948a32c50
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/invented-markup.rule.spec.ts
@@ -0,0 +1,54 @@
+import { INVENTED_MARKUP_RULE } from '../invented-markup.rule';
+
+const { detect, fix } = INVENTED_MARKUP_RULE;
+
+describe('INVENTED_MARKUP_RULE', () => {
+ it('strips a highlight span the source never had', () => {
+ const sourceText = 'is not a valid calling code';
+ const translationText =
+ 'n\'est pas un <span class="highlight">code d\'appel valide</span>';
+
+ expect(detect(translationText, sourceText)).toBe(true);
+ expect(fix(translationText, sourceText)).toBe(
+ "n'est pas un code d'appel valide",
+ );
+ });
+
+ it('strips nested direction spans', () => {
+ const translationText =
+ '<span dir="rtl"><span dir="ltr">כשל זמני</span></span>';
+
+ expect(fix(translationText, 'Temporary Failure')).toBe('כשל זמני');
+ });
+
+ it('strips bold added around numbers and punctuation', () => {
+ expect(fix('過去<b>12</b>時間', 'Last 12 hours')).toBe('過去12時間');
+ expect(fix('API名<b>(</b>単数形<b>)</b>', 'API Name (Singular)')).toBe(
+ 'API名(単数形)',
+ );
+ });
+
+ it('leaves a translation alone when the source carries markup of its own', () => {
+ const sourceText = 'Your workspace <0>{name}</0> was deleted.';
+ const translationText = 'Ваш простор <0>{name}</0> је обрисан.';
+
+ expect(detect(translationText, sourceText)).toBe(false);
+ });
+
+ it('leaves a Lingui-tagged source alone even when the translation uses html', () => {
+ expect(detect('Votre <b>espace</b>', 'Your <0>workspace</0>')).toBe(false);
+ });
+
+ it('does not flag a plain translation, a comparison, or a missing source', () => {
+ expect(detect('Rechercher', 'Search')).toBe(false);
+ expect(detect('moins de < 5 minutes', 'less than < 5 minutes')).toBe(false);
+ expect(detect('<b>Recherche</b>', undefined)).toBe(false);
+ });
+
+ it('is idempotent', () => {
+ const once = fix('過去<b>4</b>時間', 'Last 4 hours');
+
+ expect(detect(once, 'Last 4 hours')).toBe(false);
+ expect(fix(once, 'Last 4 hours')).toBe(once);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts
new file mode 100644
index 00000000000..5b661c1ba70
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/corrupted-model-output.rule.ts
@@ -0,0 +1,130 @@
+import { type NormalizationRule } from '../types/normalization-rule.type';
+
+// Trailing characters the machine translator leaves behind once its response
+// envelope bleeds into the string. Only stripped when the source does not end
+// the same way, so legitimate quotes and brackets survive.
+const DEBRIS_CHARACTERS = new Set([
+ ' ',
+ '\t',
+ '\n',
+ '\r',
+ ' ',
+ '`',
+ ']',
+ "'",
+ '"',
+]);
+
+// Fragments of the translator's own JSON/markdown envelope. A salvaged prefix
+// carrying one of these was cut inside the envelope rather than after it.
+const ENVELOPE_REGEX = /pluralForm|```|\\",\\"|\[/;
+
+const ICU_ARGUMENT_REGEX = /\{\s*([A-Za-z0-9_]+)\s*[,}]/g;
+const ICU_TAG_REGEX = /<\/?\d+>/g;
+
+function hasBalancedBraces(text: string): boolean {
+ let depth = 0;
+
+ for (const character of text) {
+ if (character === '{') {
+ depth += 1;
+ continue;
+ }
+
+ if (character === '}') {
+ depth -= 1;
+
+ if (depth < 0) {
+ return false;
+ }
+ }
+ }
+
+ return depth === 0;
+}
+
+// The first unmatched '}' is where the translation ends and the envelope begins.
+function cutAtUnmatchedBrace(text: string): string | undefined {
+ let depth = 0;
+
+ for (let index = 0; index < text.length; index += 1) {
+ const character = text[index];
+
+ if (character === '{') {
+ depth += 1;
+ continue;
+ }
+
+ if (character === '}') {
+ if (depth === 0) {
+ return text.slice(0, index);
+ }
+
+ depth -= 1;
+ }
+ }
+
+ return undefined;
+}
+
+function trimDebris(text: string, sourceText: string): string {
+ let end = text.length;
+
+ while (
+ end > 0 &&
+ DEBRIS_CHARACTERS.has(text[end - 1]) &&
+ !sourceText.endsWith(text[end - 1])
+ ) {
+ end -= 1;
+ }
+
+ return text.slice(0, end);
+}
+
+function icuShape(text: string): string {
+ const argumentNames = [...text.matchAll(ICU_ARGUMENT_REGEX)]
+ .map(([, name]) => name)
+ .sort();
+ const tags = [...text.matchAll(ICU_TAG_REGEX)].map(([tag]) => tag).sort();
+
+ return JSON.stringify([argumentNames, tags]);
+}
+
+function isCorrupted(text: string, sourceText?: string): boolean {
+ return (
+ sourceText !== undefined &&
+ hasBalancedBraces(sourceText) &&
+ !hasBalancedBraces(text)
+ );
+}
+
+function salvage(text: string, sourceText?: string): string {
+ const source = sourceText ?? '';
+ const cutText = cutAtUnmatchedBrace(text);
+
+ // Balanced text has nothing to cut. Otherwise an unclosed '{' is all that is
+ // left, with no clean tail to cut after, so the translation is dropped rather
+ // than guessed at.
+ if (cutText === undefined) {
+ return hasBalancedBraces(text) ? text : '';
+ }
+
+ const salvagedText = trimDebris(cutText, source);
+
+ const isSalvageable =
+ salvagedText !== '' &&
+ hasBalancedBraces(salvagedText) &&
+ (!ENVELOPE_REGEX.test(salvagedText) || ENVELOPE_REGEX.test(source)) &&
+ icuShape(salvagedText) === icuShape(source);
+
+ // An empty repair tells the runner to delete the translation rather than
+ // replace it, so the locale falls back to English until Crowdin retranslates.
+ return isSalvageable ? salvagedText : '';
+}
+
+export const CORRUPTED_MODEL_OUTPUT_RULE: NormalizationRule = {
+ name: 'corrupted-model-output',
+ needsSourceText: true,
+ detect: isCorrupted,
+ fix: salvage,
+};
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts
new file mode 100644
index 00000000000..0f1c0a5eb02
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-argument.rule.ts
@@ -0,0 +1,86 @@
+import { type NormalizationRule } from '../types/normalization-rule.type';
+
+const ARGUMENT_NAME_REGEX = /^\{\s*([A-Za-z0-9_]+)\s*[,}]/;
+
+// Arguments the message itself takes, ignoring plural and select case bodies:
+// a translation may add cases its locale needs, but never a new argument, which
+// the caller has no value for.
+function topLevelArgumentNames(text: string): Set<string> {
+ const names = new Set<string>();
+ let depth = 0;
+
+ for (let index = 0; index < text.length; index += 1) {
+ const character = text[index];
+
+ if (character === '{') {
+ if (depth === 0) {
+ const match = ARGUMENT_NAME_REGEX.exec(text.slice(index));
+
+ if (match !== null) names.add(match[1]);
+ }
+
+ depth += 1;
+ continue;
+ }
+
+ if (character === '}') depth -= 1;
+ }
+
+ return names;
+}
+
+function inventedArgumentNames(text: string, sourceText: string): string[] {
+ const sourceNames = topLevelArgumentNames(sourceText);
+
+ return [...topLevelArgumentNames(text)].filter(
+ (name) => !sourceNames.has(name),
+ );
+}
+
+function renameArgument(text: string, from: string, to: string): string {
+ return text.replace(
+ new RegExp(`\\{\\s*${from}\\s*(?=[,}])`, 'g'),
+ `{${to}`,
+ );
+}
+
+function hasInventedArgument(text: string, sourceText?: string): boolean {
+ return (
+ sourceText !== undefined && inventedArgumentNames(text, sourceText).length > 0
+ );
+}
+
+function repairInventedArgument(text: string, sourceText?: string): string {
+ const source = sourceText ?? '';
+ const sourceNames = [...topLevelArgumentNames(source)];
+
+ const repairedText = inventedArgumentNames(text, source).reduce(
+ (accumulator, inventedName) => {
+ // Only a difference in spelling can be repaired: the argument is the same
+ // one, so restoring the source's casing makes it resolve again.
+ const intendedName = sourceNames.find(
+ (name) =>
+ name.toLowerCase() === inventedName.toLowerCase() &&
+ !topLevelArgumentNames(accumulator).has(name),
+ );
+
+ return intendedName === undefined
+ ? accumulator
+ : renameArgument(accumulator, inventedName, intendedName);
+ },
+ text,
+ );
+
+ // An argument the source never had has no value to render, so an unrepaired
+ // translation is dropped rather than shipped with a dangling placeholder.
+ return inventedArgumentNames(repairedText, source).length > 0
+ ? ''
+ : repairedText;
+}
+
+export const INVENTED_ARGUMENT_RULE: NormalizationRule = {
+ name: 'invented-argument',
+ needsSourceText: true,
+ detect: hasInventedArgument,
+ fix: repairInventedArgument,
+};
diff --git a/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts
new file mode 100644
index 00000000000..f411e13e73b
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/rules/invented-markup.rule.ts
@@ -0,0 +1,31 @@
+import { type NormalizationRule } from '../types/normalization-rule.type';
+
+// Any markup tag, including Lingui's numbered ones (<0>), so a source carrying
+// tags of its own is left entirely alone.
+const TAG_PATTERN = '<\\/?[A-Za-z0-9][^<>]*>';
+const TAG_REGEX = new RegExp(TAG_PATTERN);
+const ALL_TAGS_REGEX = new RegExp(TAG_PATTERN, 'g');
+
+// The machine translator sometimes decorates a plain string with markup of its
+// own - RTL spans, bold around numbers, highlight spans - which the reader then
+// sees as literal tags.
+function hasInventedMarkup(text: string, sourceText?: string): boolean {
+ return (
+ sourceText !== undefined &&
+ !TAG_REGEX.test(sourceText) &&
+ TAG_REGEX.test(text)
+ );
+}
+
+// Stripping everything leaves nothing to say, and the empty result drops the
+// translation rather than shipping a blank string.
+function removeInventedMarkup(text: string): string {
+ return text.replace(ALL_TAGS_REGEX, '').trim();
+}
+
+export const INVENTED_MARKUP_RULE: NormalizationRule = {
+ name: 'invented-markup',
+ needsSourceText: true,
+ detect: hasInventedMarkup,
+ fix: removeInventedMarkup,
+};
diff --git a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
index 59d45525ffd..acee21fa150 100644
--- a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
+++ b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts
@@ -1,6 +1,11 @@
export type NormalizationRule = {
name: string;
- detect: (text: string) => boolean;
- fix: (text: string) => string;
+ detect: (text: string, sourceText?: string) => boolean;
+ // Returning an empty string means the translation could not be salvaged and
+ // should be deleted rather than replaced.
+ fix: (text: string, sourceText?: string) => string;
+ // Set by rules that compare a translation against its source but select every
+ // string rather than narrowing with sourceFilter.
+ needsSourceText?: boolean;
sourceFilter?: (sourceText: string) => boolean;
};
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts
deleted file mode 100644
index e6c0a9aaf19..00000000000
--- a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts
+++ /dev/null
@@ -1,57 +0,0 @@
-import { CrowdinApiError } from '../../errors/crowdin-api.error';
-import { isIdenticalTranslationError } from '../is-identical-translation-error.util';
-
-const buildError = (body: string) => new CrowdinApiError(400, body);
-
-describe('isIdenticalTranslationError', () => {
- it('recognises the identical-translation error code', () => {
- const error = buildError(
- JSON.stringify({
- errors: [
- {
- error: {
- key: 'text',
- errors: [
- {
- code: 'identicalTranslation',
- message: 'Identical translation already saved',
- },
- ],
- },
- },
- ],
- }),
- );
-
- expect(isIdenticalTranslationError(error)).toBe(true);
- });
-
- it('does not treat another failure as identical because of its wording', () => {
- const error = buildError(
- JSON.stringify({
- errors: [
- {
- error: {
- key: 'stringId',
- errors: [
- {
- code: 'notFound',
- message: 'String is identical to a deleted one',
- },
- ],
- },
- },
- ],
- }),
- );
-
- expect(isIdenticalTranslationError(error)).toBe(false);
- });
-
- it('rejects a non-JSON body and any non-Crowdin error', () => {
- expect(isIdenticalTranslationError(buildError('<html>502</html>'))).toBe(
- false,
- );
- expect(isIdenticalTranslationError(new Error('identical'))).toBe(false);
- });
-});
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts
new file mode 100644
index 00000000000..7e697143882
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-translation-already-present-error.util.spec.ts
@@ -0,0 +1,64 @@
+import { CrowdinApiError } from '../../errors/crowdin-api.error';
+import { isTranslationAlreadyPresentError } from '../is-translation-already-present-error.util';
+
+const buildError = (body: string) => new CrowdinApiError(400, body);
+
+const buildDetailError = (detail: { code: string; message: string }) =>
+ buildError(
+ JSON.stringify({
+ errors: [{ error: { key: 'translation', errors: [detail] } }],
+ }),
+ );
+
+describe('isTranslationAlreadyPresentError', () => {
+ it('recognises the identical-translation error code', () => {
+ expect(
+ isTranslationAlreadyPresentError(
+ buildDetailError({
+ code: 'identicalTranslation',
+ message: 'Identical translation already saved',
+ }),
+ ),
+ ).toBe(true);
+ });
+
+ it('recognises the duplicate-translation validation error', () => {
+ expect(
+ isTranslationAlreadyPresentError(
+ buildDetailError({
+ code: 'validationError',
+ message: 'Duplicate translation. Please vote or approve the original.',
+ }),
+ ),
+ ).toBe(true);
+ });
+
+ it('does not swallow an unrelated validation error', () => {
+ expect(
+ isTranslationAlreadyPresentError(
+ buildDetailError({
+ code: 'validationError',
+ message: 'Translation is too long',
+ }),
+ ),
+ ).toBe(false);
+ });
+
+ it('does not treat another failure as present because of its wording', () => {
+ expect(
+ isTranslationAlreadyPresentError(
+ buildDetailError({
+ code: 'notFound',
+ message: 'String is identical to a deleted one',
+ }),
+ ),
+ ).toBe(false);
+ });
+
+ it('rejects a non-JSON body and any non-Crowdin error', () => {
+ expect(
+ isTranslationAlreadyPresentError(buildError('<html>502</html>')),
+ ).toBe(false);
+ expect(isTranslationAlreadyPresentError(new Error('identical'))).toBe(false);
+ });
+});
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts
index 2f5916ec298..b9f13b9a80a 100644
--- a/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts
+++ b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts
@@ -1,6 +1,6 @@
import { type CrowdinContext } from '../types/crowdin-context.type';
import { crowdinRequest } from './crowdin-request.util';
-import { isIdenticalTranslationError } from './is-identical-translation-error.util';
+import { isTranslationAlreadyPresentError } from './is-translation-already-present-error.util';
export async function addTranslation(
context: CrowdinContext,
@@ -20,6 +20,6 @@ export async function addTranslation(
},
);
} catch (error) {
- if (!isIdenticalTranslationError(error)) throw error;
+ if (!isTranslationAlreadyPresentError(error)) throw error;
}
}
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts
index 1361543689c..f70dfb8e41d 100644
--- a/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts
+++ b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts
@@ -15,12 +15,12 @@ export function evaluateRules({
rule.sourceFilter !== undefined &&
(sourceText === undefined || !rule.sourceFilter(sourceText));
- if (isFilteredOut || !rule.detect(accumulator.fixedText)) {
+ if (isFilteredOut || !rule.detect(accumulator.fixedText, sourceText)) {
return accumulator;
}
return {
- fixedText: rule.fix(accumulator.fixedText),
+ fixedText: rule.fix(accumulator.fixedText, sourceText),
ruleNames: [...accumulator.ruleNames, rule.name],
};
},
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts
deleted file mode 100644
index 7d2a419c8f7..00000000000
--- a/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts
+++ /dev/null
@@ -1,23 +0,0 @@
-import { CrowdinApiError } from '../errors/crowdin-api.error';
-
-const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation';
-
-export function isIdenticalTranslationError(error: unknown): boolean {
- if (!(error instanceof CrowdinApiError)) return false;
-
- type ErrorBody = {
- errors?: Array<{ error?: { errors?: Array<{ code?: string }> } }>;
- };
-
- try {
- const body = JSON.parse(error.body) as ErrorBody;
-
- return (body.errors ?? []).some((entry) =>
- (entry.error?.errors ?? []).some(
- (detail) => detail.code === IDENTICAL_TRANSLATION_CODE,
- ),
- );
- } catch {
- return false;
- }
-}
diff --git a/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts
new file mode 100644
index 00000000000..e8a7e459fc9
--- /dev/null
+++ b/packages/twenty-utils/crowdin-normalizer/utils/is-translation-already-present-error.util.ts
@@ -0,0 +1,33 @@
+import { CrowdinApiError } from '../errors/crowdin-api.error';
+
+const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation';
+const VALIDATION_ERROR_CODE = 'validationError';
+
+// validationError also covers unrelated refusals, so the wording has to narrow
+// it down to the one that means the text we wanted is already there.
+const DUPLICATE_TRANSLATION_REGEX = /duplicate translation/i;
+
+type CrowdinErrorBody = {
+ errors?: Array<{
+ error?: { errors?: Array<{ code?: string; message?: string }> };
+ }>;
+};
+
+export function isTranslationAlreadyPresentError(error: unknown): boolean {
+ if (!(error instanceof CrowdinApiError)) return false;
+
+ try {
+ const body = JSON.parse(error.body) as CrowdinErrorBody;
+
+ return (body.errors ?? []).some((entry) =>
+ (entry.error?.errors ?? []).some(
+ (detail) =>
+ detail.code === IDENTICAL_TRANSLATION_CODE ||
+ (detail.code === VALIDATION_ERROR_CODE &&
+ DUPLICATE_TRANSLATION_REGEX.test(detail.message ?? '')),
+ ),
+ );
+ } catch {
+ return false;
+ }
+}