The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
fix: repair escaped inline-code tags in Crowdin docs translations (#24397)
Abdul Rahman9ec11bd7aa5a
diff --git a/.github/workflows/ci-utils-test.yaml b/.github/workflows/ci-utils-test.yamlnew file mode 100644index 00000000000..a662faa33fe--- /dev/null+++ b/.github/workflows/ci-utils-test.yaml@@ -0,0 +1,45 @@+name: CI Utils Test++permissions:+ contents: read++on:+ push:+ branches:+ - main++ pull_request:++concurrency:+ group: ${{ github.workflow }}-${{ github.ref }}+ cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}++jobs:+ changed-files-check:+ uses: ./.github/workflows/changed-files.yaml+ with:+ files: |+ packages/twenty-utils/**+ utils-test:+ needs: changed-files-check+ if: needs.changed-files-check.outputs.any_changed == 'true'+ timeout-minutes: 10+ runs-on: ubuntu-latest+ steps:+ - name: Fetch custom Github Actions and base branch history+ uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1+ with:+ fetch-depth: 10+ - name: Install dependencies+ uses: ./.github/actions/yarn-install+ - name: Run utils tests+ run: npx nx test twenty-utils+ ci-utils-test-status-check:+ if: always() && !cancelled()+ timeout-minutes: 5+ runs-on: ubuntu-latest+ needs: [changed-files-check, utils-test]+ steps:+ - name: Fail job if any needs failed+ if: contains(needs.*.result, 'failure')+ run: exit 1diff --git a/.github/workflows/docs-i18n-pull.yaml b/.github/workflows/docs-i18n-pull.yamlindex 810f636d748..00821920f65 100644--- a/.github/workflows/docs-i18n-pull.yaml+++ b/.github/workflows/docs-i18n-pull.yaml@@ -81,6 +81,12 @@ jobs:env:CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}+ - name: Normalize translations in Crowdin+ if: github.event_name != 'pull_request' && (inputs.force_pull == true || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=2 --apply --rules=escaped-inline-code-tags || echo "::warning::translation normalization failed (non-blocking)"+ env:+ CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}+# Pull docs translations from Crowdin one language at a time# This avoids build timeout issues when processing all languages at once- name: Pull translated docs from Crowdindiff --git a/.github/workflows/i18n-pull.yaml b/.github/workflows/i18n-pull.yamlindex 09bc5ea0013..1a6f47f2e63 100644--- a/.github/workflows/i18n-pull.yaml+++ b/.github/workflows/i18n-pull.yaml@@ -88,8 +88,8 @@ jobs:run: sudo chown -R runner:docker .# Fix encoding issues (escaped Unicode like \u62db -> 招) and push fixes back to Crowdin- - name: Fix translation encoding and sync to Crowdin- run: npx ts-node packages/twenty-utils/fix-crowdin-translations.ts+ - name: Normalize translations in Crowdin+ run: npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode || echo "::warning::translation normalization failed (non-blocking)"env:CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}diff --git a/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.tsnew file mode 100644index 00000000000..1a96b7f4a46--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/constants/normalization-rules.constant.ts@@ -0,0 +1,8 @@+import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../rules/escaped-inline-code-tags.rule';+import { ESCAPED_UNICODE_RULE } from '../rules/escaped-unicode.rule';+import { type NormalizationRule } from '../types/normalization-rule.type';++export const NORMALIZATION_RULES: NormalizationRule[] = [+ ESCAPED_UNICODE_RULE,+ ESCAPED_INLINE_CODE_TAGS_RULE,+];diff --git a/packages/twenty-utils/crowdin-normalizer/errors/crowdin-api.error.ts b/packages/twenty-utils/crowdin-normalizer/errors/crowdin-api.error.tsnew file mode 100644index 00000000000..4518bc02d10--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/errors/crowdin-api.error.ts@@ -0,0 +1,8 @@+export class CrowdinApiError extends Error {+ constructor(+ readonly status: number,+ readonly body: string,+ ) {+ super(`Crowdin API error: ${status} ${body}`);+ }+}diff --git a/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.tsnew file mode 100644index 00000000000..40f2a76ba06--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts@@ -0,0 +1,255 @@+import { NORMALIZATION_RULES } from './constants/normalization-rules.constant';+import { type CrowdinContext } from './types/crowdin-context.type';+import { type CrowdinTranslation } from './types/crowdin-translation.type';+import { type NormalizationRule } from './types/normalization-rule.type';+import { addTranslation } from './utils/add-translation.util';+import { deleteTranslation } from './utils/delete-translation.util';+import { evaluateRules } from './utils/evaluate-rules.util';+import { fetchLanguageTranslations } from './utils/fetch-language-translations.util';+import { fetchSourceStringsById } from './utils/fetch-source-strings-by-id.util';+import { fetchTargetLanguageIds } from './utils/fetch-target-language-ids.util';+import { getCrowdinTokenOrThrow } from './utils/get-crowdin-token-or-throw.util';+import { mapWithConcurrency } from './utils/map-with-concurrency.util';++const FETCH_CONCURRENCY = 10;+const MAX_PREVIEWED_FINDINGS = 15;++type NormalizationFinding = {+ languageId: string;+ stringId: number;+ translationId: number;+ sourceText: string;+ originalText: string;+ fixedText: string;+ ruleNames: string[];+};++function getArgumentValue(name: string): string | undefined {+ return process.argv+ .find((argument) => argument.startsWith(`--${name}=`))+ ?.split('=')[1];+}++function parseProjectIdOrThrow(): number {+ const rawProjectId = getArgumentValue('project');+ const projectId = Number(rawProjectId);++ if (!Number.isInteger(projectId) || projectId <= 0) {+ throw new Error(+ `Missing or invalid --project=${rawProjectId ?? ''}, expected a project id`,+ );+ }++ return projectId;+}++function selectRulesOrThrow(): NormalizationRule[] {+ const requestedNames = (getArgumentValue('rules') ?? '')+ .split(',')+ .map((name) => name.trim())+ .filter(Boolean);++ const availableNames = NORMALIZATION_RULES.map((rule) => rule.name);+ const unknownNames = requestedNames.filter(+ (name) => !availableNames.includes(name),+ );++ if (requestedNames.length === 0 || unknownNames.length > 0) {+ throw new Error(+ `Missing or unknown --rules=${unknownNames.join(',')}. Available: ${availableNames.join(', ')}`,+ );+ }++ return NORMALIZATION_RULES.filter((rule) =>+ requestedNames.includes(rule.name),+ );+}++function collectFindings({+ rules,+ sourceStrings,+ languageId,+ translations,+}: {+ rules: NormalizationRule[];+ sourceStrings: Map<number, string> | undefined;+ languageId: string;+ translations: CrowdinTranslation[];+}): NormalizationFinding[] {+ return translations.flatMap((translation) => {+ const sourceText = sourceStrings?.get(translation.stringId);+ const { fixedText, ruleNames } = evaluateRules({+ rules,+ sourceText,+ translationText: translation.text,+ });++ if (ruleNames.length === 0) return [];++ return [+ {+ languageId,+ stringId: translation.stringId,+ translationId: translation.translationId,+ sourceText: sourceText ?? '',+ originalText: translation.text,+ fixedText,+ ruleNames,+ },+ ];+ });+}++async function scan({+ context,+ rules,+ sourceStrings,+}: {+ context: CrowdinContext;+ rules: NormalizationRule[];+ sourceStrings: Map<number, string> | undefined;+}): Promise<NormalizationFinding[]> {+ const candidateStringIds = sourceStrings+ ? [...sourceStrings.entries()]+ .filter(([, sourceText]) =>+ rules.some((rule) => rule.sourceFilter?.(sourceText)),+ )+ .map(([stringId]) => stringId)+ : [];++ const isTargeted = rules.every((rule) => rule.sourceFilter);++ if (isTargeted) {+ console.log(+ `Strategy: targeted - ${candidateStringIds.length} source strings match a rule filter`,+ );+ } else {+ console.log('Strategy: bulk - every translation of every language');+ }++ const languageIds = await fetchTargetLanguageIds(context);++ console.log(`Languages: ${languageIds.length}`);++ const perLanguage = await mapWithConcurrency({+ items: languageIds,+ limit: FETCH_CONCURRENCY,+ handler: async (languageId) => {+ const translations = await fetchLanguageTranslations(context, {+ languageId,+ stringIds: isTargeted ? candidateStringIds : undefined,+ });++ const findings = collectFindings({+ rules,+ sourceStrings,+ languageId,+ translations,+ });++ console.log(+ ` ${languageId}: ${translations.length} translations, ${findings.length} to fix`,+ );++ return findings;+ },+ });++ return perLanguage.flat();+}++async function repairOne(+ context: CrowdinContext,+ finding: NormalizationFinding,+): Promise<boolean> {+ try {+ await addTranslation(context, {+ stringId: finding.stringId,+ languageId: finding.languageId,+ text: finding.fixedText,+ });+ await deleteTranslation(context, { translationId: finding.translationId });++ return true;+ } catch (error) {+ console.error(+ ` Failed to repair string ${finding.stringId} (${finding.languageId}): ${error}`,+ );++ return false;+ }+}++async function repair(+ context: CrowdinContext,+ findings: NormalizationFinding[],+): Promise<number> {+ const outcomes: boolean[] = [];++ for (const finding of findings) {+ outcomes.push(await repairOne(context, finding));+ }++ return outcomes.filter((isRepaired) => !isRepaired).length;+}++async function main() {+ const context: CrowdinContext = {+ token: getCrowdinTokenOrThrow(),+ projectId: parseProjectIdOrThrow(),+ };+ const rules = selectRulesOrThrow();+ const isApply = process.argv.includes('--apply');++ console.log(+ `Project ${context.projectId} - normalize translations (${isApply ? 'APPLY' : 'DRY-RUN'})`,+ );+ console.log(`Rules: ${rules.map((rule) => rule.name).join(', ')}`);++ const needsSourceStrings = rules.some((rule) => rule.sourceFilter);+ const sourceStrings = needsSourceStrings+ ? await fetchSourceStringsById(context)+ : undefined;++ const findings = await scan({ context, rules, sourceStrings });++ console.log(`\n=== Found ${findings.length} corrupted translation(s) ===`);++ for (const finding of findings.slice(0, MAX_PREVIEWED_FINDINGS)) {+ console.log(+ `\n[${finding.languageId}] string ${finding.stringId} (${finding.ruleNames.join(', ')})`,+ );+ if (finding.sourceText) console.log(` source: ${finding.sourceText}`);+ console.log(` trans: ${finding.originalText}`);+ console.log(` fixed: ${finding.fixedText}`);+ }++ if (findings.length === 0) {+ console.log('Nothing to normalize.');++ return;+ }++ if (!isApply) {+ console.log('\nDry-run only. Re-run with --apply to repair.');++ return;+ }++ console.log('\n=== Applying repairs ===');++ const failed = await repair(context, findings);++ console.log(+ `Repaired ${findings.length - failed} translation(s) in Crowdin.`,+ );++ if (failed > 0) {+ throw new Error(`${failed} translation(s) could not be normalized`);+ }+}++main().catch((error) => {+ console.error(`Error: ${error instanceof Error ? error.message : error}`);+ process.exit(1);+});diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-inline-code-tags.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-inline-code-tags.rule.spec.tsnew file mode 100644index 00000000000..7b448f29f44--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-inline-code-tags.rule.spec.ts@@ -0,0 +1,37 @@+import { ESCAPED_INLINE_CODE_TAGS_RULE } from '../escaped-inline-code-tags.rule';++const { detect, fix, sourceFilter } = ESCAPED_INLINE_CODE_TAGS_RULE;++describe('ESCAPED_INLINE_CODE_TAGS_RULE', () => {+ it('repairs an escaped tag inside an inline-code span', () => {+ const translationText = 'Remplacez `<path>` par votre chemin';++ expect(detect(translationText)).toBe(true);+ expect(fix(translationText)).toBe('Remplacez `<path>` par votre chemin');+ });++ it('repairs numeric and hexadecimal entities', () => {+ expect(fix('Utilisez `<img>` ici')).toBe('Utilisez `<img>` ici');+ });++ it('leaves escaped angle brackets outside inline code untouched', () => {+ expect(fix('Entourez-le de `<Trans>` et gardez <3 en prose')).toBe(+ 'Entourez-le de `<Trans>` et gardez <3 en prose',+ );+ });++ it('does not flag a translation whose inline code is already correct', () => {+ expect(detect('Remplacez `<path>` par votre chemin')).toBe(false);+ });++ it('only selects sources that carry a tag inside inline code', () => {+ expect(sourceFilter?.('Replace `<path>` with your path')).toBe(true);+ expect(sourceFilter?.('Escape a literal < in prose')).toBe(false);+ });++ it('is idempotent', () => {+ const once = fix('Remplacez `<path>` par `<autre>`');++ expect(fix(once)).toBe(once);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.tsnew file mode 100644index 00000000000..876799b5161--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/__tests__/escaped-unicode.rule.spec.ts@@ -0,0 +1,31 @@+import { ESCAPED_UNICODE_RULE } from '../escaped-unicode.rule';++const { detect, fix } = ESCAPED_UNICODE_RULE;++describe('ESCAPED_UNICODE_RULE', () => {+ it('restores characters that leaked as literal \\uXXXX sequences', () => {+ const translationText = '\\u62db\\u8058';++ expect(detect(translationText)).toBe(true);+ expect(fix(translationText)).toBe('招聘');+ });++ it('restores surrogate pairs', () => {+ expect(fix('ok \\ud83d\\ude00')).toBe('ok 😀');+ });++ it('applies to any string, since the corruption can appear anywhere', () => {+ expect(ESCAPED_UNICODE_RULE.sourceFilter).toBeUndefined();+ expect(fix('\\u00e9')).toBe('é');+ });++ it('leaves text without escaped sequences untouched', () => {+ expect(detect('招聘')).toBe(false);+ });++ it('is idempotent', () => {+ const once = fix('\\u62db');++ expect(fix(once)).toBe(once);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.tsnew file mode 100644index 00000000000..4b76ca9c8f1--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-inline-code-tags.rule.ts@@ -0,0 +1,35 @@+import { type NormalizationRule } from '../types/normalization-rule.type';++const INLINE_CODE_SPAN_REGEX = /`[^`\n]+`/g;+const ESCAPED_TAG_REGEX = /<|>|�*60;|�*62;|�*3c;|�*3e;/i;++function inlineCodeSpans(text: string): string[] {+ return text.match(INLINE_CODE_SPAN_REGEX) ?? [];+}++function sourceHasTagInInlineCode(sourceText: string): boolean {+ return inlineCodeSpans(sourceText).some(+ (span) => span.includes('<') || span.includes('>'),+ );+}++function hasEscapedTagInInlineCode(text: string): boolean {+ return inlineCodeSpans(text).some((span) => ESCAPED_TAG_REGEX.test(span));+}++function unescapeTagsInInlineCode(text: string): string {+ return text.replace(INLINE_CODE_SPAN_REGEX, (span) =>+ span+ .replace(/</gi, '<')+ .replace(/>/gi, '>')+ .replace(/�*60;|�*3c;/gi, '<')+ .replace(/�*62;|�*3e;/gi, '>'),+ );+}++export const ESCAPED_INLINE_CODE_TAGS_RULE: NormalizationRule = {+ name: 'escaped-inline-code-tags',+ detect: hasEscapedTagInInlineCode,+ fix: unescapeTagsInInlineCode,+ sourceFilter: sourceHasTagInInlineCode,+};diff --git a/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.tsnew file mode 100644index 00000000000..5bd0a441713--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/rules/escaped-unicode.rule.ts@@ -0,0 +1,15 @@+import { type NormalizationRule } from '../types/normalization-rule.type';++const ESCAPED_UNICODE_REGEX = /\\u[0-9a-fA-F]{4}/;++function unescapeUnicode(text: string): string {+ return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex) =>+ String.fromCharCode(parseInt(hex, 16)),+ );+}++export const ESCAPED_UNICODE_RULE: NormalizationRule = {+ name: 'escaped-unicode',+ detect: (text) => ESCAPED_UNICODE_REGEX.test(text),+ fix: unescapeUnicode,+};diff --git a/packages/twenty-utils/crowdin-normalizer/types/crowdin-context.type.ts b/packages/twenty-utils/crowdin-normalizer/types/crowdin-context.type.tsnew file mode 100644index 00000000000..e74a1e85ca1--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/types/crowdin-context.type.ts@@ -0,0 +1,4 @@+export type CrowdinContext = {+ token: string;+ projectId: number;+};diff --git a/packages/twenty-utils/crowdin-normalizer/types/crowdin-translation.type.ts b/packages/twenty-utils/crowdin-normalizer/types/crowdin-translation.type.tsnew file mode 100644index 00000000000..6dc272fe995--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/types/crowdin-translation.type.ts@@ -0,0 +1,5 @@+export type CrowdinTranslation = {+ stringId: number;+ translationId: number;+ text: string;+};diff --git a/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.tsnew file mode 100644index 00000000000..59d45525ffd--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/types/normalization-rule.type.ts@@ -0,0 +1,6 @@+export type NormalizationRule = {+ name: string;+ detect: (text: string) => boolean;+ fix: (text: string) => string;+ sourceFilter?: (sourceText: string) => boolean;+};diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/evaluate-rules.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/evaluate-rules.util.spec.tsnew file mode 100644index 00000000000..3afca2e1f97--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/evaluate-rules.util.spec.ts@@ -0,0 +1,49 @@+import { NORMALIZATION_RULES } from '../../constants/normalization-rules.constant';+import { evaluateRules } from '../evaluate-rules.util';++describe('evaluateRules', () => {+ it('reports every rule that fired and applies them in sequence', () => {+ const result = evaluateRules({+ rules: NORMALIZATION_RULES,+ sourceText: 'Replace `<path>` with your path',+ translationText: '\\u62db `<path>`',+ });++ expect(result.fixedText).toBe('招 `<path>`');+ expect(result.ruleNames).toEqual([+ 'escaped-unicode',+ 'escaped-inline-code-tags',+ ]);+ });++ it('skips source-filtered rules when the source string is unknown', () => {+ const result = evaluateRules({+ rules: NORMALIZATION_RULES,+ sourceText: undefined,+ translationText: 'Remplacez `<path>` par votre chemin',+ });++ expect(result.ruleNames).toEqual([]);+ });++ it('skips source-filtered rules when the source does not match', () => {+ const result = evaluateRules({+ rules: NORMALIZATION_RULES,+ sourceText: 'Escape a literal < in prose',+ translationText: 'Échappez un < littéral dans le texte',+ });++ expect(result.ruleNames).toEqual([]);+ });++ it('returns the translation untouched when no rule fires', () => {+ const result = evaluateRules({+ rules: NORMALIZATION_RULES,+ sourceText: 'Replace `<path>` with your path',+ translationText: 'Remplacez `<path>` par votre chemin',+ });++ expect(result.fixedText).toBe('Remplacez `<path>` par votre chemin');+ expect(result.ruleNames).toEqual([]);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.tsnew file mode 100644index 00000000000..e6c0a9aaf19--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/is-identical-translation-error.util.spec.ts@@ -0,0 +1,57 @@+import { CrowdinApiError } from '../../errors/crowdin-api.error';+import { isIdenticalTranslationError } from '../is-identical-translation-error.util';++const buildError = (body: string) => new CrowdinApiError(400, body);++describe('isIdenticalTranslationError', () => {+ it('recognises the identical-translation error code', () => {+ const error = buildError(+ JSON.stringify({+ errors: [+ {+ error: {+ key: 'text',+ errors: [+ {+ code: 'identicalTranslation',+ message: 'Identical translation already saved',+ },+ ],+ },+ },+ ],+ }),+ );++ expect(isIdenticalTranslationError(error)).toBe(true);+ });++ it('does not treat another failure as identical because of its wording', () => {+ const error = buildError(+ JSON.stringify({+ errors: [+ {+ error: {+ key: 'stringId',+ errors: [+ {+ code: 'notFound',+ message: 'String is identical to a deleted one',+ },+ ],+ },+ },+ ],+ }),+ );++ expect(isIdenticalTranslationError(error)).toBe(false);+ });++ it('rejects a non-JSON body and any non-Crowdin error', () => {+ expect(isIdenticalTranslationError(buildError('<html>502</html>'))).toBe(+ false,+ );+ expect(isIdenticalTranslationError(new Error('identical'))).toBe(false);+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/utils/__tests__/map-with-concurrency.util.spec.ts b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/map-with-concurrency.util.spec.tsnew file mode 100644index 00000000000..8513b432edb--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/__tests__/map-with-concurrency.util.spec.ts@@ -0,0 +1,51 @@+import { mapWithConcurrency } from '../map-with-concurrency.util';++describe('mapWithConcurrency', () => {+ it('keeps results in input order regardless of completion order', async () => {+ const results = await mapWithConcurrency({+ items: [30, 10, 20],+ limit: 3,+ handler: (delay) =>+ new Promise<number>((resolve) =>+ setTimeout(() => resolve(delay), delay),+ ),+ });++ expect(results).toEqual([30, 10, 20]);+ });++ it('never runs more than `limit` handlers at once', async () => {+ let running = 0;+ let peak = 0;++ await mapWithConcurrency({+ items: Array.from({ length: 20 }, (_, index) => index),+ limit: 3,+ handler: async (item) => {+ running++;+ peak = Math.max(peak, running);+ await new Promise((resolve) => setTimeout(resolve, 1));+ running--;++ return item;+ },+ });++ expect(peak).toBe(3);+ });++ it('rejects a limit that would leave every item unhandled', async () => {+ await expect(+ mapWithConcurrency({ items: [1], limit: 0, handler: jest.fn() }),+ ).rejects.toThrow('at least 1');+ });++ it('returns an empty array without spawning workers', async () => {+ const handler = jest.fn();++ expect(await mapWithConcurrency({ items: [], limit: 5, handler })).toEqual(+ [],+ );+ expect(handler).not.toHaveBeenCalled();+ });+});diff --git a/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.tsnew file mode 100644index 00000000000..2f5916ec298--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/add-translation.util.ts@@ -0,0 +1,25 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { crowdinRequest } from './crowdin-request.util';+import { isIdenticalTranslationError } from './is-identical-translation-error.util';++export async function addTranslation(+ context: CrowdinContext,+ {+ stringId,+ languageId,+ text,+ }: { stringId: number; languageId: string; text: string },+): Promise<void> {+ try {+ await crowdinRequest(+ context,+ `/projects/${context.projectId}/translations`,+ {+ method: 'POST',+ body: JSON.stringify({ stringId, languageId, text }),+ },+ );+ } catch (error) {+ if (!isIdenticalTranslationError(error)) throw error;+ }+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/crowdin-request.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/crowdin-request.util.tsnew file mode 100644index 00000000000..3dd80078678--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/crowdin-request.util.ts@@ -0,0 +1,31 @@+import { CrowdinApiError } from '../errors/crowdin-api.error';+import { type CrowdinContext } from '../types/crowdin-context.type';++const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';++export async function crowdinRequest<T>(+ { token }: CrowdinContext,+ endpoint: string,+ options: RequestInit = {},+): Promise<T | undefined> {+ const response = await fetch(`${CROWDIN_BASE_URL}${endpoint}`, {+ ...options,+ headers: {+ Authorization: `Bearer ${token}`,+ 'Content-Type': 'application/json',+ ...options.headers,+ },+ });++ const body = await response.text();++ if (!response.ok) {+ if (response.status === 404) return undefined;++ throw new CrowdinApiError(response.status, body);+ }++ if (!body) return undefined;++ return JSON.parse(body) as T;+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/delete-translation.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/delete-translation.util.tsnew file mode 100644index 00000000000..7a5fd3f3d3f--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/delete-translation.util.ts@@ -0,0 +1,13 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { crowdinRequest } from './crowdin-request.util';++export async function deleteTranslation(+ context: CrowdinContext,+ { translationId }: { translationId: number },+): Promise<void> {+ await crowdinRequest(+ context,+ `/projects/${context.projectId}/translations/${translationId}`,+ { method: 'DELETE' },+ );+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.tsnew file mode 100644index 00000000000..1361543689c--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/evaluate-rules.util.ts@@ -0,0 +1,29 @@+import { type NormalizationRule } from '../types/normalization-rule.type';++export function evaluateRules({+ rules,+ sourceText,+ translationText,+}: {+ rules: NormalizationRule[];+ sourceText: string | undefined;+ translationText: string;+}): { fixedText: string; ruleNames: string[] } {+ return rules.reduce<{ fixedText: string; ruleNames: string[] }>(+ (accumulator, rule) => {+ const isFilteredOut =+ rule.sourceFilter !== undefined &&+ (sourceText === undefined || !rule.sourceFilter(sourceText));++ if (isFilteredOut || !rule.detect(accumulator.fixedText)) {+ return accumulator;+ }++ return {+ fixedText: rule.fix(accumulator.fixedText),+ ruleNames: [...accumulator.ruleNames, rule.name],+ };+ },+ { fixedText: translationText, ruleNames: [] },+ );+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/fetch-all-pages.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/fetch-all-pages.util.tsnew file mode 100644index 00000000000..76096e265ae--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/fetch-all-pages.util.ts@@ -0,0 +1,31 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { crowdinRequest } from './crowdin-request.util';++const PAGE_SIZE = 500;++export async function fetchAllPages<TItem>(+ context: CrowdinContext,+ buildEndpoint: (pagination: string) => string,+): Promise<TItem[]> {+ const items: TItem[] = [];+ let offset = 0;++ while (true) {+ type PagedResponse = { data: Array<{ data: TItem }> };++ const response = await crowdinRequest<PagedResponse>(+ context,+ buildEndpoint(`limit=${PAGE_SIZE}&offset=${offset}`),+ );++ if (!response || response.data.length === 0) break;++ items.push(...response.data.map((item) => item.data));++ if (response.data.length < PAGE_SIZE) break;++ offset += PAGE_SIZE;+ }++ return items;+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/fetch-language-translations.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/fetch-language-translations.util.tsnew file mode 100644index 00000000000..f0a81f4fda8--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/fetch-language-translations.util.ts@@ -0,0 +1,47 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { type CrowdinTranslation } from '../types/crowdin-translation.type';+import { fetchAllPages } from './fetch-all-pages.util';++const MAX_STRING_IDS_PER_REQUEST = 50;++function chunkStringIds(stringIds: number[]): number[][] {+ const batches: number[][] = [];++ for (+ let index = 0;+ index < stringIds.length;+ index += MAX_STRING_IDS_PER_REQUEST+ ) {+ batches.push(stringIds.slice(index, index + MAX_STRING_IDS_PER_REQUEST));+ }++ return batches;+}++export async function fetchLanguageTranslations(+ context: CrowdinContext,+ { languageId, stringIds }: { languageId: string; stringIds?: number[] },+): Promise<CrowdinTranslation[]> {+ const stringIdBatches = stringIds ? chunkStringIds(stringIds) : [undefined];++ const translations: Array<Partial<CrowdinTranslation>> = [];++ for (const batch of stringIdBatches) {+ const stringIdsFilter = batch ? `&stringIds=${batch.join(',')}` : '';++ translations.push(+ ...(await fetchAllPages<Partial<CrowdinTranslation>>(+ context,+ (pagination) =>+ `/projects/${context.projectId}/languages/${languageId}/translations?${pagination}${stringIdsFilter}`,+ )),+ );+ }++ return translations.filter(+ (translation): translation is CrowdinTranslation =>+ typeof translation.stringId === 'number' &&+ typeof translation.translationId === 'number' &&+ typeof translation.text === 'string',+ );+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/fetch-source-strings-by-id.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/fetch-source-strings-by-id.util.tsnew file mode 100644index 00000000000..6d25d15a5dd--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/fetch-source-strings-by-id.util.ts@@ -0,0 +1,26 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { fetchAllPages } from './fetch-all-pages.util';++export async function fetchSourceStringsById(+ context: CrowdinContext,+): Promise<Map<number, string>> {+ type SourceString = {+ id?: number;+ text?: string | Record<string, string>;+ };++ const sourceStrings = await fetchAllPages<SourceString>(+ context,+ (pagination) => `/projects/${context.projectId}/strings?${pagination}`,+ );++ return new Map(+ sourceStrings+ .filter(+ (sourceString): sourceString is { id: number; text: string } =>+ typeof sourceString.id === 'number' &&+ typeof sourceString.text === 'string',+ )+ .map((sourceString) => [sourceString.id, sourceString.text]),+ );+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/fetch-target-language-ids.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/fetch-target-language-ids.util.tsnew file mode 100644index 00000000000..f8ff9c3ec05--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/fetch-target-language-ids.util.ts@@ -0,0 +1,15 @@+import { type CrowdinContext } from '../types/crowdin-context.type';+import { crowdinRequest } from './crowdin-request.util';++export async function fetchTargetLanguageIds(+ context: CrowdinContext,+): Promise<string[]> {+ type ProjectResponse = { data: { targetLanguageIds: string[] } };++ const response = await crowdinRequest<ProjectResponse>(+ context,+ `/projects/${context.projectId}`,+ );++ return response?.data.targetLanguageIds ?? [];+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/get-crowdin-token-or-throw.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/get-crowdin-token-or-throw.util.tsnew file mode 100644index 00000000000..9105f71c2ec--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/get-crowdin-token-or-throw.util.ts@@ -0,0 +1,11 @@+export function getCrowdinTokenOrThrow(): string {+ const token = process.env.CROWDIN_PERSONAL_TOKEN;++ if (!token) {+ throw new Error(+ 'CROWDIN_PERSONAL_TOKEN environment variable not set. Get your token from: https://twenty.crowdin.com/u/settings#api-key',+ );+ }++ return token;+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.tsnew file mode 100644index 00000000000..7d2a419c8f7--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/is-identical-translation-error.util.ts@@ -0,0 +1,23 @@+import { CrowdinApiError } from '../errors/crowdin-api.error';++const IDENTICAL_TRANSLATION_CODE = 'identicalTranslation';++export function isIdenticalTranslationError(error: unknown): boolean {+ if (!(error instanceof CrowdinApiError)) return false;++ type ErrorBody = {+ errors?: Array<{ error?: { errors?: Array<{ code?: string }> } }>;+ };++ try {+ const body = JSON.parse(error.body) as ErrorBody;++ return (body.errors ?? []).some((entry) =>+ (entry.error?.errors ?? []).some(+ (detail) => detail.code === IDENTICAL_TRANSLATION_CODE,+ ),+ );+ } catch {+ return false;+ }+}diff --git a/packages/twenty-utils/crowdin-normalizer/utils/map-with-concurrency.util.ts b/packages/twenty-utils/crowdin-normalizer/utils/map-with-concurrency.util.tsnew file mode 100644index 00000000000..43c39e87e54--- /dev/null+++ b/packages/twenty-utils/crowdin-normalizer/utils/map-with-concurrency.util.ts@@ -0,0 +1,32 @@+export async function mapWithConcurrency<TItem, TResult>({+ items,+ limit,+ handler,+}: {+ items: TItem[];+ limit: number;+ handler: (item: TItem) => Promise<TResult>;+}): Promise<TResult[]> {+ if (limit < 1) {+ throw new Error(+ `mapWithConcurrency needs a limit of at least 1, got ${limit}`,+ );+ }++ const results: TResult[] = Array.from({ length: items.length });+ let cursor = 0;++ const worker = async () => {+ while (cursor < items.length) {+ const index = cursor++;++ results[index] = await handler(items[index]);+ }+ };++ await Promise.all(+ Array.from({ length: Math.min(limit, items.length) }, worker),+ );++ return results;+}diff --git a/packages/twenty-utils/fix-crowdin-translations.ts b/packages/twenty-utils/fix-crowdin-translations.tsdeleted file mode 100644index 77ed276317f..00000000000--- a/packages/twenty-utils/fix-crowdin-translations.ts+++ /dev/null@@ -1,251 +0,0 @@-/**- * Script to fix encoding issues in Crowdin translations- *- * This script:- * 1. Fetches translations from Crowdin API that have escaped Unicode sequences- * 2. Deletes the corrupted translations- * 3. Adds corrected translations (or activates existing corrected suggestions)- *- * Usage:- * CROWDIN_PERSONAL_TOKEN=xxx npx ts-node packages/twenty-utils/fix-crowdin-translations.ts- *- * The token can be obtained from: https://twenty.crowdin.com/u/settings#api-key- */--const CROWDIN_BASE_URL = 'https://twenty.api.crowdin.com/api/v2';-const CROWDIN_PROJECT_ID = 1;--type CrowdinTranslation = {- stringId: number;- translationId: number;- text: string;-};--async function getToken(): Promise<string> {- const token = process.env.CROWDIN_PERSONAL_TOKEN;-- if (!token) {- console.error(- 'Error: CROWDIN_PERSONAL_TOKEN environment variable not set',- );- console.error(- 'Get your token from: https://twenty.crowdin.com/u/settings#api-key',- );- process.exit(1);- }-- return token;-}--async function crowdinRequest<T>(- endpoint: string,- token: string,- options: RequestInit = {},-): Promise<T | null> {- const url = `${CROWDIN_BASE_URL}${endpoint}`;- const response = await fetch(url, {- ...options,- headers: {- Authorization: `Bearer ${token}`,- 'Content-Type': 'application/json',- ...options.headers,- },- });-- if (!response.ok) {- if (response.status === 404) return null;- const text = await response.text();- throw new Error(`Crowdin API error: ${response.status} ${text}`);- }-- const text = await response.text();-- if (!text) return null;-- return JSON.parse(text) as T;-}--async function getProjectLanguages(token: string): Promise<string[]> {- type ProjectResponse = {- data: {- targetLanguageIds: string[];- };- };-- const response = await crowdinRequest<ProjectResponse>(- `/projects/${CROWDIN_PROJECT_ID}`,- token,- );-- return response?.data.targetLanguageIds || [];-}--async function getTranslationsForLanguage(- token: string,- languageId: string,-): Promise<CrowdinTranslation[]> {- const translations: CrowdinTranslation[] = [];- let offset = 0;- const limit = 500;-- while (true) {- type TranslationsResponse = {- data: Array<{- data: {- stringId: number;- translationId: number;- text: string;- };- }>;- };-- const response = await crowdinRequest<TranslationsResponse>(- `/projects/${CROWDIN_PROJECT_ID}/languages/${languageId}/translations?limit=${limit}&offset=${offset}`,- token,- );-- if (!response || response.data.length === 0) break;-- for (const item of response.data) {- translations.push({- stringId: item.data.stringId,- translationId: item.data.translationId,- text: item.data.text,- });- }-- if (response.data.length < limit) break;- offset += limit;- }-- return translations;-}--async function deleteTranslation(- token: string,- translationId: number,-): Promise<boolean> {- const result = await crowdinRequest(- `/projects/${CROWDIN_PROJECT_ID}/translations/${translationId}`,- token,- { method: 'DELETE' },- );-- return result === null; // null means success (empty response)-}--async function addTranslation(- token: string,- stringId: number,- languageId: string,- text: string,-): Promise<boolean> {- const url = `${CROWDIN_BASE_URL}/projects/${CROWDIN_PROJECT_ID}/translations`;- const response = await fetch(url, {- method: 'POST',- headers: {- Authorization: `Bearer ${token}`,- 'Content-Type': 'application/json',- },- body: JSON.stringify({ stringId, languageId, text }),- });-- // Success if added OR if identical already exists (meaning correct version is now active)- if (response.ok) return true;-- const data = (await response.json()) as { errors?: Array<{ error?: { errors?: Array<{ message?: string }> } }> };- const errorMsg = data?.errors?.[0]?.error?.errors?.[0]?.message || '';-- // "Identical translation already saved" means the correct version was already a suggestion- // and is now the active translation after we deleted the corrupted one- if (errorMsg.includes('identical')) return true;-- throw new Error(`Failed to add translation: ${JSON.stringify(data)}`);-}--function hasEscapedUnicode(text: string): boolean {- // Match literal \uXXXX sequences in the text- // The text comes JSON-decoded, so \\u in the original becomes \u in the string- return /\\u[0-9a-fA-F]{4}/.test(text);-}--function fixEscapedUnicode(text: string): string {- return text.replace(/\\u([0-9a-fA-F]{4})/g, (_match, hex) => {- try {- return String.fromCharCode(parseInt(hex, 16));- } catch {- return _match;- }- });-}--async function main() {- const token = await getToken();-- console.log('Fetching project languages...');- const languages = await getProjectLanguages(token);- console.log(`Languages: ${languages.length}`);-- let totalFixed = 0;-- for (const languageId of languages) {- process.stdout.write(`Checking ${languageId}...`);-- try {- const translations = await getTranslationsForLanguage(token, languageId);-- // Find translations with escaped Unicode- const toFix = translations.filter((t) => hasEscapedUnicode(t.text));-- if (toFix.length === 0) {- console.log(` ${translations.length} translations, 0 issues`);- continue;- }-- console.log(- ` ${translations.length} translations, ${toFix.length} to fix`,- );-- let fixedInLang = 0;-- // Fix each translation- for (const translation of toFix) {- const fixedText = fixEscapedUnicode(translation.text);-- try {- // Delete the corrupted translation- await deleteTranslation(token, translation.translationId);-- // Add the corrected translation (or let existing suggestion become active)- await addTranslation(- token,- translation.stringId,- languageId,- fixedText,- );-- fixedInLang++;- totalFixed++;- process.stdout.write('.');- } catch (error) {- console.log(- `\n Failed to fix string ${translation.stringId}: ${error}`,- );- }- }-- if (fixedInLang > 0) {- console.log(` ✓ Fixed ${fixedInLang}`);- }- } catch (error) {- console.log(` error: ${error}`);- }- }-- console.log(`\nDone! Fixed ${totalFixed} translations in Crowdin.`);-}--main().catch((error) => {- console.error('Error:', error);- process.exit(1);-});diff --git a/packages/twenty-utils/jest.config.mjs b/packages/twenty-utils/jest.config.mjsnew file mode 100644index 00000000000..b6f8f01e44e--- /dev/null+++ b/packages/twenty-utils/jest.config.mjs@@ -0,0 +1,16 @@+const jestConfig = {+ displayName: 'twenty-utils',+ preset: '../../jest.preset.js',+ testEnvironment: 'node',+ testEnvironmentOptions: {},+ transform: {+ '^.+\\.[tj]s$': [+ '@swc/jest',+ { jsc: { parser: { syntax: 'typescript' } } },+ ],+ },+ moduleFileExtensions: ['ts', 'js'],+ coverageDirectory: './coverage',+};++export default jestConfig;diff --git a/packages/twenty-utils/project.json b/packages/twenty-utils/project.jsonnew file mode 100644index 00000000000..55d90fa0f03--- /dev/null+++ b/packages/twenty-utils/project.json@@ -0,0 +1,9 @@+{+ "name": "twenty-utils",+ "$schema": "../../node_modules/nx/schemas/project-schema.json",+ "sourceRoot": "packages/twenty-utils",+ "projectType": "application",+ "targets": {+ "test": {}+ }+}diff --git a/packages/twenty-utils/translation-qa-report.ts b/packages/twenty-utils/translation-qa-report.tsindex b5cdf67eefb..260ed043f44 100644--- a/packages/twenty-utils/translation-qa-report.ts+++ b/packages/twenty-utils/translation-qa-report.ts@@ -196,7 +196,7 @@ These are the most critical - placeholders must match exactly:*To auto-fix encoding issues, run:*\`\`\`-CROWDIN_PERSONAL_TOKEN=xxx npx ts-node packages/twenty-utils/fix-crowdin-translations.ts+CROWDIN_PERSONAL_TOKEN=xxx npx tsx packages/twenty-utils/crowdin-normalizer/normalize-crowdin-translations.ts --project=1 --apply --rules=escaped-unicode\`\`\``;