An open source design system that's fully customizable and agent ready
TypeScript
feat(ci): add check:i18n-catalog to the repo check suite (#4925)
nynexman44645a0cf745323a
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.mdindex b0cf37b827..107ab50e64 100644--- a/CONTRIBUTING.md+++ b/CONTRIBUTING.md@@ -607,8 +607,9 @@ pnpm build`pnpm lint:strict` runs `pnpm check:repo` first, which covers `check:sync`,`check:package-boundaries`, `check:changesets`, `check:demo-media`,-`check:executable-bits`, `check:cli-structure`, and `check:use-client` — so a-green `lint:strict` also clears the changeset and `'use client'` gates.+`check:executable-bits`, `check:cli-structure`, `check:use-client`, and+`check:i18n-catalog` — so a green `lint:strict` also clears the changeset and+`'use client'` gates.Also attach **before/after screenshots for any visual change**, and update theStorybook story for anything you added or altered.diff --git a/package.json b/package.jsonindex 3ae1670c50..f1fe8bf6fa 100644--- a/package.json+++ b/package.json@@ -11,12 +11,13 @@"test:watch": "vitest","test:coverage": "vitest --coverage","check:package-boundaries": "node scripts/check-package-boundaries.js",- "check:repo": "pnpm check:sync && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts",+ "check:repo": "pnpm check:sync && pnpm check:package-boundaries && pnpm check:changesets && pnpm check:demo-media && pnpm check:executable-bits && pnpm check:cli-structure && pnpm check:use-client && pnpm check:portable-scripts && pnpm check:i18n-catalog","check:changesets": "node scripts/check-changesets.mjs","check:portable-scripts": "node scripts/check-portable-scripts.mjs","check:cli-structure": "node scripts/check-cli-structure.mjs","check:executable-bits": "node scripts/check-executable-bits.mjs","check:demo-media": "node scripts/check-demo-media.mjs",+ "check:i18n-catalog": "node scripts/check-i18n-catalog.mjs","check:sync": "node scripts/check-sync.js","check:use-client": "node scripts/check-use-client.mjs","lint": "pnpm check:repo && eslint . --cache",diff --git a/scripts/check-i18n-catalog.mjs b/scripts/check-i18n-catalog.mjsnew file mode 100644index 0000000000..58d11d3434--- /dev/null+++ b/scripts/check-i18n-catalog.mjs@@ -0,0 +1,389 @@+#!/usr/bin/env node+// Copyright (c) Meta Platforms, Inc. and affiliates.+/**+ * CI gate for the i18n catalog — `node scripts/check-i18n-catalog.mjs`.+ *+ * `resolve.ts` falls through to rendering the raw key string when a key is+ * missing from every catalog, guarded only by a dev-time `warnOnce`. That is a+ * browser warning, not a gate, so a typo or a stale key ships. This asserts:+ *+ * 1. Source ⊆ catalog. Every `@astryx.*` key referenced in packages/core/src+ * and packages/lab/src exists in en.json.+ * 2. Locale parity. Every shipped locale file has en.json's key set — extra+ * keys fail (see LOCALE_MISSING_KEYS_ARE_FATAL for the missing direction).+ * 3. Every en.json entry carries a non-empty `description` — the only+ * context a Crowdin translator gets.+ *+ * Deliberately NOT checked: catalog keys with no call site. A shipped key is+ * public surface (consumers override any key through `overrides`), so an+ * "unused" key is legitimate and pruning one is a breaking change. Do not add+ * that check.+ *+ * Key references come from the TypeScript AST, reusing the call-site+ * definition in internal/eslint-plugin-astryx/i18n-key-format.js — a `t()` /+ * `translator()` / `translate()` call, or an `i18nKey:` property — so this+ * gate and that lint rule cannot disagree about what a key reference is. A+ * reference whose key is not statically knowable is reported as unverifiable+ * rather than passed or failed silently.+ */++import fs from 'node:fs';+import path from 'node:path';+import {fileURLToPath} from 'node:url';+import {createRequire} from 'node:module';++const require = createRequire(import.meta.url);+const ts = require('typescript');++const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');+const SRC_DIRS = ['packages/core/src', 'packages/lab/src'];+const LOCALES_DIR = 'packages/core/locales';++/**+ * Whether a locale missing a key that en.json has is an error.+ *+ * False, because a partial locale is the intended steady state, not a defect:+ * crowdin.yml sets `skip_untranslated_strings: true`, so Crowdin strips+ * untranslated keys out of every download, and `resolve.ts` falls back to en+ * for them without warning. Making this fatal would fail `main` today (fr-FR+ * carries 3 of 251 keys) and would red every translations PR. The other+ * direction — a key a locale has and en does not — is always an error: en is+ * the source of truth, so an extra key is stale and will never be read.+ */+const LOCALE_MISSING_KEYS_ARE_FATAL = false;++/** en.json is the source; pseudo.json is generated and gitignored. Same+ * exclusions the crowdin-download workflow applies to this directory. */+const NOT_A_SHIPPED_LOCALE = new Set(['en.json', 'pseudo.json']);++/** Translator call targets, per the i18n-key-format lint rule. */+const TRANSLATOR_CALL_NAMES = new Set(['t', 'translator', 'translate']);++const KEY_PREFIX = '@astryx.';++/** Mirrors scripts/check-use-client.mjs — source files only, no tests,+ * stories, docs, or perf fixtures (those legitimately name absent keys). */+function walk(dir) {+ const results = [];+ for (const entry of fs.readdirSync(dir, {withFileTypes: true})) {+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) {+ results.push(...walk(full));+ } else if (+ /\.[jt]sx?$/.test(entry.name) &&+ !entry.name.includes('.test.') &&+ !entry.name.includes('.stories.') &&+ !entry.name.includes('.doc.') &&+ !entry.name.includes('.perf.') &&+ !entry.name.endsWith('.d.ts')+ ) {+ results.push(full);+ }+ }+ return results;+}++function unwrap(node) {+ while (+ node &&+ (ts.isParenthesizedExpression(node) ||+ ts.isAsExpression(node) ||+ ts.isSatisfiesExpression(node) ||+ ts.isTypeAssertionExpression(node) ||+ ts.isNonNullExpression(node))+ ) {+ node = node.expression;+ }+ return node;+}++function literalText(node) {+ const n = unwrap(node);+ if (n && (ts.isStringLiteral(n) || ts.isNoSubstitutionTemplateLiteral(n))) {+ return n.text;+ }+ return null;+}++/**+ * Module-level `const NAME = '<key>'` and `const NAME = {a: '<key>'}`, so a+ * call site that reads one is still checkable. Every value of an object is+ * collected: `MAP[expr]` can select any of them.+ */+function collectConstants(sourceFile) {+ const consts = new Map();+ for (const statement of sourceFile.statements) {+ if (!ts.isVariableStatement(statement)) continue;+ for (const decl of statement.declarationList.declarations) {+ if (!ts.isIdentifier(decl.name) || !decl.initializer) continue;+ const init = unwrap(decl.initializer);+ const direct = literalText(init);+ if (direct !== null) {+ consts.set(decl.name.text, [direct]);+ } else if (ts.isObjectLiteralExpression(init)) {+ const values = [];+ for (const prop of init.properties) {+ if (!ts.isPropertyAssignment(prop)) continue;+ const value = literalText(prop.initializer);+ if (value !== null) values.push(value);+ }+ if (values.length > 0) consts.set(decl.name.text, values);+ }+ }+ }+ return consts;+}++/** Resolve an expression to the key strings it can evaluate to, or null when+ * it is not statically knowable. */+function resolveKeys(node, consts) {+ const n = unwrap(node);+ if (!n) return null;++ const direct = literalText(n);+ if (direct !== null) return [direct];++ if (ts.isConditionalExpression(n)) {+ const whenTrue = resolveKeys(n.whenTrue, consts);+ const whenFalse = resolveKeys(n.whenFalse, consts);+ return whenTrue && whenFalse ? [...whenTrue, ...whenFalse] : null;+ }+ if (ts.isIdentifier(n)) {+ return consts.get(n.text) ?? null;+ }+ // MAP[expr] — any value of MAP may be selected, so check them all.+ if (ts.isElementAccessExpression(n)) {+ const object = unwrap(n.expression);+ if (object && ts.isIdentifier(object))+ return consts.get(object.text) ?? null;+ }+ return null;+}++function isTranslatorCall(node) {+ const callee = node.expression;+ if (ts.isIdentifier(callee)) return TRANSLATOR_CALL_NAMES.has(callee.text);+ if (ts.isPropertyAccessExpression(callee) && ts.isIdentifier(callee.name)) {+ return TRANSLATOR_CALL_NAMES.has(callee.name.text);+ }+ return false;+}++function propertyName(node) {+ if (ts.isIdentifier(node.name)) return node.name.text;+ if (ts.isStringLiteral(node.name)) return node.name.text;+ return null;+}++/**+ * Extract every astryx catalog-key reference from one source file.+ * Pure — takes the text, so it is unit-testable without the filesystem.+ *+ * @param {string} source file contents+ * @param {string} fileName display name (drives .tsx parsing)+ * @returns {{refs: {line: number, key: string}[],+ * unresolved: {line: number, expr: string}[]}}+ */+function extractKeyRefs(source, fileName) {+ const sourceFile = ts.createSourceFile(+ fileName,+ source,+ ts.ScriptTarget.Latest,+ true,+ fileName.endsWith('.tsx') || fileName.endsWith('.jsx')+ ? ts.ScriptKind.TSX+ : ts.ScriptKind.TS,+ );+ const consts = collectConstants(sourceFile);+ const refs = [];+ const unresolved = [];++ const lineOf = node =>+ sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile)).line ++ 1;++ const record = (expression, node) => {+ const keys = resolveKeys(expression, consts);+ if (keys === null) {+ unresolved.push({+ line: lineOf(node),+ expr: expression.getText(sourceFile).replace(/\s+/g, ' ').slice(0, 80),+ });+ return;+ }+ // Non-astryx namespaces belong to their owners, per the lint rule.+ for (const key of keys) {+ if (key.startsWith(KEY_PREFIX)) refs.push({line: lineOf(node), key});+ }+ };++ const visit = node => {+ if (ts.isCallExpression(node) && isTranslatorCall(node)) {+ const arg = node.arguments[0];+ if (arg) record(arg, arg);+ }+ if (ts.isPropertyAssignment(node) && propertyName(node) === 'i18nKey') {+ record(node.initializer, node.initializer);+ }+ ts.forEachChild(node, visit);+ };+ visit(sourceFile);++ return {refs, unresolved};+}++/**+ * Validate the shape of the source catalog: every entry needs a usable+ * `defaultMessage` and a non-empty `description`.+ *+ * @param {object} catalog parsed en.json+ * @returns {string[]} human-readable problems (empty = valid)+ */+function validateSourceCatalog(catalog) {+ const problems = [];+ for (const [key, entry] of Object.entries(catalog)) {+ if (entry === null || typeof entry !== 'object' || Array.isArray(entry)) {+ problems.push(`${key}: entry is not an object`);+ continue;+ }+ if (+ typeof entry.defaultMessage !== 'string' ||+ entry.defaultMessage === ''+ ) {+ problems.push(`${key}: missing or empty "defaultMessage"`);+ }+ if (+ typeof entry.description !== 'string' ||+ entry.description.trim() === ''+ ) {+ problems.push(`${key}: missing or empty "description"`);+ }+ }+ return problems;+}++/**+ * Compare one locale's key set against en's.+ *+ * @returns {{missing: string[], extra: string[]}}+ */+function compareLocale(enKeys, localeKeys) {+ const locale = new Set(localeKeys);+ return {+ missing: [...enKeys].filter(k => !locale.has(k)),+ extra: [...locale].filter(k => !enKeys.has(k)),+ };+}++function readJson(file) {+ return JSON.parse(fs.readFileSync(file, 'utf8'));+}++function main() {+ const errors = [];+ const notes = [];++ // 1. Source ⊆ catalog+ const catalog = readJson(path.join(ROOT, LOCALES_DIR, 'en.json'));+ const enKeys = new Set(Object.keys(catalog));+ const missingRefs = [];+ const unverifiable = [];+ let refCount = 0;+ const seen = new Set();++ for (const dir of SRC_DIRS) {+ const abs = path.join(ROOT, dir);+ if (!fs.existsSync(abs)) continue;+ for (const file of walk(abs)) {+ const rel = path.relative(ROOT, file);+ const {refs, unresolved} = extractKeyRefs(+ fs.readFileSync(file, 'utf8'),+ file,+ );+ refCount += refs.length;+ for (const {line, key} of refs) {+ seen.add(key);+ if (!enKeys.has(key)) missingRefs.push(`${rel}:${line} ${key}`);+ }+ for (const {line, expr} of unresolved) {+ unverifiable.push(`${rel}:${line} ${expr}`);+ }+ }+ }++ if (missingRefs.length > 0) {+ errors.push({+ title: `${missingRefs.length} key reference(s) not in en.json`,+ lines: missingRefs,+ hint: 'Add the key to packages/core/locales/en.json, or fix the typo at the call site.',+ });+ }++ // 2. Locale parity+ const localeFiles = fs+ .readdirSync(path.join(ROOT, LOCALES_DIR))+ .filter(f => f.endsWith('.json') && !NOT_A_SHIPPED_LOCALE.has(f))+ .sort();++ for (const file of localeFiles) {+ const localeCatalog = readJson(path.join(ROOT, LOCALES_DIR, file));+ const {missing, extra} = compareLocale(enKeys, Object.keys(localeCatalog));+ if (extra.length > 0) {+ errors.push({+ title: `${file}: ${extra.length} key(s) not in en.json`,+ lines: extra,+ hint: 'en.json is the source of truth — these are stale and will never be read. Remove them, or restore the key in en.json.',+ });+ }+ if (missing.length > 0) {+ const summary = `${file}: ${missing.length}/${enKeys.size} key(s) untranslated (falls back to en)`;+ if (LOCALE_MISSING_KEYS_ARE_FATAL) {+ errors.push({title: summary, lines: missing, hint: ''});+ } else {+ notes.push(summary);+ }+ }+ }++ // 3. Translator context+ const catalogProblems = validateSourceCatalog(catalog);+ if (catalogProblems.length > 0) {+ errors.push({+ title: `${catalogProblems.length} en.json entry problem(s)`,+ lines: catalogProblems,+ hint: 'A description is the only context a Crowdin translator gets.',+ });+ }++ if (unverifiable.length > 0) {+ notes.push(+ `${unverifiable.length} key reference(s) unverifiable (key not statically knowable):\n` ++ unverifiable.map(l => ` ${l}`).join('\n'),+ );+ }++ if (errors.length > 0) {+ const count = errors.reduce((n, e) => n + e.lines.length, 0);+ console.error(`\n✗ check:i18n-catalog found ${count} problem(s):\n`);+ for (const {title, lines, hint} of errors) {+ console.error(` ${title}:`);+ for (const line of lines) console.error(` ${line}`);+ if (hint) console.error(` → ${hint}`);+ console.error('');+ }+ process.exit(1);+ }++ console.log(+ `✓ check:i18n-catalog — ${seen.size} key(s) over ${refCount} reference(s) resolve; ` ++ `${enKeys.size} en entries described; ${localeFiles.length} locale(s) consistent`,+ );+ for (const note of notes) console.log(` · ${note}`);+}++// Run as a script, but stay importable for unit tests.+if (process.argv[1] === fileURLToPath(import.meta.url)) {+ main();+}++export {extractKeyRefs, validateSourceCatalog, compareLocale};diff --git a/scripts/check-i18n-catalog.test.mjs b/scripts/check-i18n-catalog.test.mjsnew file mode 100644index 0000000000..3f69030ae6--- /dev/null+++ b/scripts/check-i18n-catalog.test.mjs@@ -0,0 +1,193 @@+// Copyright (c) Meta Platforms, Inc. and affiliates.++/**+ * @file check-i18n-catalog.test.mjs+ * Unit tests for the i18n catalog gate. The behaviour worth pinning is what+ * counts as a key reference: the AST walk has to see the keys a regex would+ * miss (a key behind a module constant) and ignore the ones a regex would+ * invent (a key in a comment or an unrelated string), and it has to declare a+ * key it cannot resolve rather than pass it.+ */++import {describe, it, expect} from 'vitest';+import {+ extractKeyRefs,+ validateSourceCatalog,+ compareLocale,+} from './check-i18n-catalog.mjs';++const keys = source => extractKeyRefs(source, 'Test.tsx').refs.map(r => r.key);+const unresolved = source =>+ extractKeyRefs(source, 'Test.tsx').unresolved.map(u => u.expr);++describe('extractKeyRefs — call sites it must see', () => {+ it('finds a literal t() argument', () => {+ expect(keys(`const s = t('@astryx.pagination.next');`)).toEqual([+ '@astryx.pagination.next',+ ]);+ });++ it('finds translator(), translate(), and a member call', () => {+ expect(+ keys(+ `translator('@astryx.a.one');\n` ++ `translate('@astryx.a.two');\n` ++ `intl.t('@astryx.a.three');`,+ ),+ ).toEqual(['@astryx.a.one', '@astryx.a.two', '@astryx.a.three']);+ });++ it('finds an i18nKey property', () => {+ expect(keys(`const o = {key: 'is', i18nKey: '@astryx.op.is'};`)).toEqual([+ '@astryx.op.is',+ ]);+ });++ it('finds a no-substitution template literal', () => {+ expect(keys('t(`@astryx.pagination.next`);')).toEqual([+ '@astryx.pagination.next',+ ]);+ });++ it('reports the line of each reference', () => {+ const {refs} = extractKeyRefs(+ `const a = 1;\nconst b = t('@astryx.a.one');\n`,+ 'Test.tsx',+ );+ expect(refs).toEqual([{line: 2, key: '@astryx.a.one'}]);+ });++ it('parses TSX and type syntax', () => {+ const source =+ `const label = t('@astryx.a.one') as string;\n` ++ `const node = <button aria-label={t('@astryx.a.two')} />;`;+ expect(keys(source)).toEqual(['@astryx.a.one', '@astryx.a.two']);+ });+});++describe('extractKeyRefs — indirection it can resolve', () => {+ it('resolves a module-level string constant', () => {+ expect(+ keys(`const K = '@astryx.dropdownMenu.label' as const;\nt(K);`),+ ).toEqual(['@astryx.dropdownMenu.label']);+ });++ it('resolves every value of a constant map, since any may be selected', () => {+ expect(+ keys(+ `const MAP = {warning: '@astryx.input.warning', error: '@astryx.input.error'};\n` ++ `t(MAP[status.type]);`,+ ),+ ).toEqual(['@astryx.input.warning', '@astryx.input.error']);+ });++ it('resolves both branches of a conditional', () => {+ expect(+ keys(+ `t(copied ? '@astryx.timestamp.copied' : '@astryx.timestamp.copy');`,+ ),+ ).toEqual(['@astryx.timestamp.copied', '@astryx.timestamp.copy']);+ });+});++describe('extractKeyRefs — what it must not invent', () => {+ it('ignores keys in comments', () => {+ expect(+ keys(+ `// t('@astryx.not.real')\n/* @astryx.also.not.real */\nconst x = 1;`,+ ),+ ).toEqual([]);+ });++ it('ignores a string that is not a translator argument', () => {+ expect(+ keys(`const doc = 'call t("@astryx.example.key") to translate';`),+ ).toEqual([]);+ });++ it('ignores non-astryx namespaces, which belong to their owners', () => {+ expect(keys(`t('@myapp.thing.label');`)).toEqual([]);+ });+});++describe('extractKeyRefs — what it must declare unverifiable', () => {+ it('reports a key read off an object field', () => {+ expect(unresolved(`t(operator.i18nKey);`)).toEqual(['operator.i18nKey']);+ });++ it('reports a built key rather than guessing at it', () => {+ expect(unresolved('t(`@astryx.status.${kind}`);')).toEqual([+ '`@astryx.status.${kind}`',+ ]);+ });++ it('reports an unknown identifier, and does not count it as a reference', () => {+ const {refs, unresolved: u} = extractKeyRefs(+ `t(keyFromProps);`,+ 'Test.tsx',+ );+ expect(refs).toEqual([]);+ expect(u).toHaveLength(1);+ });++ it('reports a conditional whose branches are not all resolvable', () => {+ expect(unresolved(`t(x ? '@astryx.a.one' : dynamicKey);`)).toHaveLength(1);+ });+});++describe('validateSourceCatalog', () => {+ const entry = (over = {}) => ({+ defaultMessage: 'Next',+ description: 'Aria label for the next-page button.',+ ...over,+ });++ it('accepts a well-formed catalog', () => {+ expect(validateSourceCatalog({'@astryx.a.one': entry()})).toEqual([]);+ });++ it('rejects a missing description', () => {+ const problems = validateSourceCatalog({+ '@astryx.a.one': {defaultMessage: 'Next'},+ });+ expect(problems).toEqual(['@astryx.a.one: missing or empty "description"']);+ });++ it('rejects an empty or whitespace-only description', () => {+ for (const description of ['', ' ', '\n']) {+ expect(+ validateSourceCatalog({'@astryx.a.one': entry({description})}),+ ).toEqual(['@astryx.a.one: missing or empty "description"']);+ }+ });++ it('rejects a missing defaultMessage', () => {+ expect(+ validateSourceCatalog({'@astryx.a.one': {description: 'ctx'}}),+ ).toEqual(['@astryx.a.one: missing or empty "defaultMessage"']);+ });++ it('rejects a bare string entry', () => {+ expect(validateSourceCatalog({'@astryx.a.one': 'Next'})).toEqual([+ '@astryx.a.one: entry is not an object',+ ]);+ });+});++describe('compareLocale', () => {+ const en = new Set(['@astryx.a.one', '@astryx.a.two']);++ it('reports nothing for an exact match', () => {+ expect(compareLocale(en, ['@astryx.a.one', '@astryx.a.two'])).toEqual({+ missing: [],+ extra: [],+ });+ });++ it('separates missing from extra', () => {+ expect(compareLocale(en, ['@astryx.a.one', '@astryx.a.stale'])).toEqual({+ missing: ['@astryx.a.two'],+ extra: ['@astryx.a.stale'],+ });+ });+});