Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
feat(i18n): add hash-based incremental docs translator
diegosouzapwafeee02e4ad3
diff --git a/scripts/i18n/run-translation.mjs b/scripts/i18n/run-translation.mjsnew file mode 100755index 0000000000..5001716093--- /dev/null+++ b/scripts/i18n/run-translation.mjs@@ -0,0 +1,598 @@+#!/usr/bin/env node+/**+ * OmniRoute — Docs translation pipeline (hash-based, incremental).+ *+ * Source of truth: `config/i18n.json` (locale list) and the original English+ * markdown files at the repo root (`CLAUDE.md`, `GEMINI.md`, `README.md`, …)+ * plus `docs/*.md`.+ *+ * Targets land in `docs/i18n/<locale>/...` mirroring the source layout, with a+ * header (top H1 + language bar) and an `---` separator before the translated+ * body. This is the same shape the existing `check-docs-sync.mjs` already+ * understands.+ *+ * State: `.i18n-state.json` stores a SHA-256 hash for every source file and+ * for every produced target. Re-runs only retranslate files whose source hash+ * changed or whose target file is missing.+ *+ * Usage (driven by npm scripts in package.json):+ * npm run i18n:run+ * npm run i18n:run -- --locale=pt-BR+ * npm run i18n:run -- --files=CLAUDE.md,docs/ARCHITECTURE.md+ * npm run i18n:run -- --force+ * npm run i18n:run:dry+ *+ * Backend (configured via env, never committed):+ * OMNIROUTE_TRANSLATION_API_URL e.g. https://cloud.omniroute.dev/v1+ * OMNIROUTE_TRANSLATION_API_KEY bearer token (kept out of logs)+ * OMNIROUTE_TRANSLATION_MODEL e.g. cx/gpt-5.4-mini+ * OMNIROUTE_TRANSLATION_TIMEOUT_MS optional, default 60000+ * OMNIROUTE_TRANSLATION_CONCURRENCY optional, default 4+ */++import { promises as fs, existsSync } from "node:fs";+import path from "node:path";+import crypto from "node:crypto";+import process from "node:process";+import { fileURLToPath, pathToFileURL } from "node:url";++// Prettier is loaded lazily on first use so the script still runs (with a+// warning) in environments where node_modules has not been installed. The+// formatter is applied to every translated file before its hash is recorded,+// so a subsequent lint-staged Prettier pass cannot mutate the file content+// out from under `.i18n-state.json`.+let prettierMod = null;+async function getPrettier() {+ if (prettierMod !== null) return prettierMod;+ try {+ prettierMod = await import("prettier");+ } catch {+ prettierMod = false;+ }+ return prettierMod;+}++async function formatMarkdown(content, fileName) {+ const p = await getPrettier();+ if (!p) return content;+ try {+ return await p.format(content, { parser: "markdown", filepath: fileName });+ } catch (err) {+ logWarn(`prettier could not format ${fileName}: ${err.message}`);+ return content;+ }+}++const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));+const ROOT = path.resolve(SCRIPT_DIR, "..", "..");+const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");+const STATE_PATH = path.join(ROOT, ".i18n-state.json");+const DOCS_I18N_DIR = path.join(ROOT, "docs", "i18n");+const DOCS_DIR = path.join(ROOT, "docs");++// ----- Source set ----------------------------------------------------------+//+// Root-level markdown files that should be translated as `docs/i18n/<loc>/<name>`.+// Strict-mirror files (`llm.txt`, `CHANGELOG.md`) are intentionally NOT in this+// list — they are handled by `scripts/check-docs-sync.mjs` rules and are kept+// in sync by other tooling. Adding them here would conflict with that script.+const ROOT_DOC_SOURCES = [+ "CLAUDE.md",+ "GEMINI.md",+ "AGENTS.md",+ "CONTRIBUTING.md",+ "SECURITY.md",+ "CODE_OF_CONDUCT.md",+ "README.md",+];++// File names inside `docs/` that should NOT be translated. Anything else with+// a `.md` extension at the top of `docs/` is treated as a source.+const DOCS_EXCLUDED_NAMES = new Set([+ "I18N.md", // Translator tooling docs — kept English-only for operators.+ "README.md", // Section index files — auto-generated, not prose translation targets.+]);++// Sub-trees we never recurse into when collecting sources.+const DOCS_EXCLUDED_SUBDIRS = new Set([+ "i18n",+ "screenshots",+ "superpowers",+ "diagrams",+ "reports",+]);++// ----- Helpers -------------------------------------------------------------++function logInfo(...parts) {+ console.log("[i18n-run]", ...parts);+}++function logWarn(...parts) {+ console.warn("[i18n-run] WARN", ...parts);+}++function logError(...parts) {+ console.error("[i18n-run] ERROR", ...parts);+}++function sha256(buffer) {+ return crypto.createHash("sha256").update(buffer).digest("hex");+}++function parseArgs(argv) {+ const opts = {+ locales: null,+ files: null,+ force: false,+ dryRun: false,+ concurrency: null,+ };+ for (const arg of argv.slice(2)) {+ if (arg === "--force") opts.force = true;+ else if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;+ else if (arg.startsWith("--locale="))+ opts.locales = arg+ .slice(9)+ .split(",")+ .map((s) => s.trim())+ .filter(Boolean);+ else if (arg.startsWith("--locales="))+ opts.locales = arg+ .slice(10)+ .split(",")+ .map((s) => s.trim())+ .filter(Boolean);+ else if (arg.startsWith("--files="))+ opts.files = arg+ .slice(8)+ .split(",")+ .map((s) => s.trim())+ .filter(Boolean);+ else if (arg.startsWith("--concurrency=")) opts.concurrency = Number(arg.slice(14));+ else if (arg === "--help" || arg === "-h") {+ console.log(+ [+ "Usage: node scripts/i18n/run-translation.mjs [options]",+ "",+ " --locale=<csv> Target locales (default: all except `en`)",+ " --files=<csv> Relative paths to translate (default: all sources)",+ " --force Retranslate even when hashes match",+ " --dry-run Report what would happen but never call the API",+ " --concurrency=<n> Parallel API requests (default: env CONCURRENCY or 4)",+ ].join("\n")+ );+ process.exit(0);+ }+ }+ return opts;+}++async function loadConfig() {+ const raw = await fs.readFile(CONFIG_PATH, "utf8");+ const cfg = JSON.parse(raw);+ if (!cfg.default || !Array.isArray(cfg.locales)) {+ throw new Error("config/i18n.json: invalid shape (need `default` and `locales[]`)");+ }+ return cfg;+}++async function loadState() {+ if (!existsSync(STATE_PATH)) return { sources: {} };+ try {+ const raw = await fs.readFile(STATE_PATH, "utf8");+ const parsed = JSON.parse(raw);+ return parsed && typeof parsed === "object" && parsed.sources ? parsed : { sources: {} };+ } catch (err) {+ logWarn(`could not parse ${path.relative(ROOT, STATE_PATH)} — starting fresh (${err.message})`);+ return { sources: {} };+ }+}++async function saveState(state) {+ const json = JSON.stringify(state, null, 2) + "\n";+ await fs.writeFile(STATE_PATH, json, "utf8");+}++async function collectDocsSources() {+ const found = [];+ for (const entry of await fs.readdir(DOCS_DIR, { withFileTypes: true })) {+ if (entry.isFile() && entry.name.endsWith(".md") && !DOCS_EXCLUDED_NAMES.has(entry.name)) {+ found.push(`docs/${entry.name}`);+ } else if (entry.isDirectory() && !DOCS_EXCLUDED_SUBDIRS.has(entry.name)) {+ // Recurse one level for organized doc groups (e.g. docs/features/*.md).+ const sub = path.join(DOCS_DIR, entry.name);+ for (const child of await fs.readdir(sub, { withFileTypes: true })) {+ if (+ child.isFile() &&+ child.name.endsWith(".md") &&+ !DOCS_EXCLUDED_NAMES.has(child.name) &&+ child.name.toLowerCase() !== "readme.md"+ ) {+ found.push(`docs/${entry.name}/${child.name}`);+ }+ }+ }+ }+ return found;+}++async function collectAllSources() {+ const rootSources = [];+ for (const name of ROOT_DOC_SOURCES) {+ const abs = path.join(ROOT, name);+ if (existsSync(abs)) rootSources.push(name);+ }+ const docsSources = await collectDocsSources();+ return [...rootSources, ...docsSources].sort();+}++function targetPathFor(relSource, locale) {+ // Root MDs (`CLAUDE.md`, …) → `docs/i18n/<loc>/CLAUDE.md`+ if (!relSource.includes("/")) {+ return path.join(DOCS_I18N_DIR, locale, relSource);+ }+ // `docs/X.md` → `docs/i18n/<loc>/docs/X.md`+ // `docs/features/Y.md` → `docs/i18n/<loc>/docs/features/Y.md`+ return path.join(DOCS_I18N_DIR, locale, relSource);+}++function relativeBackToRoot(targetAbsPath) {+ // From the target file's directory back to the repo root, used to build the+ // "🇺🇸 English" link in the language bar.+ const targetDir = path.dirname(targetAbsPath);+ const rel = path.relative(targetDir, ROOT);+ return rel === "" ? "." : rel;+}++function buildLanguageBar(relSource, locale, config) {+ const targetAbs = targetPathFor(relSource, locale);+ const targetDir = path.dirname(targetAbs);+ const rootRel = relativeBackToRoot(targetAbs);++ const parts = [];+ // English link → source file relative to target dir.+ const enRel = path.relative(targetDir, path.join(ROOT, relSource));+ parts.push(`🇺🇸 [English](${enRel.split(path.sep).join("/")})`);++ for (const entry of config.locales) {+ if (entry.code === "en" || entry.code === locale) continue;+ const peerAbs = targetPathFor(relSource, entry.code);+ const peerRel = path.relative(targetDir, peerAbs).split(path.sep).join("/");+ parts.push(`${entry.flag} [${entry.code}](${peerRel})`);+ }++ return `🌐 **Languages:** ${parts.join(" · ")}`;+ // Quiet the unused linter warning for rootRel — kept here for future expansion.+ void rootRel;+}++function extractTopHeading(markdown) {+ const m = markdown.match(/^# (.+)\r?\n/);+ return m ? m[1].trim() : null;+}++function stripTopHeading(markdown) {+ return markdown.replace(/^# .+\r?\n+/, "");+}++// ----- Translator backend --------------------------------------------------++function requireEnv(name) {+ const v = process.env[name];+ if (!v || !v.trim()) {+ throw new Error(+ `Missing required env var: ${name}. Set it in .env (see docs/guides/I18N.md → "Translation pipeline").`+ );+ }+ return v.trim();+}++function backendConfig() {+ const apiUrl = requireEnv("OMNIROUTE_TRANSLATION_API_URL").replace(/\/$/, "");+ const apiKey = requireEnv("OMNIROUTE_TRANSLATION_API_KEY");+ const model = requireEnv("OMNIROUTE_TRANSLATION_MODEL");+ const timeoutMs = Number(process.env.OMNIROUTE_TRANSLATION_TIMEOUT_MS || 60000);+ return { apiUrl, apiKey, model, timeoutMs };+}++async function callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry = 0) {+ const ctrl = new AbortController();+ const timer = setTimeout(() => ctrl.abort(), timeoutMs);+ try {+ const res = await fetch(`${apiUrl}/chat/completions`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ Authorization: `Bearer ${apiKey}`,+ },+ body: JSON.stringify({+ model,+ messages,+ temperature: 0.15,+ stream: false,+ }),+ signal: ctrl.signal,+ });+ if (!res.ok) {+ const text = await res.text().catch(() => "");+ const transient = res.status === 408 || res.status === 429 || res.status >= 500;+ if (transient && retry < 1) {+ const wait = 1500 + retry * 1500;+ logWarn(`upstream ${res.status} — retrying after ${wait}ms`);+ await new Promise((r) => setTimeout(r, wait));+ return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);+ }+ throw new Error(`upstream ${res.status}: ${text.slice(0, 200)}`);+ }+ const json = await res.json();+ const content = json?.choices?.[0]?.message?.content;+ if (typeof content !== "string" || !content) {+ throw new Error("upstream returned empty content");+ }+ return content;+ } catch (err) {+ if (err?.name === "AbortError") {+ if (retry < 1) {+ logWarn(`timeout after ${timeoutMs}ms — retrying`);+ return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);+ }+ throw new Error(`timeout after ${timeoutMs}ms`);+ }+ if (+ retry < 1 &&+ err instanceof TypeError &&+ /fetch failed|ECONN|ENOTFOUND|network/i.test(String(err.cause ?? err.message))+ ) {+ logWarn(`network error: ${err.message} — retrying`);+ await new Promise((r) => setTimeout(r, 1500));+ return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);+ }+ throw err;+ } finally {+ clearTimeout(timer);+ }+}++const SYSTEM_PROMPT = (englishName, native) =>+ [+ `You are a professional translator for technical software documentation.`,+ `Translate the user's markdown content into ${englishName} (native: ${native}).`,+ `Preserve all markdown syntax EXACTLY: headings, lists, code blocks (\`\`\`), inline code (\`...\`), links, images, tables, blockquotes, HTML tags.`,+ `Do NOT translate: source code, URLs, file paths, command names (npm/git/curl/node/etc), environment variable names (UPPER_SNAKE_CASE),`,+ `version numbers, package names, shell flags, function/class identifiers, JSON keys.`,+ `Translate ALL prose, including comments inside code blocks IF they are clearly prose comments (lines starting with # or //).`,+ `Return ONLY the translated markdown — no preamble, no explanation, no surrounding fences.`,+ ].join(" ");++// Splits a markdown body into chunks of <= maxChars, breaking on top-level `## ` headings only.+function chunkMarkdown(markdown, maxChars = 6000) {+ if (markdown.length <= maxChars) return [markdown];+ const lines = markdown.split("\n");+ const chunks = [];+ let buf = [];+ let size = 0;+ for (const line of lines) {+ if (line.startsWith("## ") && size > maxChars * 0.5) {+ chunks.push(buf.join("\n"));+ buf = [line];+ size = line.length;+ } else {+ buf.push(line);+ size += line.length + 1;+ }+ }+ if (buf.length) chunks.push(buf.join("\n"));+ return chunks;+}++async function translateBody(body, localeEntry, backend) {+ const englishName = localeEntry.english ?? localeEntry.name;+ const native = localeEntry.native ?? localeEntry.name;+ const system = SYSTEM_PROMPT(englishName, native);+ const chunks = chunkMarkdown(body);+ const translated = [];+ for (let i = 0; i < chunks.length; i++) {+ const messages = [+ { role: "system", content: system },+ { role: "user", content: chunks[i] },+ ];+ const out = await callChat(messages, backend);+ translated.push(out.trim());+ if (chunks.length > 1) {+ logInfo(+ ` chunk ${i + 1}/${chunks.length} translated (${chunks[i].length} → ${out.length} chars)`+ );+ }+ }+ // Re-join with a blank line between chunks (we split on `## ` headings).+ return translated.join("\n\n");+}++// Simple promise-based semaphore (avoid runtime deps).+function createLimiter(max) {+ let active = 0;+ const queue = [];+ const next = () => {+ if (!queue.length || active >= max) return;+ active++;+ const { fn, resolve, reject } = queue.shift();+ fn()+ .then((v) => {+ active--;+ resolve(v);+ next();+ })+ .catch((err) => {+ active--;+ reject(err);+ next();+ });+ };+ return (fn) =>+ new Promise((resolve, reject) => {+ queue.push({ fn, resolve, reject });+ next();+ });+}++// ----- Main ----------------------------------------------------------------++async function main() {+ const opts = parseArgs(process.argv);+ const config = await loadConfig();+ const allSources = await collectAllSources();+ const state = await loadState();++ const sources = opts.files ? allSources.filter((s) => opts.files.includes(s)) : allSources;+ if (opts.files) {+ const missing = opts.files.filter((f) => !allSources.includes(f));+ if (missing.length) {+ logWarn(`--files contains paths not in the source set: ${missing.join(", ")}`);+ }+ }++ const docsExcluded = new Set(config.docsExcluded ?? ["en"]);+ let targetLocales = config.locales.map((l) => l.code).filter((code) => !docsExcluded.has(code));+ if (opts.locales) {+ targetLocales = targetLocales.filter((code) => opts.locales.includes(code));+ const missing = opts.locales.filter((c) => !config.locales.some((l) => l.code === c));+ if (missing.length) {+ logWarn(`--locale contains codes not in config/i18n.json: ${missing.join(", ")}`);+ }+ }++ logInfo(`sources: ${sources.length}`);+ logInfo(`locales: ${targetLocales.length} (${targetLocales.join(", ")})`);+ logInfo(`dry-run: ${opts.dryRun ? "yes" : "no"}, force: ${opts.force ? "yes" : "no"}`);++ // Read backend env up front so dry-run can still print masked summary.+ let backend = null;+ if (!opts.dryRun) {+ backend = backendConfig();+ if (opts.concurrency) backend.concurrency = opts.concurrency;+ else backend.concurrency = Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);+ logInfo(+ `backend: ${backend.apiUrl} (model=${backend.model}, concurrency=${backend.concurrency}, timeout=${backend.timeoutMs}ms)`+ );+ } else {+ const apiUrl = (process.env.OMNIROUTE_TRANSLATION_API_URL || "").replace(/\/$/, "");+ logInfo(`backend (dry-run): ${apiUrl || "<unset>"}`);+ }++ const limit = createLimiter(opts.dryRun ? 1 : backend.concurrency);++ let stats = { translated: 0, skipped: 0, failed: 0, considered: 0 };+ const failures = [];++ // Precompute source hashes once per source.+ const sourceHashes = new Map();+ for (const rel of sources) {+ const abs = path.join(ROOT, rel);+ const buf = await fs.readFile(abs);+ sourceHashes.set(rel, { hash: sha256(buf), text: buf.toString("utf8") });+ }++ // Build a flat queue of (source, locale) work units.+ const tasks = [];+ for (const rel of sources) {+ const { hash: sourceHash } = sourceHashes.get(rel);+ const entry =+ state.sources[rel] || (state.sources[rel] = { source_hash: sourceHash, locales: {} });+ entry.source_hash = sourceHash;++ for (const locale of targetLocales) {+ stats.considered++;+ const targetAbs = targetPathFor(rel, locale);+ const previous = entry.locales[locale];+ const sourceChanged = previous?.source_hash !== sourceHash;+ const missingTarget = !existsSync(targetAbs);+ if (!opts.force && !sourceChanged && !missingTarget) {+ stats.skipped++;+ continue;+ }+ tasks.push({ rel, locale, targetAbs, sourceChanged, missingTarget });+ }+ }++ logInfo(+ `work units: ${tasks.length} (skipped up-to-date: ${stats.skipped} of ${stats.considered})`+ );++ if (opts.dryRun) {+ for (const t of tasks) {+ console.log(` [DRY] ${t.rel} → ${path.relative(ROOT, t.targetAbs)}`);+ }+ logInfo(`dry-run complete — would translate ${tasks.length} files`);+ return;+ }++ const startMs = Date.now();++ await Promise.all(+ tasks.map((task) =>+ limit(async () => {+ const localeEntry = config.locales.find((l) => l.code === task.locale);+ const sourceText = sourceHashes.get(task.rel).text;+ const sourceHash = sourceHashes.get(task.rel).hash;+ const topHeading = extractTopHeading(sourceText);+ const body = stripTopHeading(sourceText);++ let translatedBody;+ try {+ translatedBody = await translateBody(body, localeEntry, backend);+ } catch (err) {+ stats.failed++;+ failures.push({ rel: task.rel, locale: task.locale, error: err.message });+ logError(`${task.rel} [${task.locale}] failed: ${err.message}`);+ return;+ }++ const heading = topHeading+ ? `# ${topHeading} (${localeEntry.native})`+ : `# ${path.basename(task.rel, ".md")} (${localeEntry.native})`;+ const langBar = buildLanguageBar(task.rel, task.locale, config);+ const rawContent = `${heading}\n\n${langBar}\n\n---\n\n${translatedBody.trim()}\n`;+ // Pre-format with Prettier (markdown parser) so the on-disk content+ // matches what `lint-staged` would produce. This keeps `target_hash`+ // stable across commit hooks.+ const finalContent = await formatMarkdown(rawContent, task.targetAbs);++ await fs.mkdir(path.dirname(task.targetAbs), { recursive: true });+ await fs.writeFile(task.targetAbs, finalContent, "utf8");++ const targetHash = sha256(Buffer.from(finalContent, "utf8"));+ state.sources[task.rel].locales[task.locale] = {+ source_hash: sourceHash,+ target_hash: targetHash,+ updated_at: new Date().toISOString(),+ };++ stats.translated++;+ logInfo(`✓ ${task.rel} → ${task.locale} (${translatedBody.length} chars)`);+ })+ )+ );++ // Save state even on partial failure so future runs only retry what failed.+ await saveState(state);++ const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);+ logInfo(+ `summary: translated=${stats.translated}, skipped=${stats.skipped}, failed=${stats.failed}, total considered=${stats.considered}, elapsed=${elapsedSec}s`+ );++ if (failures.length) {+ logWarn(`${failures.length} failures:`);+ for (const f of failures) console.warn(` - ${f.rel} [${f.locale}]: ${f.error}`);+ process.exit(1);+ }+}++const isDirectRun = import.meta.url === pathToFileURL(process.argv[1]).href;+if (isDirectRun) {+ main().catch((err) => {+ logError(err?.stack || err?.message || String(err));+ process.exit(1);+ });+}