Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

22.4 KB
Raw Blame
import fs from 'node:fs/promises'
import path from 'node:path'
import { pathToFileURL } from 'node:url'
import process from 'node:process'

// TypeScript 7 is a native CLI; AST consumers still need the legacy JavaScript API.
import ts from 'typescript-api'

import { canonicalGenericRenderings } from './locale-generic-ui-terms.mjs'
import { LOCALIZATION_SURFACES, readLocalizationSurface } from './localization-surfaces.mjs'
import { repairTranslatedValue } from './locale-translation-policy.mjs'
import {
  interpolationVariablesMatch,
  pluralFamilyDefaults,
  resolveLocaleSourceKey,
  syncLocalePluralForms
} from './locale-plural-forms.mjs'

const SOURCE_EXTENSIONS = new Set(['.ts', '.tsx', '.js', '.jsx', '.mts', '.cts'])
const SKIP_PATH_PARTS = new Set(['.git', 'dist', 'node_modules', 'out', '__snapshots__', 'assets'])
const LOCALIZATION_FUNCTION_NAMES = new Set([
  't',
  'translate',
  'translateMain',
  'translateSearchKeyword'
])
const PLACEHOLDER_RE = /\{\{[^}]+\}\}/g
export const LOCALIZATION_SOURCE_ROOTS = LOCALIZATION_SURFACES.desktop.sourceRoots

function normalizePath(root, filePath) {
  return path.relative(root, filePath).split(path.sep).join('/')
}

function isSkippedFile(root, filePath) {
  const relative = normalizePath(root, filePath)
  if (
    relative.endsWith('.d.ts') ||
    // Build output (bundled engines, protocol schemas) is not translatable source.
    relative.includes('.generated.') ||
    relative.includes('.test.') ||
    relative.includes('.spec.') ||
    relative.includes('/__tests__/')
  ) {
    return true
  }
  return relative.split('/').some((part) => SKIP_PATH_PARTS.has(part))
}

export async function collectSourceFiles(root, dir) {
  const entries = await fs.readdir(dir, { withFileTypes: true })
  const files = []

  for (const entry of entries) {
    const fullPath = path.join(dir, entry.name)
    if (entry.isDirectory()) {
      if (!SKIP_PATH_PARTS.has(entry.name)) {
        files.push(...(await collectSourceFiles(root, fullPath)))
      }
      continue
    }
    if (
      entry.isFile() &&
      SOURCE_EXTENSIONS.has(path.extname(entry.name)) &&
      !isSkippedFile(root, fullPath)
    ) {
      files.push(fullPath)
    }
  }

  return files
}

function flattenCatalogKeys(value, prefix = '') {
  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
    return prefix ? [prefix] : []
  }
  return Object.entries(value).flatMap(([key, child]) =>
    flattenCatalogKeys(child, prefix ? `${prefix}.${key}` : key)
  )
}

function expressionNameText(node) {
  if (ts.isIdentifier(node)) {
    return node.text
  }
  if (ts.isPropertyAccessExpression(node)) {
    return `${expressionNameText(node.expression) ?? ''}.${node.name.text}`.replace(/^\./, '')
  }
  return undefined
}

function reportAt(root, filePath, sourceFile, node, key, fallback) {
  const position = sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile))
  return {
    filePath: normalizePath(root, filePath),
    line: position.line + 1,
    column: position.character + 1,
    key,
    fallback
  }
}

export function collectLocalizationKeyReferences(filePath, sourceText, root = process.cwd()) {
  const sourceKind =
    filePath.endsWith('.tsx') || filePath.endsWith('.jsx') ? ts.ScriptKind.TSX : ts.ScriptKind.TS
  const sourceFile = ts.createSourceFile(
    filePath,
    sourceText,
    ts.ScriptTarget.Latest,
    true,
    sourceKind
  )
  const references = []

  function visit(node) {
    if (ts.isCallExpression(node)) {
      const name = expressionNameText(node.expression)
      const functionName = name?.split('.').at(-1)
      const firstArg = node.arguments[0]
      if (
        functionName &&
        LOCALIZATION_FUNCTION_NAMES.has(functionName) &&
        firstArg &&
        ts.isStringLiteralLike(firstArg)
      ) {
        const secondArg = node.arguments[1]
        const fallback = secondArg && ts.isStringLiteralLike(secondArg) ? secondArg.text : undefined
        const keys = pluralFamilyDefaults(ts, firstArg.text, fallback, node.arguments[2]) ?? [
          [firstArg.text, fallback]
        ]
        for (const [key, value] of keys) {
          references.push(reportAt(root, filePath, sourceFile, firstArg, key, value))
        }
      }
    }

    ts.forEachChild(node, visit)
  }

  visit(sourceFile)
  return references
}

function formatMissingReferences(missing) {
  return missing
    .map(
      (reference) => `${reference.filePath}:${reference.line}:${reference.column} ${reference.key}`
    )
    .join('\n')
}

function formatMissingKeys(label, keys) {
  return keys.map((key) => `${label}: ${key}`).join('\n')
}

// Language packs require nested names even though i18next also accepts flat keys.
function collectDottedCatalogKeys(value, prefix = '') {
  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
    return []
  }
  return Object.entries(value).flatMap(([key, child]) => {
    const keyPath = prefix ? `${prefix}.${key}` : key
    return [...(key.includes('.') ? [keyPath] : []), ...collectDottedCatalogKeys(child, keyPath)]
  })
}

function reportDottedCatalogKeys(fileName, catalog) {
  const dottedKeys = collectDottedCatalogKeys(catalog)
  if (dottedKeys.length === 0) {
    return 0
  }
  console.error(`${fileName} has keys with a dot in the name. Write them as nested objects.`)
  console.error(formatMissingKeys('dotted', dottedKeys.slice(0, 20)))
  if (dottedKeys.length > 20) {
    console.error(`...and ${dottedKeys.length - 20} more dotted keys`)
  }
  return 1
}

function normalizeInterpolationVariables(value) {
  return collectInterpolationVariables(value)
    .map((variable) => variable.slice(2, -2))
    .join('|')
}

function formatFallbackReferenceGroups(groups) {
  return groups
    .map(({ key, references }) => {
      const locations = references
        .map(
          (reference) =>
            `  ${reference.filePath}:${reference.line}:${reference.column} ${JSON.stringify(reference.fallback)}`
        )
        .join('\n')
      return `${key}\n${locations}`
    })
    .join('\n\n')
}

function groupFallbackReferencesByKey(references) {
  const byKey = new Map()

  for (const reference of references) {
    if (typeof reference.fallback !== 'string') {
      continue
    }
    const existing = byKey.get(reference.key) ?? []
    existing.push(reference)
    byKey.set(reference.key, existing)
  }
  return byKey
}

// Why: the catalog holds one value per key, so a second sentence on the same key never renders.
function collectReusedFallbackKeys(references) {
  return [...groupFallbackReferencesByKey(references).entries()]
    .filter(([, keyReferences]) => new Set(keyReferences.map((r) => r.fallback)).size > 1)
    .map(([key, keyReferences]) => ({ key, references: keyReferences }))
}

/** Prints keys reused for different fallback text; returns true when any exist. */
function reportReusedFallbackKeys(references) {
  const reusedFallbackKeys = collectReusedFallbackKeys(references)
  if (reusedFallbackKeys.length === 0) {
    return false
  }
  console.error('Localization keys are reused for different fallback text; give each its own key.')
  console.error('')
  console.error(formatFallbackReferenceGroups(reusedFallbackKeys))
  return true
}

function collectInconsistentFallbackVariables(references) {
  return [...groupFallbackReferencesByKey(references).entries()]
    .map(([key, keyReferences]) => {
      const uniqueFallbackVariables = new Set(
        keyReferences.map((reference) => normalizeInterpolationVariables(reference.fallback))
      )
      return {
        key,
        references: keyReferences,
        uniqueFallbackVariableCount: uniqueFallbackVariables.size
      }
    })
    .filter(({ uniqueFallbackVariableCount }) => uniqueFallbackVariableCount > 1)
}

function collectInterpolationVariables(value) {
  if (typeof value === 'string') {
    const matches = value.match(PLACEHOLDER_RE) ?? []
    return [...matches].sort()
  }
  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
    return []
  }
  return Object.values(value).flatMap((child) => collectInterpolationVariables(child))
}

function flattenCatalogEntries(value, prefix = '', entries = new Map()) {
  if (typeof value === 'string') {
    entries.set(prefix, value)
    return entries
  }
  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
    return entries
  }
  for (const [key, child] of Object.entries(value)) {
    flattenCatalogEntries(child, prefix ? `${prefix}.${key}` : key, entries)
  }
  return entries
}

function getCatalogEntry(catalog, key) {
  return key.split('.').reduce((cursor, part) => cursor?.[part], catalog)
}

function setCatalogEntry(catalog, key, value) {
  const parts = key.split('.')
  let cursor = catalog
  for (const part of parts.slice(0, -1)) {
    if (typeof cursor[part] !== 'object' || cursor[part] === null || Array.isArray(cursor[part])) {
      cursor[part] = {}
    }
    cursor = cursor[part]
  }
  cursor[parts.at(-1)] = value
}

function collectLocaleParityIssues(enCatalog, localeCatalog, localeName) {
  const enEntries = flattenCatalogEntries(enCatalog)
  const localeEntries = flattenCatalogEntries(localeCatalog)
  const missingInLocale = [...enEntries.keys()].filter((key) => !localeEntries.has(key))
  const extraInLocale = [...localeEntries.keys()].filter(
    (key) => !resolveLocaleSourceKey(enEntries, localeName, key)
  )
  const interpolationMismatches = []

  for (const [key, localeValue] of localeEntries) {
    const sourceKey = resolveLocaleSourceKey(enEntries, localeName, key)
    if (!sourceKey) {
      continue
    }
    const enVariables = collectInterpolationVariables(enEntries.get(sourceKey))
    const localeVariables = collectInterpolationVariables(localeValue)
    if (!interpolationVariablesMatch(sourceKey, enVariables, localeVariables)) {
      interpolationMismatches.push(key)
    }
  }

  return { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches }
}

function referencesMissingFallbacks(missing) {
  return missing.filter((reference) => typeof reference.fallback !== 'string')
}

function collectMissingCatalogEntries(missing) {
  const entries = new Map()

  for (const reference of missing) {
    if (typeof reference.fallback !== 'string') {
      continue
    }
    if (!entries.has(reference.key)) {
      entries.set(reference.key, reference.fallback)
    }
  }

  return entries
}

function applyMissingEnglishEntries(catalog, missing) {
  const entries = collectMissingCatalogEntries(missing)
  let changed = 0

  for (const [key, fallback] of entries) {
    if (getCatalogEntry(catalog, key) !== undefined) {
      continue
    }
    setCatalogEntry(catalog, key, fallback)
    changed += 1
  }

  return changed
}

// Check repair policy too: parity passed while repair rewrote translated generic terms into English (#12113).
export function collectGenericTermRegressions(enEntries, localeEntries, localeName) {
  const renderings = canonicalGenericRenderings(localeName)
  if (renderings.length === 0) {
    return []
  }

  const regressions = []
  for (const [key, enValue] of enEntries) {
    const localeValue = localeEntries.get(key)
    if (typeof enValue !== 'string' || typeof localeValue !== 'string') {
      continue
    }
    const repaired = repairTranslatedValue({ key, enValue, localeValue, locale: localeName })
    if (repaired === localeValue) {
      continue
    }
    // Why: {{agent}} is an interpolation name, not English copy the reader sees.
    const repairedCopy = repaired.replace(PLACEHOLDER_RE, '')
    for (const { form, terms } of renderings) {
      if (!localeValue.includes(form) || repaired.includes(form)) {
        continue
      }
      if (
        terms.some((term) => new RegExp(`(^|[^A-Za-z])${term}($|[^A-Za-z])`).test(repairedCopy))
      ) {
        regressions.push({ key, form, localeValue, repaired })
        break
      }
    }
  }
  return regressions
}

function formatGenericTermRegressions(regressions) {
  return regressions
    .map((entry) => `${entry.key}: ${entry.form} -> English (${entry.repaired})`)
    .join('\n')
}

// Why: JSON.parse silently keeps the last duplicate key.
function collectDuplicateCatalogKeys(fileName, text) {
  const sourceFile = ts.parseJsonText(fileName, text)
  const duplicates = []
  const visit = (node, prefix) => {
    if (!ts.isObjectLiteralExpression(node)) {
      return
    }
    const seen = new Set()
    for (const property of node.properties) {
      if (!ts.isPropertyAssignment(property)) {
        continue
      }
      const name = property.name.text
      const key = prefix ? `${prefix}.${name}` : name
      if (seen.has(name)) {
        const { line } = sourceFile.getLineAndCharacterOfPosition(property.getStart(sourceFile))
        duplicates.push(`${fileName}:${line + 1} ${key}`)
      }
      seen.add(name)
      visit(property.initializer, key)
    }
  }
  for (const statement of sourceFile.statements) {
    visit(statement.expression, '')
  }
  return duplicates
}

async function readCatalogFile(catalogPath) {
  const text = await fs.readFile(catalogPath, 'utf8')
  const duplicates = collectDuplicateCatalogKeys(path.basename(catalogPath), text)
  if (duplicates.length > 0) {
    console.error('Localization catalog has duplicate keys; JSON keeps only the last value.')
    console.error('')
    console.error(duplicates.map((duplicate) => `- ${duplicate}`).join('\n'))
    return undefined
  }
  return JSON.parse(text)
}

function verifyLocaleCatalog(enCatalog, localeName, localeCatalog) {
  const { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches } =
    collectLocaleParityIssues(enCatalog, localeCatalog, localeName)
  const genericTermRegressions = collectGenericTermRegressions(enEntries, localeEntries, localeName)

  // Feature PRs own English declarations; missing target translations use i18next fallback until localization supplies them.
  console.log(
    `${localeName}.json coverage: ${enEntries.size - missingInLocale.length}/${enEntries.size} translated, ${missingInLocale.length} missing.`
  )

  if (
    extraInLocale.length > 0 ||
    interpolationMismatches.length > 0 ||
    genericTermRegressions.length > 0
  ) {
    console.error(`Locale catalog validation failed for ${localeName}.json.`)
    if (extraInLocale.length > 0) {
      console.error('')
      console.error(formatMissingKeys('extra', extraInLocale.slice(0, 20)))
      if (extraInLocale.length > 20) {
        console.error(`...and ${extraInLocale.length - 20} more extra keys`)
      }
    }
    if (interpolationMismatches.length > 0) {
      console.error('')
      console.error(
        formatMissingKeys('interpolation mismatch', interpolationMismatches.slice(0, 20))
      )
      if (interpolationMismatches.length > 20) {
        console.error(`...and ${interpolationMismatches.length - 20} more interpolation mismatches`)
      }
    }
    if (genericTermRegressions.length > 0) {
      console.error('')
      console.error(
        'repair-locale-catalog would rewrite these translated terms back to English.',
        'Treat the term as generic in config/scripts/locale-generic-ui-terms.mjs',
        'instead of listing its translation as a mistranslation.'
      )
      console.error(formatGenericTermRegressions(genericTermRegressions.slice(0, 20)))
      if (genericTermRegressions.length > 20) {
        console.error(`...and ${genericTermRegressions.length - 20} more generic term regressions`)
      }
    }
    return 1
  }

  console.log(`Verified ${localeEntries.size} existing ${localeName}.json entries.`)
  return 0
}

function parseArgs(argv) {
  const pluginCatalogs = []
  for (let index = 0; index < argv.length; index += 1) {
    const argument = argv[index]
    if (argument === '--plugin-catalog') {
      const catalogPath = argv[index + 1]
      if (!catalogPath || catalogPath.startsWith('--')) {
        throw new Error('--plugin-catalog requires a JSON catalog path')
      }
      pluginCatalogs.push(catalogPath)
      index += 1
    } else if (argument.startsWith('--plugin-catalog=')) {
      pluginCatalogs.push(argument.slice('--plugin-catalog='.length))
    }
  }
  return {
    fix: argv.includes('--fix'),
    pluginCatalogs,
    surface: readLocalizationSurface(argv)
  }
}

async function reportPluginCatalog(root, catalog, pluginCatalogPath) {
  const resolvedPath = path.resolve(root, pluginCatalogPath)
  let pluginCatalog
  try {
    pluginCatalog = JSON.parse(await fs.readFile(resolvedPath, 'utf8'))
  } catch (error) {
    console.error(
      `Could not read plugin catalog ${normalizePath(root, resolvedPath)}: ${error instanceof Error ? error.message : String(error)}`
    )
    return 1
  }
  const { enEntries, localeEntries, missingInLocale, extraInLocale, interpolationMismatches } =
    collectLocaleParityIssues(catalog, pluginCatalog)
  const translated = enEntries.size - missingInLocale.length - interpolationMismatches.length
  const coverage = enEntries.size === 0 ? 100 : (translated / enEntries.size) * 100
  console.log(
    `Plugin catalog ${normalizePath(root, resolvedPath)}: ${translated}/${enEntries.size} core keys (${coverage.toFixed(1)}% coverage), ${localeEntries.size} catalog entries.`
  )
  if (missingInLocale.length > 0) {
    console.log(formatMissingKeys('missing', missingInLocale.slice(0, 20)))
    if (missingInLocale.length > 20) {
      console.log(`...and ${missingInLocale.length - 20} more missing keys`)
    }
  }
  if (extraInLocale.length > 0) {
    console.log(formatMissingKeys('extra', extraInLocale.slice(0, 20)))
  }
  if (interpolationMismatches.length > 0) {
    console.log(formatMissingKeys('interpolation mismatch', interpolationMismatches.slice(0, 20)))
  }
  // Missing plugin translation can fall back; mismatched variables in present value can mislead or break UI.
  return interpolationMismatches.length > 0 ? 1 : 0
}

export async function collectLocalizationReferences(root, roots = LOCALIZATION_SOURCE_ROOTS) {
  const sourceRoots = roots.map((sourceRoot) => path.join(root, sourceRoot))
  const references = []

  for (const sourceRoot of sourceRoots) {
    const files = await collectSourceFiles(root, sourceRoot)
    for (const filePath of files) {
      references.push(
        ...collectLocalizationKeyReferences(filePath, await fs.readFile(filePath, 'utf8'), root)
      )
    }
  }

  return references
}

export async function main(
  root = process.cwd(),
  options = parseArgs(process.argv.slice(2)),
  sharedReferences
) {
  const surface = options.surface ?? LOCALIZATION_SURFACES.desktop
  const localesDir = path.join(root, surface.localesDir)
  const catalogPath = path.join(localesDir, 'en.json')
  const enCatalogLabel = normalizePath(root, catalogPath)
  const catalog = await readCatalogFile(catalogPath)
  if (catalog === undefined) {
    return 1
  }
  const pluginCatalogs = options.pluginCatalogs ?? []
  if (pluginCatalogs.length > 0) {
    if (options.fix) {
      console.error('--fix cannot be combined with --plugin-catalog')
      return 1
    }
    for (const pluginCatalogPath of pluginCatalogs) {
      const result = await reportPluginCatalog(root, catalog, pluginCatalogPath)
      if (result !== 0) {
        return result
      }
    }
    return 0
  }
  if (reportDottedCatalogKeys('en.json', catalog) !== 0) {
    return 1
  }
  let catalogKeys = new Set(flattenCatalogKeys(catalog))
  const references =
    sharedReferences ?? (await collectLocalizationReferences(root, surface.sourceRoots))

  // Why: runs before --fix so an ambiguous key never gets its first sentence written to en.json.
  if (reportReusedFallbackKeys(references)) {
    return 1
  }

  const missing = references.filter((reference) => !catalogKeys.has(reference.key))
  if (missing.length > 0) {
    const missingFallbacks = referencesMissingFallbacks(missing)
    if (options.fix && missingFallbacks.length === 0) {
      const added = applyMissingEnglishEntries(catalog, missing)
      await fs.writeFile(catalogPath, `${JSON.stringify(catalog, null, 2)}\n`, 'utf8')
      catalogKeys = new Set(flattenCatalogKeys(catalog))
      console.log(`Added ${added} missing localization key(s) to en.json.`)
    } else {
      if (options.fix && missingFallbacks.length > 0) {
        console.error('Some missing localization keys do not have string fallbacks to bootstrap.')
        console.error('')
        console.error(formatMissingReferences(missingFallbacks))
        return 1
      }
      console.error(`Localization keys are missing from ${enCatalogLabel}.`)
      console.error('')
      console.error(formatMissingReferences(missing))
      console.error('')
      console.error(`Run \`${surface.syncCatalogCommand}\` to add keys with string fallbacks.`)
      return 1
    }
  }

  const remainingMissing = references.filter((reference) => !catalogKeys.has(reference.key))
  if (remainingMissing.length > 0) {
    console.error(`Localization keys are missing from ${enCatalogLabel}.`)
    console.error('')
    console.error(formatMissingReferences(remainingMissing))
    return 1
  }

  const inconsistentFallbackVariables = collectInconsistentFallbackVariables(references)
  if (inconsistentFallbackVariables.length > 0) {
    console.error('Localization keys are used with inconsistent interpolation placeholders.')
    console.error('')
    console.error(formatFallbackReferenceGroups(inconsistentFallbackVariables))
    return 1
  }

  console.log(`Verified ${references.length} localization key references against en.json.`)

  const localeFiles = (await fs.readdir(localesDir))
    .filter(
      (fileName) =>
        fileName.endsWith('.json') &&
        fileName !== 'en.json' &&
        !fileName.startsWith('.') &&
        !fileName.includes('-catalog-cache')
    )
    .sort()

  for (const fileName of localeFiles) {
    const localeName = fileName.replace(/\.json$/, '')
    const localeCatalogPath = path.join(localesDir, fileName)
    let localeCatalog = await readCatalogFile(localeCatalogPath)
    if (localeCatalog === undefined || reportDottedCatalogKeys(fileName, localeCatalog) !== 0) {
      return 1
    }
    if (surface.fillsPluralForms) {
      const plural = await syncLocalePluralForms(localeCatalogPath, localeCatalog, options.fix)
      if (!plural.ok) {
        console.error(
          `Run \`${surface.syncCatalogCommand}\` to copy them from each \`_other\` form.`
        )
        return 1
      }
      localeCatalog = plural.catalog
    }
    const exitCode = verifyLocaleCatalog(catalog, localeName, localeCatalog)
    if (exitCode !== 0) {
      console.error('')
      console.error('Fix or retire the existing target entry in a localization PR.')
      return exitCode
    }
  }

  return 0
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  process.exit(await main())
}