Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
3.4 KB
Raw Blame
/**
 * Whitespace- and comment-insensitive token normalizer (refactor wave 1, CQ7).
 *
 * One definition shared by the two move proofs:
 * - the migrations full-record golden (`test/migrations-golden.test.ts`), which
 *   hashes the normalized source of every handler / verify body and the helpers
 *   they reference, never `Function.prototype.toString()` (coverage
 *   instrumentation and re-indentation change that text);
 * - `scripts/verify-move-only.ts`, which proves a move-only commit keeps the
 *   same token stream.
 *
 * Normal form: the TypeScript scanner's token stream with trivia (whitespace,
 * newlines, comments, shebang) dropped, each token rendered as its exact source
 * text and joined with a single space. Template literal and string contents are
 * kept byte for byte, so SQL inside a template literal is never normalized.
 * An optional rename map rewrites identifiers (for `Mechanical-Rename: yes`
 * commits such as `pullFailed -> run.pullFailed`).
 */

import ts from 'typescript';

export interface NormalizeOptions {
  /** Identifier rewrites applied to identifier tokens, e.g. `{ pullFailed: 'run.pullFailed' }`. */
  renameMap?: Readonly<Record<string, string>>;
  /** Parse as TSX when true. Default false. */
  jsx?: boolean;
}

/** Return the normalized token texts of `sourceText`. */
export function normalizeTokens(sourceText: string, opts: NormalizeOptions = {}): string[] {
  const sourceFile = ts.createSourceFile(
    opts.jsx ? 'input.tsx' : 'input.ts',
    sourceText,
    ts.ScriptTarget.Latest,
    true,
    opts.jsx ? ts.ScriptKind.TSX : ts.ScriptKind.TS,
  );
  const tokens: string[] = [];
  const visit = (node: ts.Node): void => {
    if (node.kind >= ts.SyntaxKind.FirstJSDocNode && node.kind <= ts.SyntaxKind.LastJSDocNode) return;
    if (node.kind >= ts.SyntaxKind.FirstToken && node.kind <= ts.SyntaxKind.LastToken) {
      pushToken(node, sourceFile, tokens, opts);
      return;
    }
    if (isLiteralLike(node)) {
      pushToken(node, sourceFile, tokens, opts);
      return;
    }
    const children = node.getChildren(sourceFile);
    for (const child of children) visit(child);
  };
  for (const child of sourceFile.getChildren(sourceFile)) visit(child);
  return tokens.filter((t) => t.length > 0);
}

/** Normalized source text: tokens joined by one space. */
export function normalizeSource(sourceText: string, opts: NormalizeOptions = {}): string {
  return normalizeTokens(sourceText, opts).join(' ');
}

/** Normalized text of one node of an already-parsed source file. */
export function normalizeNode(node: ts.Node, sourceFile: ts.SourceFile, opts: NormalizeOptions = {}): string {
  return normalizeSource(node.getText(sourceFile), opts);
}

function isLiteralLike(node: ts.Node): boolean {
  return (
    ts.isNoSubstitutionTemplateLiteral(node) ||
    ts.isStringLiteral(node) ||
    ts.isNumericLiteral(node) ||
    ts.isBigIntLiteral(node) ||
    ts.isRegularExpressionLiteral(node) ||
    ts.isTemplateHead(node) ||
    ts.isTemplateMiddle(node) ||
    ts.isTemplateTail(node) ||
    ts.isJsxText(node)
  );
}

function pushToken(node: ts.Node, sourceFile: ts.SourceFile, out: string[], opts: NormalizeOptions): void {
  if (node.kind === ts.SyntaxKind.EndOfFileToken) return;
  const text = node.getText(sourceFile);
  if (opts.renameMap && ts.isIdentifier(node) && Object.prototype.hasOwnProperty.call(opts.renameMap, text)) {
    out.push(...normalizeTokens(opts.renameMap[text]!));
    return;
  }
  out.push(text);
}