Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
43.9 KB
Raw Blame
import { assertManagedFilesystemWrite } from './persistence/filesystem-guard.ts';
/**
 * brain-writer — frontmatter validation/audit/auto-fix orchestrator.
 *
 * Thin layer on top of `parseMarkdown(..., {validate:true})` (the canonical
 * source of frontmatter validation rules) and `isSyncable()` (the canonical
 * brain-page filter). Three consumers call into this module: the
 * `gbrain frontmatter` CLI, the `frontmatter_integrity` doctor subcheck, and
 * the v0.22.4 migration audit phase. Single source of truth — no parallel
 * validation stack.
 *
 * Path-guard contract: writeBrainPage refuses to write outside the source
 * path. Pre-write backups are the safety contract (works for both git and
 * non-git brain repos; the existing src/core/dry-fix.ts:getWorkingTreeStatus
 * rejects non-git repos as unsafe, which is the wrong shape for brain
 * rewrites). Backups live under ~/.gbrain/backups/frontmatter/... instead of
 * beside source files so bulk repair never litters the user's workspace.
 */

import { createHash } from 'crypto';
import { loadBoundedYaml as yamlLoad } from './data-frontmatter.ts';
import { existsSync, readFileSync, readdirSync, copyFileSync, writeFileSync, mkdirSync, lstatSync } from 'fs';
import { join, relative, resolve, dirname, basename } from 'path';
import type { BrainEngine } from './engine.ts';
import { isConnectorSourceKind } from './persistence/connector-kinds.ts';
import type { ProgressReporter } from './progress.ts';
import { gbrainPath } from './config.ts';
import { collectGitVisibleFiles } from './git-visible-files.ts';
import {
  classifyImportHold,
  parseMarkdown,
  type ParseValidationCode,
  type ParseValidationError,
  type ParseWarningCode,
  yamlAliasesWithinLimit,
  yamlBlockError,
} from './markdown.ts';
import { DEFAULT_METAFILE_PAGES_MODE, isMarkdownFilePath, isSyncable, pruneDir, readMetafilePagesMode, slugifyPath, type MetafilePagesMode } from './sync.ts';
import { diskMetafileHeads, metafilePagePredicate } from './sync-metafile-pages.ts';
import { relativeEscapesRoot } from './path-confine.ts';

export type { ParseValidationCode };

/** Frontmatter validation is defined only for Markdown page files. */
export function isFrontmatterScannablePath(path: string, metafilePage?: (path: string) => boolean): boolean {
  return isMarkdownFilePath(path) && isSyncable(path, { strategy: 'markdown', metafilePage });
}

/** GBRA-150: `isFrontmatterScannablePath` for paths relative to `root`, where a typed nested metafile is a page under `mode`. */
export function frontmatterScannableUnder(root: string, mode: MetafilePagesMode = DEFAULT_METAFILE_PAGES_MODE): (path: string) => boolean {
  const metafilePage = metafilePagePredicate(mode, diskMetafileHeads(root));
  return path => isFrontmatterScannablePath(path, metafilePage);
}

export interface AuditFix {
  code: ParseValidationCode | ParseWarningCode;
  description: string;
}

/**
 * #5988: per code, how many findings ingestion imports anyway (`recoverable`:
 * e.g. YAML it reads by quoting a value) versus how many make the file a held
 * import (`unrecoverable`: YAML it cannot read safely, a conflicting slug, a
 * protected key in an unclosed fence).
 */
export type RecoverabilityByCode = Partial<Record<ParseValidationCode, { recoverable: number; unrecoverable: number }>>;

/**
 * #5988: files `gbrain repair frontmatter` can fix, by the cause it fixes:
 * recoverable `YAML_PARSE`, `needs_interpretation` (the held reading needs a
 * fold or a duplicate-key choice), a `#`-leading value YAML read as a comment
 * (`FRONTMATTER_COMMENT_VALUE` on title/name/description/summary),
 * `NESTED_QUOTES`, `NULL_BYTES`, `MISSING_CLOSE`, `SLUG_MISMATCH`. A file
 * counts once in `files` and once per cause in `by_code`.
 */
export interface RepairableReport {
  files: number;
  by_code: Partial<Record<RepairableCause, number>>;
  sample: string[];
}

export type RepairableCause = 'YAML_PARSE' | 'needs_interpretation' | 'FRONTMATTER_COMMENT_VALUE' | 'NESTED_QUOTES' | 'NULL_BYTES' | 'MISSING_CLOSE' | 'SLUG_MISMATCH';

const REPAIRABLE_ERRORS: ReadonlySet<string> = new Set(['NESTED_QUOTES', 'NULL_BYTES', 'MISSING_CLOSE', 'SLUG_MISMATCH']);
const RESCUE_KEYS: ReadonlySet<string> = new Set(['title', 'name', 'description', 'summary']);

export interface PerSourceReport {
  source_id: string;
  source_path: string;
  total: number;
  errors_by_code: Partial<Record<ParseValidationCode, number>>;
  recoverability_by_code?: RecoverabilityByCode;
  repairable?: RepairableReport;
  sample: { path: string; codes: ParseValidationCode[] }[];
  ignoredMissingOpen: number;
  /** Did this source finish the walk, get interrupted, or never start?
   * 'scanned' = full walk completed.
   * 'partial' = deadline/abort fired mid-walk; counts reflect prefix only.
   * 'skipped' = source was never visited (outer loop broke before reaching it).
   * 'missing' = the registered local_path does not exist, so nothing was checked (#7889). */
  status: 'scanned' | 'partial' | 'skipped' | 'missing';
  /** Count of .md files actually parsed (numerator for "scanned ~N of M" doctor message).
   * Distinct from `total` (which counts ERRORS) and `ignoredMissingOpen`. */
  files_scanned: number;
  /** DB-side denominator for the partial-state message. Coarse — DB pages and
   * on-disk syncable files are overlapping but not identical sets. NULL when the
   * COUNT query failed or wasn't issued. */
  db_page_count?: number | null;
}

export interface AuditReport {
  ok: boolean;
  total: number;
  errors_by_code: Partial<Record<ParseValidationCode, number>>;
  recoverability_by_code?: RecoverabilityByCode;
  per_source: PerSourceReport[];
  scanned_at: string;
  ignored_missing_open?: number;
  /** True when any source got `status: 'partial'` or `'skipped'`. Doctor uses
   * this to render the warn message and to ensure `ok` is false even when the
   * scanned prefix happened to be clean (codex C2 fix). */
  partial: boolean;
  /** Ids of sources whose registered directory is missing (`status: 'missing'`); `ok` is false when any is. */
  missing_sources?: string[];
  /** Source id where deadline/abort fired mid-walk, or null when no partial. */
  aborted_at_source: string | null;
}

const SAMPLE_PER_SOURCE = 20;

// ---------------------------------------------------------------------------
// Frontmatter backups
// ---------------------------------------------------------------------------

export function makeFrontmatterBackupRunId(date = new Date()): string {
  return date.toISOString().replace(/[:.]/g, '-');
}

export interface FrontmatterBackupOpts {
  sourcePath?: string;
  backupRoot?: string;
  runId?: string;
}

function sourceKey(sourcePath: string): string {
  return createHash('sha256').update(resolve(sourcePath)).digest('hex').slice(0, 12);
}

export function defaultFrontmatterBackupRoot(runId = makeFrontmatterBackupRunId()): string {
  return gbrainPath('backups', 'frontmatter', runId);
}

export function createFrontmatterBackup(filePath: string, opts: FrontmatterBackupOpts = {}): string {
  const resolvedFile = resolve(filePath);
  const resolvedSource = resolve(opts.sourcePath ?? dirname(resolvedFile));
  const rel = relative(resolvedSource, resolvedFile);
  const safeRel = rel && !relativeEscapesRoot(rel) ? rel : basename(resolvedFile);
  const root = opts.backupRoot ?? defaultFrontmatterBackupRoot(opts.runId);
  const backupPath = join(root, sourceKey(resolvedSource), safeRel + '.bak');
  mkdirSync(dirname(backupPath), { recursive: true });
  copyFileSync(resolvedFile, backupPath);
  return backupPath;
}

// ---------------------------------------------------------------------------
// autoFixFrontmatter
// ---------------------------------------------------------------------------

/**
 * A cosmetic line rewrite is kept only when it reads back as the same value
 * (#6157). A line that does not parse on its own is a repair target, so the
 * rewrite is allowed. A line over the alias limit is never rewritten: its
 * value could expand exponentially when compared.
 */
function rewriteKeepsYamlValue(before: string, after: string): boolean {
  if (!yamlAliasesWithinLimit(before)) return false;
  let original: unknown;
  try {
    original = yamlLoad(before);
  } catch {
    return true;
  }
  try {
    return JSON.stringify(yamlLoad(after)) === JSON.stringify(original);
  } catch {
    return false;
  }
}

/**
 * Mechanical auto-repair for the fixable subset of validation codes:
 *   - NULL_BYTES        — strip \x00 characters
 *   - NESTED_QUOTES     — rewrite `"... "inner" ..."` to single-quoted outer
 *   - MISSING_CLOSE     — insert `---` before the first heading found inside
 *                          the YAML zone
 *   - SLUG_MISMATCH     — remove `slug:` line (gbrain derives slug from path)
 *                          when `opts.filePath` is given. It must be the path
 *                          the slug derives from (relative to the brain or
 *                          source root, #5053), never an absolute path.
 *
 * Idempotent: running twice is a no-op on already-clean input. Any error class
 * not in the list above is left untouched (e.g. EMPTY_FRONTMATTER, YAML_PARSE,
 * MISSING_OPEN — those need human review).
 */
export function autoFixFrontmatter(
  content: string,
  opts?: { filePath?: string },
): { content: string; fixes: AuditFix[] } {
  const fixes: AuditFix[] = [];
  let working = content;

  // 1. NULL_BYTES — strip them. Cheap, byte-level. Run first so subsequent
  //    line-based passes don't trip on stray nulls.
  if (working.indexOf('\x00') >= 0) {
    working = working.replace(/\x00/g, '');
    fixes.push({ code: 'NULL_BYTES', description: 'Stripped null bytes' });
  }

  // 2. MISSING_CLOSE — if there's an opener but no closer at all, insert
  //    `---` immediately before the first heading-shaped line (best-effort
  //    guess at where the frontmatter was meant to end).
  //
  //    Find the closer FIRST, scanning the full zone — do not stop at the
  //    first `#`-prefixed line. A `#` line between the opening and closing
  //    `---` is a YAML comment (comments are valid anywhere in a YAML
  //    document), not a markdown heading; only the genuine absence of a
  //    closing `---` counts as MISSING_CLOSE. Mirrors the fix applied to
  //    the parseMarkdown validator in #2153 — this is the sibling
  //    reimplementation in the auto-fixer and had the same bug (it broke
  //    out of the scan on the first heading-shaped line, so a `#` comment
  //    appearing before a real closing fence was misdetected as
  //    MISSING_CLOSE and the fix inserted a spurious `---` that split
  //    valid frontmatter in two, pushing the real keys into the body).
  {
    const lines = working.split('\n');
    let firstNonEmpty = -1;
    for (let i = 0; i < lines.length; i++) {
      if (lines[i].trim().length > 0) { firstNonEmpty = i; break; }
    }
    if (firstNonEmpty >= 0 && lines[firstNonEmpty].trim() === '---') {
      let closeIdx = -1;
      for (let i = firstNonEmpty + 1; i < lines.length; i++) {
        if (lines[i].trim() === '---') { closeIdx = i; break; }
      }
      if (closeIdx === -1) {
        let headingIdx = -1;
        for (let i = firstNonEmpty + 1; i < lines.length; i++) {
          if (/^#{1,6}\s/.test(lines[i].trim())) { headingIdx = i; break; }
        }
        if (headingIdx >= 0) {
          const fixed = [
            ...lines.slice(0, headingIdx),
            '---',
            '',
            ...lines.slice(headingIdx),
          ];
          working = fixed.join('\n');
          fixes.push({
            code: 'MISSING_CLOSE',
            description: `Inserted closing --- before heading at line ${headingIdx + 1}`,
          });
        }
      }
    }
  }

  // Both step 3a and step 3 produce NESTED_QUOTES fix records on different
  // patterns. When both fire on the same file, push ONE merged record rather
  // than two — keeps the audit count honest about distinct files affected.
  let nestedQuotesFixed = false;

  // 3a. Canonical-style normalization for `tags:` / `aliases:` flow arrays.
  //     Post-v0.37.5.0 validator (PR #1229), `tags: ["yc", "w2025"]` is already
  //     valid YAML and no longer flagged. This pass rewrites it to the
  //     canonical single-quoted form (`tags: ['yc', 'w2025']`) so disk-side
  //     `frontmatter validate --fix` produces output consistent with the
  //     v0.37.9.0 serializer. Allow-list keys deliberately scoped to
  //     `tags` / `aliases` — extending to arbitrary keys would rewrite typed
  //     arrays (e.g. `scores: ["1", "2"]` would lose numeric intent).
  {
    const lines = working.split('\n');
    let firstNonEmpty = -1;
    for (let i = 0; i < lines.length; i++) {
      if (lines[i].trim().length > 0) { firstNonEmpty = i; break; }
    }
    if (firstNonEmpty >= 0 && lines[firstNonEmpty].trim() === '---') {
      let closeIdx = lines.length;
      for (let i = firstNonEmpty + 1; i < lines.length; i++) {
        if (lines[i].trim() === '---') { closeIdx = i; break; }
      }
      let fixedAny = false;
      for (let i = firstNonEmpty + 1; i < closeIdx; i++) {
        // Allow-list: only `tags` and `aliases` (the keys this wave targets).
        const arrMatch = lines[i].match(/^(\s*(?:tags|aliases)\s*:\s*)\[(.*)\]\s*$/);
        if (!arrMatch || !arrMatch[2].includes('"')) continue;
        const [, prefix, inner] = arrMatch;
        // Quote-aware comma split — items may contain commas inside quotes.
        const items: string[] = [];
        let current = '';
        let inQuote = false;
        for (let j = 0; j < inner.length; j++) {
          const ch = inner[j];
          if (ch === '"' && (j === 0 || inner[j - 1] !== '\\')) {
            inQuote = !inQuote;
          } else if (ch === ',' && !inQuote) {
            items.push(current.trim());
            current = '';
          } else {
            current += ch;
          }
        }
        if (current.trim()) items.push(current.trim());

        // Re-quote: single quotes by default, double-quote fallback when the
        // item contains an apostrophe (YAML's single-quoted form would need
        // `''` escaping which the validator accepts but reads poorly).
        const reQuoted = items.map(v => {
          const clean = v.replace(/^"|"$/g, '').trim();
          if (!clean) return "''";
          return clean.includes("'") ? `"${clean}"` : `'${clean}'`;
        });
        const rewritten = `${prefix}[${reQuoted.join(', ')}]`;
        if (!rewriteKeepsYamlValue(lines[i], rewritten)) continue;
        lines[i] = rewritten;
        fixedAny = true;
      }
      if (fixedAny) {
        working = lines.join('\n');
        fixes.push({
          code: 'NESTED_QUOTES',
          description: 'Normalized JSON-style double-quoted tag/alias arrays to single-quoted YAML',
        });
        nestedQuotesFixed = true;
      }
    }
  }

  // 3. NESTED_QUOTES — rewrite `key: "...inner..."` lines that have 3+ unescaped
  //    double-quotes by switching the outer wrapper to single quotes and
  //    leaving inner quotes alone. Only when the closed block fails to parse
  //    (#6157): valid YAML, such as a block-scalar continuation line shaped
  //    like `Key: "a", then "b"`, is never rewritten.
  {
    const lines = working.split('\n');
    let firstNonEmpty = -1;
    for (let i = 0; i < lines.length; i++) {
      if (lines[i].trim().length > 0) { firstNonEmpty = i; break; }
    }
    if (firstNonEmpty >= 0 && lines[firstNonEmpty].trim() === '---') {
      let closeIdx = lines.length;
      for (let i = firstNonEmpty + 1; i < lines.length; i++) {
        if (lines[i].trim() === '---') { closeIdx = i; break; }
      }
      const block = lines.slice(firstNonEmpty + 1, closeIdx).join('\n');
      const blockParses = closeIdx < lines.length && yamlBlockError(block) === null && yamlAliasesWithinLimit(block);
      let fixedAny = false;
      for (let i = firstNonEmpty + 1; !blockParses && i < closeIdx; i++) {
        const m = lines[i].match(/^(\s*[A-Za-z_][\w-]*\s*:\s*)"(.*)"\s*(.*)$/);
        if (!m) continue;
        const [, prefix, inner, trailing] = m;
        let count = 0;
        for (let j = 0; j < inner.length; j++) {
          if (inner[j] === '"' && (j === 0 || inner[j - 1] !== '\\')) count++;
        }
        // Total " on the line includes the two outer quotes the regex
        // captured, plus whatever's in inner. We need 3+ to trigger.
        if (count >= 1) {
          // Inner already has unescaped " — outer wrap is causing the YAML
          // parse failure. Rewrite to 'single-quoted'. YAML escapes `'` inside
          // a single-quoted string by doubling it.
          const escapedInner = inner.replace(/'/g, "''");
          lines[i] = `${prefix}'${escapedInner}'${trailing ? ' ' + trailing : ''}`.replace(/\s+$/, '');
          fixedAny = true;
        }
      }
      if (fixedAny) {
        working = lines.join('\n');
        if (!nestedQuotesFixed) {
          fixes.push({
            code: 'NESTED_QUOTES',
            description: 'Rewrote nested double-quoted YAML values to single-quoted',
          });
          nestedQuotesFixed = true;
        }
      }
    }
  }

  // 4. SLUG_MISMATCH — remove `slug:` line if filePath is provided and the
  //    declared slug doesn't match the path-derived one. Per PR #392 spec,
  //    gbrain derives slug from path; the field shouldn't be in frontmatter.
  if (opts?.filePath) {
    const expectedSlug = slugifyPath(opts.filePath);
    // Use the (possibly partially-fixed) working content to detect whether
    // the slug field is present and mismatched.
    const re = /^slug:\s*(.+?)\s*$/m;
    const m = working.match(re);
    // #3772: keep a normalization-equivalent slug (its slugified spelling IS
    // the path-derived slug) — export stamps these to preserve legacy page
    // identities, and stripping it here would re-key the page on next import.
    const declaredSlug = m ? m[1].replace(/^["']|["']$/g, '') : '';
    if (m && declaredSlug !== expectedSlug && slugifyPath(declaredSlug) !== expectedSlug) {
      working = working.replace(re, '').replace(/\n{3,}/g, '\n\n');
      fixes.push({
        code: 'SLUG_MISMATCH',
        description: `Removed mismatched slug field (was "${m[1]}", expected "${expectedSlug}")`,
      });
    }
  }

  return { content: working, fixes };
}

/** Keys whose `#`-leading value is rescued (quoted) under `includeAmbiguous`. */
const COMMENT_RESCUE_KEYS: ReadonlySet<string> = new Set(['title', 'name', 'description', 'summary']);

/** What ingestion reads from content: the values a safe repair must keep. */
function ingestedReading(content: string): string {
  const parsed = parseMarkdown(content, undefined, { validate: true });
  if (parsed.errors?.some(error => error.code === 'YAML_PARSE' && !error.recoverable)) return '';
  return JSON.stringify([parsed.frontmatter, parsed.title, parsed.type, parsed.tags]);
}

/** Parses with no YAML error at all (strict, no recovery needed) and earns no import hold. */
function strictlyReadable(candidate: string): boolean {
  const parsed = parseMarkdown(candidate, undefined, { validate: true });
  return !parsed.errors?.some(error => error.code === 'YAML_PARSE') && classifyImportHold(parsed) === null;
}

/**
 * Swap the YAML block between the fences for `block` (as `recoverFrontmatter`
 * returns it: rows joined by `\n`, each keeping its own `\r`, a trailing empty
 * row). Every line outside the block stays byte-identical, BOM included.
 */
function replaceFrontmatterBlock(content: string, block: string): string | null {
  const lines = content.split('\n');
  const open = lines.findIndex(line => line.trim().length > 0);
  if (open < 0 || !/^---[ \t]*(?:ya?ml)?[ \t]*$/i.test(lines[open]!.replace(/^\uFEFF/, '').replace(/\r$/, ''))) return null;
  const close = lines.findIndex((line, i) => i > open && /^---[\t ]*\r?$/.test(line));
  const rows = block.split('\n');
  if (close < 0 || rows.pop() !== '') return null;
  return [...lines.slice(0, open + 1), ...rows, ...lines.slice(close)].join('\n');
}

/**
 * #5988: rewrite frontmatter that ingestion can read but a strict YAML parser
 * cannot. Safe by default: only the `quote` recoveries `parseMarkdown`
 * applied, line by line (only recovered lines change; CRLF and BOM kept), and
 * only when the rewritten file strict-parses to exactly the values ingestion
 * read. `includeAmbiguous` adds the interpretive proposals (unquoted
 * continuation lines folded into a value, a duplicated key resolved to the
 * later line, an unclosed `[`/`{` quoted, a `#`-leading title quoted), each
 * applied only when the result strict-parses with no hold.
 */
export function repairRecoverableFrontmatter(
  content: string,
  opts: { includeAmbiguous?: boolean } = {},
): { content: string; fixes: AuditFix[] } {
  const fixes: AuditFix[] = [];
  let working = content;

  const parsed = parseMarkdown(working, undefined, { validate: true });
  const quotes = (parsed.warnings ?? []).filter(w => w.code === 'FRONTMATTER_RECOVERED' && w.kind === 'quote' && w.original !== undefined && w.replacement !== undefined);
  if (quotes.length > 0) {
    const lines = working.split('\n');
    const applied = quotes.every(w => {
      const line = lines[w.line - 1];
      if (line === undefined) return false;
      const eol = line.endsWith('\r') ? '\r' : '';
      if (line.slice(0, line.length - eol.length) !== w.original) return false;
      lines[w.line - 1] = w.replacement + eol;
      return true;
    });
    const candidate = lines.join('\n');
    if (applied && strictlyReadable(candidate) && ingestedReading(candidate) === ingestedReading(working)) {
      working = candidate;
      for (const w of quotes) fixes.push({ code: 'YAML_PARSE', description: `Quoted the value of "${w.key}" at line ${w.line} (value unchanged)` });
    }
  }

  if (!opts.includeAmbiguous) return { content: working, fixes };

  const proposal = parseMarkdown(working, undefined, { validate: true }).recovery;
  if (proposal?.status === 'needs_interpretation') {
    const candidate = replaceFrontmatterBlock(working, proposal.block);
    if (candidate !== null && strictlyReadable(candidate)) {
      working = candidate;
      for (const step of proposal.steps) {
        fixes.push({ code: 'YAML_PARSE', description: step.kind === 'fold' ? `Folded the unquoted lines after "${step.key}" (line ${step.line}) into its value`
          : step.kind === 'dup' ? `Kept the later "${step.key}" (line ${step.line}) and dropped the earlier one (line ${step.otherLine})`
          : step.kind === 'unclosed' ? `Quoted the unclosed [ or { value of "${step.key}" at line ${step.line} as text`
          : `Quoted the value of "${step.key}" at line ${step.line} (value unchanged)` });
      }
    }
  }

  const comments = (parseMarkdown(working, undefined, { validate: true }).warnings ?? [])
    .filter(w => w.code === 'FRONTMATTER_COMMENT_VALUE' && COMMENT_RESCUE_KEYS.has(w.key));
  if (comments.length > 0) {
    const lines = working.split('\n');
    for (const w of comments) {
      const line = lines[w.line - 1]!;
      const eol = line.endsWith('\r') ? '\r' : '';
      const value = line.slice(0, line.length - eol.length).replace(/^[A-Za-z_][\w-]*:[ \t]+/, '').trimEnd();
      lines[w.line - 1] = `${w.key}: ${JSON.stringify(value)}${eol}`;
    }
    const candidate = lines.join('\n');
    if (strictlyReadable(candidate)) {
      working = candidate;
      for (const w of comments) fixes.push({ code: 'FRONTMATTER_COMMENT_VALUE', description: `Quoted the #-leading value of "${w.key}" at line ${w.line} so it is no longer read as a comment` });
    }
  }

  return { content: working, fixes };
}

// ---------------------------------------------------------------------------
// writeBrainPage — path-guarded write with centralized backup
// ---------------------------------------------------------------------------

export class BrainWriterError extends Error {
  code: string;
  hint?: string;
  constructor(code: string, message: string, hint?: string) {
    super(message);
    this.name = 'BrainWriterError';
    this.code = code;
    this.hint = hint;
  }
}

/**
 * Path-guarded brain page writer. Always writes a backup under
 * ~/.gbrain/backups/frontmatter/... before any in-place mutation (the contract
 * that replaces git-tree-clean for non-git brain repos). Throws
 * BrainWriterError if filePath is not under sourcePath.
 */
export function writeBrainPage(
  filePath: string,
  content: string,
  opts: { sourcePath: string; autoFix?: boolean; backupRoot?: string; backupRunId?: string },
): { fixes: AuditFix[]; backupPath?: string } {
  assertManagedFilesystemWrite(filePath);
  const resolvedSource = resolve(opts.sourcePath);
  const resolvedTarget = resolve(filePath);
  if (resolvedTarget !== resolvedSource && !resolvedTarget.startsWith(resolvedSource + '/')) {
    throw new BrainWriterError(
      'PATH_OUTSIDE_SOURCE',
      `writeBrainPage: ${filePath} is not under ${opts.sourcePath}`,
      'Pass --source <id> matching the source the file lives in.',
    );
  }

  let toWrite = content;
  let fixes: AuditFix[] = [];
  if (opts.autoFix) {
    const result = autoFixFrontmatter(content, { filePath: relative(resolvedSource, resolvedTarget) });
    toWrite = result.content;
    fixes = result.fixes;
  }

  let backupPath: string | undefined;
  if (existsSync(filePath)) {
    backupPath = createFrontmatterBackup(filePath, {
      sourcePath: opts.sourcePath,
      backupRoot: opts.backupRoot,
      runId: opts.backupRunId,
    });
  } else {
    mkdirSync(dirname(filePath), { recursive: true });
  }
  writeFileSync(filePath, toWrite, 'utf8');
  return { fixes, backupPath };
}

// ---------------------------------------------------------------------------
// scanBrainSources
// ---------------------------------------------------------------------------

interface SourceRow {
  id: string;
  local_path: string | null;
  kind: string | null;
}

export interface ScanOpts {
  /** Limit scan to one source. When omitted, all registered sources with a
   *  local_path are scanned. */
  sourceId?: string;
  /** Missing frontmatter is optional metadata coverage for broad document
   * sources. Set true for curated page repos that require every file to carry
   * YAML frontmatter. */
  strictMissingOpen?: boolean;
  onProgress?: ProgressReporter;
  /** Outer-loop abort: checked at source boundaries in scanBrainSources.
   * Does NOT interrupt the synchronous file-walk inside a single source —
   * use `deadline` for that. */
  signal?: AbortSignal;
  /** Hard wall-clock bound for the synchronous walk. When set, scanOneSource
   * checks `Date.now() > deadline` before parsing each file and returns
   * partial state when exceeded.
   *
   * This is the load-bearing mid-walk interruption mechanism. AbortSignal.timeout
   * cannot interrupt sync readdirSync/lstatSync/readFileSync (event loop blocked),
   * so a deadline epoch-ms check is the only way to actually bound wall-clock
   * inside the visit loop. Worst-case overshoot: one file's parse time. */
  deadline?: number;
  /** Async per-source DB hook: doctor uses this to fetch a coarse denominator
   * (`SELECT COUNT(*) FROM pages WHERE source_id = $1 AND deleted_at IS NULL`)
   * for the "scanned ~N of M" partial message. Returns null on query failure;
   * scanBrainSources stamps the result on PerSourceReport.db_page_count.
   * Optional — when omitted, db_page_count stays undefined. */
  dbPageCountForSource?: (sourceId: string) => Promise<number | null>;
  /** Test seam — fired by walkDir once per directory it descends into
   * (post-pruneDir). Production callers don't pass this; the regression
   * suite uses it to assert descent-time pruning directly. */
  visitDir?: (dirPath: string) => void;
  /** Parse only each file's frontmatter head (`frontmatterScanHead`) when that
   * provably yields the same findings; the doctor scan sets it. */
  frontmatterOnly?: boolean;
}

/**
 * The prefix of `content` through its closing frontmatter fence line, when
 * validating that prefix yields exactly the errors, warnings, recovery and
 * slug of the whole file; otherwise `content` itself. Proven only for the
 * plain shape every frontmatter reader agrees on: the file starts with a
 * `---` line (no BOM, blank lines or language selector), no line before the
 * first `---` close starts with `---`, the close is a bare `---` line after
 * at least one block line, and the file has no NUL byte (NULL_BYTES reads
 * the whole file). Everything after the close is body, which the findings
 * never read.
 */
export function frontmatterScanHead(content: string): string {
  if (!content.startsWith('---') || content.includes('\0')) return content;
  const first = content.indexOf('\n');
  if (first < 0 || (first !== 3 && content.slice(0, first) !== '---\r')) return content;
  for (let start = first + 1, line = 1; ; line++) {
    const end = content.indexOf('\n', start);
    if (end < 0) return content;
    const text = content.slice(start, end);
    if (text.trim() === '---') return line >= 2 && (text === '---' || text === '---\r') ? content.slice(0, end + 1) : content;
    if (text.startsWith('---')) return content;
    start = end + 1;
  }
}

/** Timeout-arm winner for the COUNT-vs-deadline race in scanBrainSources.
 *  A unique object so it can never collide with a legitimate COUNT result
 *  (number | null). Module-private. */
const DEADLINE_SENTINEL: unique symbol = Symbol('gbrain.scan.deadline');

export async function scanBrainSources(
  engine: BrainEngine,
  opts: ScanOpts = {},
): Promise<AuditReport> {
  const sources = await listSources(engine, opts.sourceId);
  const metafilePages = await readMetafilePagesMode(engine);
  const totals: Partial<Record<ParseValidationCode, number>> = {};
  const recoverabilityTotals: RecoverabilityByCode = {};
  const perSource: PerSourceReport[] = [];
  let grandTotal = 0;
  let ignoredMissingOpen = 0;
  let abortedAtSource: string | null = null;

  // Helper: mark sources from index i onward as 'skipped'. Used at every
  // between-source abort point (top of loop AND after the COUNT await).
  const markRemainingSkipped = (startIdx: number) => {
    for (let j = startIdx; j < sources.length; j++) {
      const skipped = sources[j];
      if (!skipped.local_path) continue;
      perSource.push({
        source_id: skipped.id,
        source_path: skipped.local_path,
        total: 0,
        errors_by_code: {},
        sample: [],
        ignoredMissingOpen: 0,
        status: 'skipped',
        files_scanned: 0,
      });
    }
  };

  for (let i = 0; i < sources.length; i++) {
    const src = sources[i];
    // Between-source abort check: AbortSignal works here (no sync I/O blocking
    // the event loop at this boundary). For mid-walk interruption use deadline.
    // `>=` not `>`: when Date.now() lands exactly on the deadline boundary,
    // the budget IS exhausted — proceeding to scanOneSource would let the
    // next source eat its own budget on top of what's already been spent.
    // (Strict `>` was a real flake source on CI: GitHub Actions runners
    // landing on the boundary exactly during a Promise.race timeout.)
    if (opts.signal?.aborted || (opts.deadline && Date.now() >= opts.deadline)) {
      // Codex adversarial review #3: when deadline fires BETWEEN sources,
      // also stamp aborted_at_source with the source we were about to start.
      // Pre-fix, the doctor message said "PARTIAL SCAN" with no source name.
      if (abortedAtSource === null && src.local_path) {
        abortedAtSource = src.id;
      }
      markRemainingSkipped(i);
      break;
    }
    if (!src.local_path) continue;
    if (!existsSync(src.local_path)) {
      // #5673: a connector syncs from its provider, so a gone local_path is a leftover pointer with nothing on disk to scan.
      if (isConnectorSourceKind(src.kind)) continue;
      perSource.push({
        source_id: src.id,
        source_path: src.local_path,
        total: 0,
        errors_by_code: {},
        sample: [],
        ignoredMissingOpen: 0,
        status: 'missing',
        files_scanned: 0,
      });
      continue;
    }

    // Best-effort denominator fetch — degrades gracefully on query failure.
    // Codex adversarial #4: also race against the deadline. A wedged Postgres
    // pool can make this await hang past the budget. Without the race, we'd
    // wait indefinitely AND defeat the wall-clock guarantee.
    let dbPageCount: number | null = null;
    // Set when the deadline race's timeout arm wins: the verdict that the
    // budget is spent, independent of any later Date.now() reading. Timer
    // callbacks on loaded runners can fire measurably EARLY relative to the
    // wall clock (a +1ms pad was drifted past in practice — see the flake
    // lineage in test/brain-writer-partial-scan.test.ts and issue #2946), so
    // the hung-COUNT path must not re-derive "did the deadline fire?" from
    // the clock the timer just raced against.
    let deadlineHit = false;
    if (opts.dbPageCountForSource) {
      try {
        if (opts.deadline) {
          const remainingMs = opts.deadline - Date.now();
          if (remainingMs <= 0) {
            dbPageCount = null;
            deadlineHit = true;
          } else {
            // Race COUNT against the deadline so a hung query can't eat the
            // budget. The timeout arm resolves a private sentinel — NOT null —
            // so a deadline win is distinguishable from a COUNT that resolved
            // null (failed/absent count keeps its existing semantics).
            const raced = await Promise.race([
              opts.dbPageCountForSource(src.id),
              new Promise<typeof DEADLINE_SENTINEL>(resolve =>
                setTimeout(() => resolve(DEADLINE_SENTINEL), remainingMs)),
            ]);
            if (raced === DEADLINE_SENTINEL) {
              dbPageCount = null;
              deadlineHit = true;
            } else {
              dbPageCount = raced;
            }
          }
        } else {
          dbPageCount = await opts.dbPageCountForSource(src.id);
        }
      } catch {
        // A throwing COUNT is a failed count, not a deadline verdict.
        dbPageCount = null;
      }
    }

    // Codex adversarial #2: re-check deadline AFTER the COUNT await. If the
    // await ate the budget, we must NOT call scanOneSource — it would return
    // status='partial' with files_scanned=0, which is misleading ("partial
    // scan" when actually nothing was scanned). Mark this source + remainder
    // as 'skipped' so the doctor message is honest.
    // `deadlineHit` is the authoritative verdict for the hung-COUNT path (the
    // sentinel above); the wall-clock re-check (`>=`, matching the
    // between-source check at line ~445) still covers a COUNT that RESOLVED
    // slowly enough to eat the budget without the timer winning.
    if (opts.signal?.aborted || deadlineHit || (opts.deadline && Date.now() >= opts.deadline)) {
      if (abortedAtSource === null) {
        abortedAtSource = src.id;
      }
      markRemainingSkipped(i);
      break;
    }

    const report = scanOneSource(src.id, src.local_path, opts, metafilePages);
    report.db_page_count = dbPageCount;
    perSource.push(report);
    grandTotal += report.total;
    ignoredMissingOpen += report.ignoredMissingOpen;
    for (const [code, n] of Object.entries(report.errors_by_code)) {
      const k = code as ParseValidationCode;
      totals[k] = (totals[k] ?? 0) + (n as number);
    }
    for (const [code, split] of Object.entries(report.recoverability_by_code ?? {})) {
      const into = recoverabilityTotals[code as ParseValidationCode] ??= { recoverable: 0, unrecoverable: 0 };
      into.recoverable += split.recoverable;
      into.unrecoverable += split.unrecoverable;
    }
    if (report.status === 'partial' && abortedAtSource === null) {
      abortedAtSource = src.id;
    }
  }

  const hasPartialOrSkipped = perSource.some(r => r.status === 'partial' || r.status === 'skipped');
  const missingSources = perSource.filter(r => r.status === 'missing').map(r => r.source_id);

  return {
    // Partial scans can never be 'ok' even when the scanned prefix is clean
    // (codex outside-voice C2 — a clean prefix doesn't speak for the
    // unscanned suffix).
    ok: grandTotal === 0 && !hasPartialOrSkipped && missingSources.length === 0,
    total: grandTotal,
    errors_by_code: totals,
    recoverability_by_code: recoverabilityTotals,
    per_source: perSource,
    scanned_at: new Date().toISOString(),
    ignored_missing_open: ignoredMissingOpen || undefined,
    partial: hasPartialOrSkipped,
    ...(missingSources.length ? { missing_sources: missingSources } : {}),
    aborted_at_source: abortedAtSource,
  };
}

function scanOneSource(
  sourceId: string,
  sourcePath: string,
  opts: ScanOpts,
  metafilePages: MetafilePagesMode,
): PerSourceReport {
  const errorsByCode: Partial<Record<ParseValidationCode, number>> = {};
  const recoverability: RecoverabilityByCode = {};
  const sample: PerSourceReport['sample'] = [];
  const repairable: RepairableReport = { files: 0, by_code: {}, sample: [] };
  const rootResolved = resolve(sourcePath);
  const scannable = frontmatterScannableUnder(rootResolved, metafilePages);
  let scanned = 0;
  let total = 0;
  let ignoredMissingOpen = 0;
  let interrupted = false;

  const visitFile = (absPath: string): boolean | void => {
    // Per-file deadline + abort gate. Deadline is the load-bearing
    // wall-clock bound (sync I/O blocks the event loop so timer-based
    // AbortSignal.timeout can't fire mid-walk — codex C1).
    if (opts.deadline && Date.now() > opts.deadline) {
      interrupted = true;
      return false;
    }
    if (opts.signal?.aborted) {
      interrupted = true;
      return false;
    }
    // visitDir is consulted from walkDir directly (passed below). The
    // per-file visit closure doesn't need it.
    const relPath = relative(rootResolved, absPath);
    // Frontmatter is a Markdown-only contract. Multimodal sync deliberately
    // admits images under the markdown strategy, but parsing JPEG/PNG bytes as
    // UTF-8 turns ordinary binary NULs into false NULL_BYTES findings.
    if (!scannable(relPath)) return true;
    scanned++;
    let content: string;
    try {
      content = readFileSync(absPath, 'utf8');
    } catch {
      return true; // skip unreadable
    }
    const expectedSlug = slugifyPath(relPath);
    const parsed = parseMarkdown(opts.frontmatterOnly ? frontmatterScanHead(content) : content, relPath, { validate: true, expectedSlug });
    const errs = (parsed.errors ?? []).filter((e) => {
      if (e.code !== 'MISSING_OPEN') return true;
      if (opts.strictMissingOpen) return true;
      ignoredMissingOpen++;
      return false;
    });
    const hold = classifyImportHold(parsed, { expectedSlug });
    const causes = new Set<RepairableCause>();
    for (const e of parsed.errors ?? []) {
      if (e.code === 'YAML_PARSE' && e.recoverable) causes.add('YAML_PARSE');
      else if (REPAIRABLE_ERRORS.has(e.code)) causes.add(e.code as RepairableCause);
    }
    if (hold?.reason === 'needs_interpretation') causes.add('needs_interpretation');
    if ((parsed.warnings ?? []).some(w => w.code === 'FRONTMATTER_COMMENT_VALUE' && RESCUE_KEYS.has(w.key))) causes.add('FRONTMATTER_COMMENT_VALUE');
    if (causes.size > 0) {
      repairable.files++;
      for (const cause of causes) repairable.by_code[cause] = (repairable.by_code[cause] ?? 0) + 1;
      if (repairable.sample.length < SAMPLE_PER_SOURCE) repairable.sample.push(relPath);
    }
    if (errs.length > 0) {
      total += errs.length;
      const codes: ParseValidationCode[] = [];
      for (const e of errs) {
        errorsByCode[e.code] = (errorsByCode[e.code] ?? 0) + 1;
        codes.push(e.code);
        const held = e.code === 'YAML_PARSE' ? !e.recoverable
          : e.code === 'SLUG_MISMATCH' ? hold?.code === 'frontmatter_slug_conflict'
          : e.code === 'MISSING_CLOSE' && hold?.code === 'invalid_frontmatter';
        const split = recoverability[e.code] ??= { recoverable: 0, unrecoverable: 0 };
        if (held) split.unrecoverable++;
        else split.recoverable++;
      }
      if (sample.length < SAMPLE_PER_SOURCE) {
        sample.push({ path: relPath, codes });
      }
    }
    if (opts.onProgress && scanned % 50 === 0) {
      opts.onProgress.tick(50);
    }
    return true;
  };

  const gitFiles = collectGitVisibleFiles(rootResolved, scannable);
  if (gitFiles) {
    if (opts.visitDir) opts.visitDir(rootResolved);
    for (const absPath of gitFiles) {
      if (visitFile(absPath) === false) break;
    }
  } else {
    walkDir(rootResolved, visitFile, opts.visitDir);
  }

  if (opts.onProgress) {
    opts.onProgress.heartbeat(`scanned ${scanned} pages in ${sourceId}`);
  }

  return {
    source_id: sourceId,
    source_path: sourcePath,
    total,
    errors_by_code: errorsByCode,
    recoverability_by_code: recoverability,
    repairable,
    sample,
    ignoredMissingOpen,
    status: interrupted ? 'partial' : 'scanned',
    files_scanned: scanned,
  };
}

/**
 * Recursive directory walker with symlink-loop protection (via lstat) and
 * descent-time pruning of vendor / hidden / generated subtrees.
 *
 * Walks `root`, calling `visit(absPath)` for each regular file. Returning
 * `false` from `visit` stops the walk (used by scanOneSource for deadline
 * + abort).
 *
 * `pruneDir(name, parentDir)` is the canonical pruning gate (single source
 * of truth at src/core/sync.ts:258 — sync, extract, and transcript-discovery
 * all share it). Skipped subtrees: `node_modules`, `.git`, `.obsidian`,
 * `*.raw`, `ops`, all dot-prefix dirs, and git submodule dirs (`.git` as
 * FILE not DIRECTORY, the gitfile pattern from v0.37.7.0 #1169).
 *
 * Pre-v0.38.2.0 this walker descended into every subtree and let
 * `isSyncable` filter at the leaf — paying the IO cost of stat'ing hundreds
 * of thousands of vendor entries that were never going to be parsed. That
 * was the root cause of the `gbrain doctor` hang on 216K-page brains
 * reported in PR #1287.
 *
 * Optional `visitDir(dir)` is fired once per directory the walker decides
 * to DESCEND INTO (post-pruneDir, post-visited-set check). Production
 * callers don't pass it; tests use it to assert descent-time pruning
 * directly. Output-based tests pass under the original bug because
 * `isSyncable` filters at the leaf — `visitDir` is the load-bearing
 * observability hook for the regression suite.
 */
/** @internal — exported for the regression test suite (visitDir-based
 * descent-time pruning assertions). Production callers should go through
 * scanBrainSources / scanOneSource. */
export function walkDir(
  root: string,
  visit: (absPath: string) => boolean | void,
  visitDir?: (dirPath: string) => void,
): void {
  const stack: string[] = [root];
  const visited = new Set<string>();
  if (visitDir) visitDir(root);
  while (stack.length > 0) {
    const dir = stack.pop()!;
    let entries: string[];
    try {
      entries = readdirSync(dir);
    } catch {
      continue;
    }
    for (const name of entries) {
      const full = join(dir, name);
      let st: ReturnType<typeof lstatSync>;
      try {
        st = lstatSync(full);
      } catch {
        continue;
      }
      if (st.isSymbolicLink()) continue; // matches sync's no-symlink policy
      if (st.isDirectory()) {
        // Descent-time prune — the actual fix for the 216K-page hang.
        if (!pruneDir(name, dir)) continue;
        const real = resolve(full);
        if (visited.has(real)) continue;
        visited.add(real);
        if (visitDir) visitDir(full);
        stack.push(full);
      } else if (st.isFile()) {
        const result = visit(full);
        if (result === false) return;
      }
    }
  }
}

async function listSources(engine: BrainEngine, sourceId?: string): Promise<SourceRow[]> {
  if (sourceId) {
    const rows = await engine.executeRaw<SourceRow>(
      `SELECT id, local_path, config->>'kind' AS kind FROM sources WHERE id = $1`,
      [sourceId],
    );
    return rows;
  }
  // #3880: archived sources are excluded from automatic all-source scanning.
  // The archived column is v34+ — fall back on older brains (house style per
  // pickSoleNonDefaultSource).
  try {
    return await engine.executeRaw<SourceRow>(
      `SELECT id, local_path, config->>'kind' AS kind FROM sources WHERE local_path IS NOT NULL AND archived IS NOT TRUE ORDER BY id`,
    );
  } catch {
    return engine.executeRaw<SourceRow>(
      `SELECT id, local_path, config->>'kind' AS kind FROM sources WHERE local_path IS NOT NULL ORDER BY id`,
    );
  }
}