Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
41.4 KB
Raw Blame
/**
 * Async, chunked Git level reads for the managed-sync reconcile (#6712): the
 * committed tree (`T`), the index (`X`), exact-OID rename hints, blob contents
 * and per-path dirtiness (`Dp`, a batched `hardenedPathDirty`). Every call goes
 * through `hardenedGitStream`, so only allow-listed, filter-free plumbing runs
 * and the checkout's clean/process filters and textconv never execute. Output
 * is parsed as it streams; nothing but the result maps is held, so a
 * 600k-path tree costs its map and one chunk.
 */
import { randomUUID } from 'node:crypto';
import { lstatSync, mkdirSync, readFileSync, realpathSync, renameSync, rmSync, writeFileSync, type BigIntStats } from 'node:fs';
import { join, relative, resolve, sep } from 'node:path';
import { configDir } from '../config.ts';
import { hardenedGitStream, hardenedStreamComplete, NulRecordSplitter, type HardenedGitStreamResult } from '../hardened-git.ts';
import { gitBlobOid, sha256 } from './digest.ts';
import { readSyncFile } from './sync-discovery.ts';
import { faultPointNow } from './fault-points.ts';
import { loopYield, type LoopYield } from './sync-yield.ts';

export type Dp = 'clean' | 'clean_eol' | 'clean_lfs' | 'modified' | 'deleted' | 'staged' | 'untracked' | 'unknown';
export interface GitTreeEntry { mode: string; oid: string }
export interface GitIndexEntry extends GitTreeEntry { stage: number }
export interface GitRenameHint { from: string; to: string; oid: string }
export interface DirtinessProbe { dp: Dp; rawOid: string | null; rawSha256: string | null }

/** A level read that did not satisfy the completeness rule (exit 0, no signal, output ended at a NUL), so its absences prove nothing. */
export class SyncTreeReadIncomplete extends Error {
  readonly code = 'reconcile_read_incomplete';
  constructor(read: string, detail: string) {
    super(`git ${read} did not finish a complete read (${detail}); nothing it omitted may be treated as absent`);
    this.name = 'SyncTreeReadIncomplete';
  }
}

const READ_TIMEOUT_MS = 120_000;
/** GST-2515: held paths per history read, so one read of a path's specs stays well inside READ_TIMEOUT_MS on a deep history. */
const HISTORY_GROUP_PATHS = 2_000;
const PROBE_TIMEOUT_MS = 60_000;
const LFS_POINTER_MAX_BYTES = 1024;
const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
const REGULAR_MODES = new Set(['100644', '100755']);

let gitStream = hardenedGitStream;
/** Replaces the Git runner (tests only: a truncated or signalled stream); no argument restores it. */
export function __setSyncGitStreamForTests(runner?: typeof hardenedGitStream): void { gitStream = runner ?? hardenedGitStream; }

function outcome(result: HardenedGitStreamResult): string {
  if (!result.ok) return `${result.reason}${result.status === undefined ? '' : ` ${result.status}`}${result.signal ? ` ${result.signal}` : ''}`;
  return result.endsWithNul || result.bytes === 0 ? 'complete' : 'output did not end at a NUL';
}

const malformed = (read: string, record: string) => new SyncTreeReadIncomplete(read, `malformed record ${JSON.stringify(record.slice(0, 80))}`);

async function readRecords(gitRoot: string, args: string[], read: string, visit: (record: Buffer) => void, shared?: LoopYield): Promise<void> {
  const splitter = new NulRecordSplitter();
  // A 600k-path read arrives faster than it parses: past the stretch budget, the stream waits one macrotask.
  const step = shared ?? loopYield();
  const visitChunk = (chunk: Buffer) => {
    if (shared) faultPointNow('sync:level_chunk', { read });
    const records = splitter.push(chunk);
    for (const record of records) visit(record);
    return step.due(records.length) ? step.pause() : undefined;
  };
  // A budget shared with the level's cursor (sync-yield.ts `turnYield`): a chunk waits a turn the cursor already spent.
  const take = (chunk: Buffer) => shared?.over() ? shared.pause().then(() => visitChunk(chunk)) : visitChunk(chunk);
  // Bun can close the stream (and hand the next chunk) before a held chunk settles: chunks chain here and the read waits for the last.
  let tail: Promise<void> | undefined;
  const result = await gitStream(gitRoot, args, { timeoutMs: READ_TIMEOUT_MS, onChunk: chunk => {
    const held = tail ? tail.then(() => take(chunk)) : take(chunk);
    if (!held) return;
    const settled: Promise<void> = held.finally(() => { if (tail === settled) tail = undefined; });
    return tail = settled;
  } });
  await tail;
  if (!hardenedStreamComplete(result) || !splitter.atRecordBoundary) throw new SyncTreeReadIncomplete(read, outcome(result));
}

/** The common modes and `ls-tree` types as one string each, so a 600k-entry map holds no per-entry copy and a read decodes none. */
const MODE_NAMES = ['100644', '100755', '120000', '160000', '040000'], TYPE_NAMES = ['blob', 'commit', 'tree'];

/** `record.toString('latin1', start, end)`, as the one string of `names` it spells when it spells one. */
function latin1Named(record: Buffer, start: number, end: number, names: readonly string[]): string {
  next: for (const name of names) {
    if (name.length !== end - start) continue;
    for (let i = 0; i < name.length; i++) if (record[start + i] !== name.charCodeAt(i)) continue next;
    return name;
  }
  return record.toString('latin1', start, end);
}

/** Whether an `ls-files -s` record (its first tab at `tab`) is exactly `<mode> <oid> 0` of `entry`: the parse it skips reads stage 0 at the entry's blob and mode. */
function sameIndexRecord(record: Buffer, tab: number, entry: GitTreeEntry): boolean {
  const { mode, oid } = entry;
  if (tab !== mode.length + oid.length + 3 || record[mode.length] !== 32 || record[tab - 2] !== 32 || record[tab - 1] !== 48) return false;
  for (let i = 0; i < mode.length; i++) if (record[i] !== mode.charCodeAt(i)) return false;
  for (let i = 0; i < oid.length; i++) if (record[mode.length + 1 + i] !== oid.charCodeAt(i)) return false;
  return true;
}

/**
 * One `<mode> <a> <b>\t<path>` record (`ls-tree`: type and OID; `ls-files -s`: OID and stage), its fields decoded straight
 * from the record's bytes as strings of their own (no substring keeps the record alive), or null when the header before the
 * tab has fewer than three fields (a fourth is ignored, as `split(' ')` did).
 */
function headerFields(record: Buffer): { mode: string; a: string; b: string; path: string } | null {
  const tab = record.indexOf(9);
  const first = tab < 0 ? -1 : record.indexOf(32);
  const second = first < 0 || first > tab ? -1 : record.indexOf(32, first + 1);
  if (second < 0 || second > tab) return null;
  const third = record.indexOf(32, second + 1);
  return { mode: latin1Named(record, 0, first, MODE_NAMES), a: latin1Named(record, first + 1, second, TYPE_NAMES), b: record.toString('latin1', second + 1, third < 0 || third > tab ? tab : third),
    path: record.toString('utf8', tab + 1) };
}

function* lines(items: readonly string[], terminator: string): Generator<Buffer> {
  for (let i = 0; i < items.length; i += 1000) yield Buffer.from(items.slice(i, i + 1000).map(item => item + terminator).join(''));
}

/**
 * `T`: blob entries of `commit` under `scope` (a Git-root-relative directory
 * prefix; empty lists the whole tree), keyed by Git-root-relative path.
 * Symlinks (120000) and gitlinks (160000) are kept with their mode so a caller
 * can mark them unknown.
 */
export async function readTreeLevel(gitRoot: string, commit: string, scope: string, shared?: LoopYield): Promise<Map<string, GitTreeEntry>> {
  const tree = new Map<string, GitTreeEntry>();
  await walkTreeLevel(gitRoot, commit, scope, (path, mode, oid) => tree.set(path, { mode, oid }), shared);
  return tree;
}

/** `readTreeLevel`'s entries handed to `visit` in `ls-tree` order as they arrive, none kept: a caller that needs only a pass over `T` holds no map of it. */
export async function walkTreeLevel(gitRoot: string, commit: string, scope: string, visit: (path: string, mode: string, oid: string) => void, shared?: LoopYield): Promise<void> {
  await readRecords(gitRoot, ['ls-tree', '-r', '-z', commit, '--', scope === '' ? '.' : scope], 'ls-tree', record => {
    const fields = headerFields(record);
    if (!fields || !OID.test(fields.b)) throw malformed('ls-tree', record.toString('utf8'));
    if (fields.a === 'blob' || fields.a === 'commit') visit(fields.path, fields.mode, fields.b);
  }, shared);
}

/** One `diff-tree -r` change of a `T` path: its entry at the new commit, or null where the new commit has none. */
export interface GitTreeChange { path: string; entry: GitTreeEntry | null }
/** The modes `readTreeLevel` keeps (blob and gitlink entries); a change to anything else is not a `T` change it can apply. */
const TREE_ENTRY_MODE = /^(?:10[0-7]{4}|120000|160000)$/;

/**
 * The changes of `T` under `scope` from commit `from` to `to` (one `diff-tree -r -z --no-renames`, raw: full OIDs),
 * or null when a record names an entry `readTreeLevel` would not list the same way: the caller reads `T` whole then.
 */
export async function readTreeChanges(gitRoot: string, from: string, to: string, scope: string, shared?: LoopYield): Promise<GitTreeChange[] | null> {
  const changes: GitTreeChange[] = [];
  let meta: string | null = null, foreign = false;
  await readRecords(gitRoot, ['diff-tree', '-r', '-z', '--no-renames', from, to, ...(scope ? ['--', scope] : [])], 'diff-tree', bytes => {
    if (meta === null) {
      meta = bytes.toString('latin1');
      if (!meta.startsWith(':')) throw malformed('diff-tree', meta);
      return;
    }
    const [, dstMode = '', , dstOid = '', status = ''] = meta.slice(1).split(' ');
    meta = null;
    const path = bytes.toString('utf8');
    if (status === 'D') changes.push({ path, entry: null });
    else if ((status === 'A' || status === 'M' || status === 'T') && TREE_ENTRY_MODE.test(dstMode) && OID.test(dstOid)) changes.push({ path, entry: { mode: dstMode, oid: dstOid } });
    else foreign = true;
  }, shared);
  if (meta !== null) throw new SyncTreeReadIncomplete('diff-tree', 'a status record had no path');
  return foreign ? null : changes;
}

/** `X`: the index (never the worktree), keyed by Git-root-relative path; a conflicted path keeps its highest stage. */
export async function readIndexLevel(gitRoot: string, shared?: LoopYield): Promise<Map<string, GitIndexEntry>> {
  const index = new Map<string, GitIndexEntry>();
  await readRecords(gitRoot, ['ls-files', '-s', '-z'], 'ls-files', record => {
    const fields = headerFields(record);
    if (!fields || !OID.test(fields.a) || !/^[0-3]$/.test(fields.b)) throw malformed('ls-files', record.toString('utf8'));
    const stage = Number(fields.b);
    if ((index.get(fields.path)?.stage ?? -1) < stage) index.set(fields.path, { mode: fields.mode, oid: fields.a, stage });
  }, shared);
  return index;
}

/**
 * `X` as committed mode reads it: only the tree paths whose index entry differs from `T` (OID, mode or a non-zero stage,
 * the highest stage kept; null: absent from the index). A 600k-path checkout whose index matches its tree holds no second
 * map of every path. Paths outside `tree` are not kept; when some tree path never appeared, the index is read whole
 * once more to name them.
 */
export async function readIndexDelta(gitRoot: string, tree: Map<string, GitTreeEntry>, shared?: LoopYield): Promise<Map<string, GitIndexEntry | null>> {
  const delta = new Map<string, GitIndexEntry | null>();
  let same = 0;
  await readRecords(gitRoot, ['ls-files', '-s', '-z'], 'ls-files', record => {
    const tab = record.indexOf(9);
    const matched = tab < 0 ? undefined : tree.get(record.toString('utf8', tab + 1));
    if (matched && sameIndexRecord(record, tab, matched)) { same++; return; }
    const fields = headerFields(record);
    if (!fields || !OID.test(fields.a) || !/^[0-3]$/.test(fields.b)) throw malformed('ls-files', record.toString('utf8'));
    const committed = tree.get(fields.path);
    if (!committed) return;
    const stage = Number(fields.b);
    if (stage === 0 && fields.a === committed.oid && fields.mode === committed.mode) { same++; return; }
    if ((delta.get(fields.path)?.stage ?? -1) < stage) delta.set(fields.path, { mode: fields.mode, oid: fields.a, stage });
  }, shared);
  if (same + delta.size === tree.size) return delta;
  const index = await readIndexLevel(gitRoot, shared);
  const whole = new Map<string, GitIndexEntry | null>();
  for (const [path, committed] of tree) {
    const indexed = index.get(path);
    if (!indexed) whole.set(path, null);
    else if (indexed.stage !== 0 || indexed.oid !== committed.oid || indexed.mode !== committed.mode) whole.set(path, indexed);
  }
  return whole;
}

/**
 * Exact-OID rename hints from `from` to `to`: a deleted and an added regular
 * file with the same blob, only when that OID has exactly one of each. Hints
 * only; never a source of candidates.
 */
export async function readRenameHints(gitRoot: string, from: string, to: string): Promise<GitRenameHint[]> {
  const byOid = new Map<string, { deleted: string[]; added: string[] }>();
  let meta: string | null = null;
  await readRecords(gitRoot, ['diff-tree', '-r', '-z', '--no-renames', from, to], 'diff-tree', bytes => {
    const record = bytes.toString('utf8');
    if (meta === null) {
      if (!record.startsWith(':')) throw malformed('diff-tree', record);
      meta = record;
      return;
    }
    const [srcMode = '', dstMode = '', srcOid = '', dstOid = '', status = ''] = meta.slice(1).split(' ');
    meta = null;
    const side = status === 'D' && REGULAR_MODES.has(srcMode) ? { oid: srcOid, key: 'deleted' as const }
      : status === 'A' && REGULAR_MODES.has(dstMode) ? { oid: dstOid, key: 'added' as const } : null;
    if (!side) return;
    const entry = byOid.get(side.oid) ?? { deleted: [], added: [] };
    entry[side.key].push(record);
    byOid.set(side.oid, entry);
  });
  if (meta !== null) throw new SyncTreeReadIncomplete('diff-tree', 'a status record had no path');
  const hints: GitRenameHint[] = [];
  for (const [oid, { deleted, added }] of byOid) {
    if (deleted.length === 1 && added.length === 1) hints.push({ from: deleted[0]!, to: added[0]!, oid });
  }
  return hints.sort((a, b) => (a.to < b.to ? -1 : a.to > b.to ? 1 : 0));
}

/**
 * #7199, GST-2515: the blob ids each path had at `commit` and its first `depth - 1` first-parent ancestors, newest first, one
 * per commit that has the path. One `cat-file --batch-check` per group of paths answers every `<commit>~k:<path>` spec (no
 * bodies are read); `onPath` runs as each path's last answer arrives, and a throw from it (an abort) kills the read and rejects.
 * A group whose read is incomplete leaves its unanswered paths out of the map; a path a spec line cannot carry has no entry.
 */
export async function pathBlobHistories(gitRoot: string, commit: string, paths: readonly string[], depth: number,
  onPath?: (path: string) => void): Promise<Map<string, string[]>> {
  const histories = new Map<string, string[]>();
  if (!OID.test(commit) || depth < 1) return histories;
  const carried = paths.filter(path => !/[\n\r]/.test(path));
  for (let start = 0; start < carried.length; start += HISTORY_GROUP_PATHS) {
    const group = carried.slice(start, start + HISTORY_GROUP_PATHS);
    let header: Buffer[] = [], answered = 0, oids: string[] = [];
    await gitStream(gitRoot, ['cat-file', '--batch-check'], { timeoutMs: READ_TIMEOUT_MS,
      stdin: lines(group.flatMap(path => Array.from({ length: depth }, (_, k) => `${commit}~${k}:${path}`)), '\n'), onChunk: chunk => {
        let at = 0;
        while (at < chunk.length) {
          const newline = chunk.indexOf(10, at);
          if (newline < 0) { header.push(chunk.subarray(at)); break; }
          const line = Buffer.concat([...header, chunk.subarray(at, newline)]).toString('utf8');
          header = []; at = newline + 1;
          if (!line.endsWith(' missing') && !line.endsWith(' ambiguous')) {
            const [oid = '', type = '', size = ''] = line.split(' ');
            if (!OID.test(oid) || !/^\d+$/.test(size)) throw malformed('cat-file', line);
            if (type === 'blob') oids.push(oid);
          }
          if (++answered % depth) continue;
          const path = group[answered / depth - 1]!;
          histories.set(path, oids);
          oids = [];
          onPath?.(path);
        }
      } }).catch((error: unknown) => { if (!(error instanceof SyncTreeReadIncomplete)) throw error; });
  }
  return histories;
}

/**
 * Blob contents by OID through one `cat-file --batch`. A blob over `maxBytes`,
 * a non-blob, a missing object, or one that would pass `maxTotalBytes` is left
 * out of the map (its bytes are still consumed); a Git failure or a short read
 * throws `SyncTreeReadIncomplete`.
 */
export async function readBlobs(gitRoot: string, oids: string[], maxBytes: number, maxTotalBytes = 64 * 1024 * 1024): Promise<Map<string, Buffer>> {
  const blobs = new Map<string, Buffer>();
  if (oids.length === 0) return blobs;
  const bad = oids.find(oid => !OID.test(oid));
  if (bad !== undefined) throw new Error(`readBlobs takes full hexadecimal object ids, not ${JSON.stringify(bad)}`);
  let header: Buffer[] = [];
  let body: { oid: string; keep: boolean; remaining: number; parts: Buffer[] } | null = null;
  let answered = 0;
  let stored = 0;
  const result = await gitStream(gitRoot, ['cat-file', '--batch'], { timeoutMs: READ_TIMEOUT_MS, stdin: lines(oids, '\n'), onChunk: chunk => {
    let at = 0;
    while (at < chunk.length) {
      if (body) {
        const take = Math.min(body.remaining, chunk.length - at);
        if (body.keep) body.parts.push(chunk.subarray(at, at + take));
        body.remaining -= take;
        at += take;
        if (body.remaining > 0) continue;
        if (chunk[at - 1] !== 10) throw new SyncTreeReadIncomplete('cat-file', `object ${body.oid} did not end with a newline`);
        if (body.keep) { const all = Buffer.concat(body.parts); blobs.set(body.oid, all.subarray(0, all.length - 1)); }
        body = null;
        continue;
      }
      const newline = chunk.indexOf(10, at);
      if (newline < 0) { header.push(chunk.subarray(at)); break; }
      const line = Buffer.concat([...header, chunk.subarray(at, newline)]).toString('utf8');
      header = [];
      at = newline + 1;
      answered += 1;
      const [oid = '', type = '', size = ''] = line.split(' ');
      if (type === 'missing' || type === 'ambiguous') continue;
      if (!/^\d+$/.test(size)) throw malformed('cat-file', line);
      const keep = type === 'blob' && Number(size) <= maxBytes && stored + Number(size) <= maxTotalBytes;
      if (keep) stored += Number(size);
      body = { oid, keep, remaining: Number(size) + 1, parts: [] };
    }
  } });
  if (!result.ok || body !== null || header.length > 0 || answered !== oids.length) {
    throw new SyncTreeReadIncomplete('cat-file', result.ok ? `${answered} of ${oids.length} objects answered` : outcome(result));
  }
  return blobs;
}

function foldCrlf(bytes: Buffer): Buffer {
  const folded = Buffer.allocUnsafe(bytes.length);
  let length = 0;
  for (let i = 0; i < bytes.length; i++) {
    if (bytes[i] === 13 && bytes[i + 1] === 10) continue;
    folded[length++] = bytes[i]!;
  }
  return folded.subarray(0, length);
}

function lfsPointer(blob: Buffer): { oid: string; size: number } | null {
  const text = blob.toString('utf8');
  if (!text.startsWith('version https://git-lfs.github.com/spec/v1\n') || !text.endsWith('\n')) return null;
  const fields = new Map(text.slice(0, -1).split('\n').map(line => [line.slice(0, line.indexOf(' ')), line.slice(line.indexOf(' ') + 1)] as const));
  const oid = /^sha256:([0-9a-f]{64})$/.exec(fields.get('oid') ?? '')?.[1];
  const size = fields.get('size') ?? '';
  return oid && /^\d+$/.test(size) ? { oid, size: Number(size) } : null;
}

const attributeSet = (value: string | undefined): boolean => value !== undefined && value !== 'unspecified' && value !== 'unset';

interface ReadFile { path: string; sha256: string; localOid: string; foldedOid: string; size: number; executable: boolean | null; statKey: string | null }

/**
 * `Dp` for each Git-root-relative path, first match wins: `untracked` (in
 * neither index nor tree), `staged` (index differs from tree in OID or mode,
 * a non-zero stage, or in only one of them), `deleted` (no file backs the path,
 * as `readSyncFile` reports it), `unknown` (a symlink or gitlink in the tree,
 * a file sync cannot read, a path `hash-object --stdin-paths` cannot carry, or
 * any Git failure for its batch), then from the raw bytes: `clean` (raw OID is
 * the tree OID with the executable bit matching), `clean_eol` (only
 * `text`/`eol` set and the CRLF-folded bytes hash to the tree OID),
 * `clean_lfs` (`filter=lfs`, the tree blob is a valid pointer whose sha256 and
 * size are the raw bytes'), else `modified`. Another filter, a
 * `working-tree-encoding`, an executable-bit-only difference, or CRLF bytes
 * that fold to the tree with no attribute (repository `core.autocrlf` may
 * apply) are `unknown`, which is never clean. Raw OIDs come from one
 * `hash-object --no-filters --stdin-paths` per batch and must agree with the
 * bytes read here; attributes from one `check-attr -z --stdin`. A present,
 * readable file also reports its raw OID and sha256 under `staged` and
 * `untracked`.
 *
 * `working` (design v3.2 §2.1, the working level is the physical bytes):
 * the index is not read. A path is `deleted` when no file backs it, whatever
 * the index says; a present file outside the tree is `untracked` with its raw
 * OID; every other present file is judged from its bytes against the tree,
 * through the same `clean_eol`/`clean_lfs` domains. So a staged edit, a staged
 * new file, `git rm --cached`, a staged deletion and a staged file removed from
 * disk probe exactly as the same disk does with a pristine index.
 *
 * Committed mode reuses a `clean` probe (#7338) where the path's lstat key and
 * its tree entry are the ones the probe was vouched under, and the index entry
 * is the tree's: a clean verdict depends on the raw bytes alone, so it needs no
 * read, hash or attribute lookup. A probe is vouched only when the stat taken
 * before the read is unchanged after the batch's hash and its `mtime` and
 * `ctime` are older than 2 s before the call began, by the earlier of the
 * process's and the filesystem's clocks (`racyFloor`, the working stat cache's
 * racy window), so an edit made during or after the read always rehashes.
 *
 * With `persist` (`sync.level_reuse`, GBRA-133 wave 4) the vouched probes also outlive the process: a process with none
 * for this root and Git root starts from the ones the last process saved (`cleanProbeFile`), each still checked against
 * the path's lstat key and tree entry, and saves its own when they changed (at most once a minute per root). A file that
 * is missing, unreadable, for another root or with any malformed entry is ignored whole. With `verify` every reused probe
 * is read and hashed anyway, and `verify` gets the reused count and the first path whose fresh probe differs.
 */
export async function probeDirtiness(root: string, gitRoot: string, paths: string[], tree: Map<string, GitTreeEntry>,
  index: Map<string, GitIndexEntry>, opts: { batch?: number; working?: boolean; step?: LoopYield; persist?: boolean; verify?: (reused: number, differs: string | null) => void } = {}): Promise<Map<string, DirtinessProbe>> {
  const batch = Math.max(1, opts.batch ?? 2000);
  const results = new Map<string, DirtinessProbe>();
  const firstOid = tree.values().next().value?.oid ?? index.values().next().value?.oid ?? '';
  const format = firstOid.length === 64 ? 'sha256' as const : 'sha1' as const;
  let realRoot: string | null = null;
  let realGitRoot: string | null = null;
  try { realRoot = realpathSync.native(root); realGitRoot = realpathSync.native(gitRoot); } catch {}
  const step = opts.step ?? loopYield();
  const cleanKey = !opts.working && realRoot !== null && realGitRoot !== null ? `${realRoot}\0${realGitRoot}` : null;
  const vouched = cleanKey === null ? null : new Map(cleanProbes.get(cleanKey) ?? (opts.persist ? loadCleanProbes(cleanKey) : undefined));
  const checked = new Map<string, DirtinessProbe>();
  let touched = false;
  const racyFrom = vouched ? racyFloor(realGitRoot!) : 0n;
  for (let start = 0; start < paths.length; start += batch) {
    const decided = new Map<string, Dp | null>();
    const read: ReadFile[] = [];
    for (const path of paths.slice(start, start + batch)) {
      // Each entry reads and hashes a file: the clock, not the entry counter, decides its stretch.
      if (step.over()) await step.pause();
      const committed = tree.get(path);
      const indexed = index.get(path);
      let dp: Dp | null = opts.working ? (committed ? null : 'untracked') : !committed && !indexed ? 'untracked'
        : !committed || !indexed || indexed.oid !== committed.oid || indexed.mode !== committed.mode || indexed.stage !== 0 ? 'staged' : null;
      let bytes: Buffer | null = null;
      let unreadable = realRoot === null || realGitRoot === null;
      const sourcePath = unreadable ? '' : relative(realRoot!, join(realGitRoot!, path)).split(sep).join('/');
      const statKey = vouched && dp === null && !unreadable ? statAt(realGitRoot!, path) : null;
      const prior = statKey && statKey !== ABSENT ? vouched!.get(path) : undefined;
      const signature = `${statKey}|${committed?.oid}|${committed?.mode}|`;
      if (prior?.startsWith(signature)) {
        const kept: DirtinessProbe = { dp: 'clean', rawOid: committed!.oid, rawSha256: prior.slice(signature.length) };
        if (!opts.verify) { results.set(path, kept); continue; }
        checked.set(path, kept);
      }
      if (vouched?.delete(path)) touched = true;
      if (!unreadable) { try { bytes = readSyncFile(root, sourcePath); } catch { unreadable = true; } }
      if ((dp === null || opts.working) && !unreadable && bytes === null) dp = 'deleted';
      const hashable = bytes !== null && !path.includes('\n') && !path.startsWith('"');
      if (dp === null && (committed!.mode === '120000' || committed!.mode === '160000' || unreadable || !hashable)) dp = 'unknown';
      decided.set(path, dp);
      if (bytes === null) { results.set(path, { dp: dp!, rawOid: null, rawSha256: null }); continue; }
      if (!hashable) { results.set(path, { dp: dp!, rawOid: null, rawSha256: sha256(bytes) }); continue; }
      let executable: boolean | null = null;
      try { executable = (lstatSync(join(realRoot!, ...sourcePath.split('/'))).mode & 0o111) !== 0; } catch {}
      const localOid = gitBlobOid(bytes, format);
      read.push({ path, sha256: sha256(bytes), localOid, size: bytes.length, executable, statKey: statKey === ABSENT ? null : statKey,
        foldedOid: bytes.includes('\r\n') ? gitBlobOid(foldCrlf(bytes), format) : localOid });
    }
    const undecided = read.filter(file => decided.get(file.path) === null).map(file => file.path);
    const [rawOids, attrs] = await Promise.all([hashRaw(gitRoot, read.map(file => file.path)), readAttributes(gitRoot, undecided)]);
    const lfs: Array<{ file: ReadFile; treeOid: string; executableMatches: boolean }> = [];
    for (const [i, file] of read.entries()) {
      if (step.due()) await step.pause();
      const rawOid = rawOids?.[i] === file.localOid ? file.localOid : null;
      const settle = (dp: Dp) => { results.set(file.path, { dp, rawOid, rawSha256: file.sha256 }); };
      const prior = decided.get(file.path);
      if (prior) { settle(prior); continue; }
      const committed = tree.get(file.path)!;
      const attr = attrs?.get(file.path);
      if (rawOid === null || attr === undefined || file.executable === null) { settle('unknown'); continue; }
      const executableMatches = file.executable === (committed.mode === '100755');
      if (rawOid === committed.oid) {
        settle(executableMatches ? 'clean' : 'unknown');
        if (executableMatches && file.statKey && vouched!.size < WORKING_STAT_CACHE_MAX_ENTRIES && statAt(realGitRoot!, file.path) === file.statKey
          && file.statKey.split(':').slice(5).every(ns => BigInt(ns) < racyFrom)) { vouched!.set(file.path, `${file.statKey}|${committed.oid}|${committed.mode}|${file.sha256}`); touched = true; }
        continue;
      }
      if (attributeSet(attr['working-tree-encoding']) || (attributeSet(attr.filter) && attr.filter !== 'lfs')) { settle('unknown'); continue; }
      if (attr.filter === 'lfs') { lfs.push({ file, treeOid: committed.oid, executableMatches }); continue; }
      const folds = file.foldedOid === committed.oid;
      if (attr.text !== 'unset' && (attributeSet(attr.text) || attributeSet(attr.eol))) settle(!folds ? 'modified' : executableMatches ? 'clean_eol' : 'unknown');
      else settle(folds ? 'unknown' : 'modified');
    }
    if (lfs.length === 0) continue;
    let pointers: Map<string, Buffer> | null = null;
    try { pointers = await readBlobs(gitRoot, [...new Set(lfs.map(entry => entry.treeOid))], LFS_POINTER_MAX_BYTES); } catch {}
    for (const { file, treeOid, executableMatches } of lfs) {
      const blob = pointers?.get(treeOid);
      const pointer = blob ? lfsPointer(blob) : null;
      const dp: Dp = !blob ? 'unknown' : !pointer || pointer.oid !== file.sha256 || pointer.size !== file.size ? 'modified'
        : executableMatches ? 'clean_lfs' : 'unknown';
      results.set(file.path, { dp, rawOid: file.localOid, rawSha256: file.sha256 });
    }
  }
  if (cleanKey !== null) {
    cleanProbes.delete(cleanKey);
    cleanProbes.set(cleanKey, vouched!);
    while (cleanProbes.size > WORKING_STAT_CACHES_REMEMBERED) cleanProbes.delete(cleanProbes.keys().next().value!);
    if (opts.persist && touched && Date.now() - (cleanProbesSavedAt.get(cleanKey) ?? -Infinity) >= CLEAN_PROBE_SAVE_EVERY_MS) saveCleanProbes(cleanKey, vouched!);
  }
  if (opts.verify) {
    const differs = [...checked].find(([path, kept]) => JSON.stringify(results.get(path)) !== JSON.stringify(kept));
    opts.verify(checked.size, differs ? `${differs[0]}: kept ${JSON.stringify(differs[1])}, read ${JSON.stringify(results.get(differs[0]))}` : null);
  }
  return results;
}

/** Committed-mode `clean` probes this process vouched, by root and Git root: path -> `<stat key>|<tree oid>|<tree mode>|<sha256>`. */
const cleanProbes = new Map<string, Map<string, string>>();
const cleanProbesSavedAt = new Map<string, number>();
const CLEAN_PROBE_SAVE_EVERY_MS = 60_000;
const CLEAN_PROBE = /^\d+:\d+:\d+:\d+:\d+:\d+:\d+\|[0-9a-f]{40}(?:[0-9a-f]{24})?\|\d{6}\|[0-9a-f]{64}$/;

/** Where `persist` keeps the vouched probes of one root and Git root (`key`): this machine's gbrain home, never the brain. */
export function cleanProbeFile(key: string): string {
  return join(configDir(), 'persistence', 'clean-probes', `${sha256(key).slice(0, 32)}.json`);
}

function loadCleanProbes(key: string): Map<string, string> | undefined {
  try {
    const doc = JSON.parse(readFileSync(cleanProbeFile(key), 'utf8')) as { v?: unknown; key?: unknown; entries?: unknown };
    if (doc.v !== 1 || doc.key !== key || !doc.entries || typeof doc.entries !== 'object' || Array.isArray(doc.entries)) return undefined;
    const list = Object.entries(doc.entries as Record<string, unknown>);
    if (list.length > WORKING_STAT_CACHE_MAX_ENTRIES || !list.every(([, entry]) => typeof entry === 'string' && CLEAN_PROBE.test(entry))) return undefined;
    return new Map(list as Array<[string, string]>);
  } catch { return undefined; }
}

function saveCleanProbes(key: string, probes: Map<string, string>): void {
  const file = cleanProbeFile(key), staged = `${file}.${randomUUID()}.tmp`;
  try {
    mkdirSync(join(file, '..'), { recursive: true, mode: 0o700 });
    writeFileSync(staged, JSON.stringify({ v: 1, key, entries: Object.fromEntries(probes) }), { mode: 0o600 });
    renameSync(staged, file);
    cleanProbesSavedAt.set(key, Date.now());
  } catch { rmSync(staged, { force: true }); }
}

/** Tests: forget this process's vouched probes and save times, as a new process starts. */
export function __resetCleanProbesForTests(): void { cleanProbes.clear(); cleanProbesSavedAt.clear(); }

/** Raw blob ids (no filters) of present files, in order, or null when Git did not answer every path. */
async function hashRaw(gitRoot: string, paths: string[]): Promise<string[] | null> {
  if (paths.length === 0) return [];
  const chunks: Buffer[] = [];
  const result = await gitStream(gitRoot, ['hash-object', '--no-filters', '--stdin-paths'],
    { timeoutMs: PROBE_TIMEOUT_MS, maxBytes: paths.length * 66, stdin: lines(paths, '\n'), onChunk: chunk => { chunks.push(chunk); } }).catch(() => null);
  if (!result?.ok) return null;
  const text = Buffer.concat(chunks).toString('utf8');
  if (!text.endsWith('\n')) return null;
  const oids = text.slice(0, -1).split('\n');
  return oids.length === paths.length && oids.every(oid => OID.test(oid)) ? oids : null;
}

const ATTRIBUTES = ['filter', 'text', 'eol', 'working-tree-encoding'] as const;

/** The four conversion attributes per path, or null when Git did not answer every path completely. */
async function readAttributes(gitRoot: string, paths: string[]): Promise<Map<string, Record<string, string>> | null> {
  const attrs = new Map<string, Record<string, string>>();
  if (paths.length === 0) return attrs;
  const splitter = new NulRecordSplitter();
  const fields: string[] = [];
  const result = await gitStream(gitRoot, ['check-attr', '-z', '--stdin', ...ATTRIBUTES],
    { timeoutMs: PROBE_TIMEOUT_MS, stdin: lines(paths, '\0'), onChunk: chunk => { for (const record of splitter.push(chunk)) fields.push(record.toString('utf8')); } }).catch(() => null);
  if (!result || !hardenedStreamComplete(result) || !splitter.atRecordBoundary || fields.length !== paths.length * ATTRIBUTES.length * 3) return null;
  for (let i = 0; i < fields.length; i += 3) {
    const entry = attrs.get(fields[i]!) ?? {};
    entry[fields[i + 1]!] = fields[i + 2]!;
    attrs.set(fields[i]!, entry);
  }
  return paths.every(path => ATTRIBUTES.every(name => attrs.get(path)?.[name] !== undefined)) ? attrs : null;
}

/**
 * The working stat cache (design v3.2 §2.1, Git's freshness contract). Each
 * Git-root-relative path maps to the `lstat` key `dev:ino:mode:uid:size:mtime_ns:ctime_ns`
 * it had when its bytes were hashed, or `absent`. A user can restore `mtime` but
 * not `ctime`, so a same-size, same-inode edit still changes the key. Plain JSON;
 * entries are self-contained, so two caches of one root merge by spreading
 * their entries.
 */
export interface WorkingStatCache { v: 1; root: string; entries: Record<string, string> }

const WORKING_RACY_WINDOW_NS = 2_000_000_000n;

/**
 * The newest time a vouched stat may carry: 2 s before the earlier of this process's clock and the filesystem's, read as
 * the mtime of a file just written in the Git directory (Git anchors its racy entries to the index's mtime). On a mount
 * whose clock lags the process's, the process clock alone vouched a file written a moment ago, and a same-size edit in
 * the same timestamp tick then kept its stale verdict. The process clock alone when the Git directory cannot be written.
 */
function racyFloor(gitRoot: string): bigint {
  const local = BigInt(Date.now()) * 1_000_000n;
  let fs: bigint | null = null;
  try {
    const dotGit = join(gitRoot, '.git');
    const dir = lstatSync(dotGit).isDirectory() ? dotGit : resolve(gitRoot, /^gitdir: (.+)$/m.exec(readFileSync(dotGit, 'utf8'))?.[1]?.trim() ?? '.git');
    const clock = join(dir, `gbrain-clock-${process.pid}-${randomUUID()}`);
    writeFileSync(clock, '');
    try { fs = lstatSync(clock, { bigint: true }).mtimeNs; } finally { rmSync(clock, { force: true }); }
  } catch {}
  return (fs !== null && fs < local ? fs : local) - WORKING_RACY_WINDOW_NS;
}
const WORKING_HASH_ATTEMPTS = 3;
const WORKING_STAT_CACHES_REMEMBERED = 64;
const ABSENT = 'absent';
const STAT_KEY = /^\d+:\d+:\d+:\d+:\d+:\d+:\d+$/;
/** Entries a cache keeps (by path order); a path beyond the bound reads as changed [estimated]. */
export const WORKING_STAT_CACHE_MAX_ENTRIES = 100_000;

/** The cache key of one bigint `lstat`, or null when it cannot vouch for the file (no stat, or no `ctime`). */
export function workingStatKey(stats: Pick<BigIntStats, 'dev' | 'ino' | 'mode' | 'uid' | 'size' | 'mtimeNs' | 'ctimeNs'> | null): string | null {
  if (!stats || !stats.ctimeNs) return null;
  return [stats.dev, stats.ino, stats.mode, stats.uid, stats.size, stats.mtimeNs, stats.ctimeNs].join(':');
}

/** `absent` for a path nothing backs, its key, or null for any other stat failure (uncertain). */
function statAt(root: string, path: string): string | null {
  try { return workingStatKey(lstatSync(join(root, path), { bigint: true })); }
  catch (error) { return (error as NodeJS.ErrnoException).code === 'ENOENT' ? ABSENT : null; }
}

function rootStill(root: string): boolean {
  try { return realpathSync.native(root) === root; } catch { return false; }
}

/**
 * The idle tick's working-mode check: every path in `paths` or in the cache whose
 * stat no longer matches its entry is changed, as is a path the cache does not
 * hold (new, racy when hashed, past the bound), a stat that fails or has no
 * `ctime`, and every path when there is no cache or its root moved. Uncertainty is
 * never "no change". Returns the changed paths, sorted, and the cache without
 * them.
 */
export function workingChanged(cache: WorkingStatCache | null, paths: Iterable<string>): { changed: string[]; cache: WorkingStatCache | null } {
  const all = new Set(paths);
  if (!cache || !rootStill(cache.root)) return { changed: [...all].sort(), cache: null };
  for (const path of Object.keys(cache.entries)) all.add(path);
  const changed: string[] = [];
  const entries: Record<string, string> = {};
  for (const path of [...all].sort()) {
    const entry = cache.entries[path];
    if (entry !== undefined && statAt(cache.root, path) === entry) entries[path] = entry;
    else changed.push(path);
  }
  return { changed, cache: { v: 1, root: cache.root, entries } };
}

/**
 * The working level of `paths` (`probeDirtiness` with `working`, every path hashed:
 * full rounds rehash unconditionally) and the stat cache it leaves. Each path is
 * stat'd before and after its bytes are hashed; a path whose stat moved is
 * discarded and rehashed, up to three attempts, and one still moving keeps its
 * last probe but no entry. An entry is written only for a stable stat whose
 * `mtime` and `ctime` are older than 2 s before that attempt began, by the earlier of
 * the process's and the filesystem's clocks (`racyFloor`; a racy entry is
 * left out, so the next pass rehashes it), and only for a probe with a level
 * (raw bytes read, or `deleted` with nothing at the path). `afterHash` runs between
 * the hash and the re-stat (tests inject an edit there).
 */
export async function hashWorkingLevel(root: string, gitRoot: string, paths: string[], tree: Map<string, GitTreeEntry>,
  opts: { batch?: number; maxEntries?: number; afterHash?: (paths: string[]) => void | Promise<void> } = {}): Promise<{ probes: Map<string, DirtinessProbe>; cache: WorkingStatCache | null }> {
  let realGitRoot: string | null = null;
  try { realGitRoot = realpathSync.native(gitRoot); } catch {}
  const statOf = (path: string) => (realGitRoot === null ? null : statAt(realGitRoot, path));
  const probes = new Map<string, DirtinessProbe>();
  const entries = new Map<string, string>();
  let pending = [...new Set(paths)];
  for (let attempt = 0; attempt < WORKING_HASH_ATTEMPTS && pending.length > 0; attempt++) {
    const racyFrom = realGitRoot === null ? BigInt(Date.now()) * 1_000_000n - WORKING_RACY_WINDOW_NS : racyFloor(realGitRoot);
    const before = new Map(pending.map(path => [path, statOf(path)]));
    const probed = await probeDirtiness(root, gitRoot, pending, tree, new Map(), { batch: opts.batch, working: true });
    await opts.afterHash?.(pending);
    const moved: string[] = [];
    for (const path of pending) {
      const probe = probed.get(path)!;
      probes.set(path, probe);
      const key = before.get(path)!;
      if (statOf(path) !== key) { moved.push(path); continue; }
      if (key === null) continue;
      const [mtimeNs = '0', ctimeNs = '0'] = key === ABSENT ? [] : key.split(':').slice(5);
      const vouched = key === ABSENT ? probe.dp === 'deleted' : probe.rawOid !== null && BigInt(mtimeNs) < racyFrom && BigInt(ctimeNs) < racyFrom;
      if (vouched) entries.set(path, key);
    }
    pending = moved;
  }
  if (realGitRoot === null) return { probes, cache: null };
  const kept = [...entries.keys()].sort().slice(0, Math.max(0, opts.maxEntries ?? WORKING_STAT_CACHE_MAX_ENTRIES));
  return { probes, cache: { v: 1, root: realGitRoot, entries: Object.fromEntries(kept.map(path => [path, entries.get(path)!])) } };
}

/** A serialized cache (a pass row's copy), or null for anything that is not one; null reads as "everything changed". */
export function parseWorkingStatCache(value: unknown): WorkingStatCache | null {
  if (!value || typeof value !== 'object') return null;
  const { v, root, entries } = value as Record<string, unknown>;
  if (v !== 1 || typeof root !== 'string' || !entries || typeof entries !== 'object' || Array.isArray(entries)) return null;
  const list = Object.entries(entries as Record<string, unknown>);
  if (list.length > WORKING_STAT_CACHE_MAX_ENTRIES || !list.every(([, key]) => typeof key === 'string' && (key === ABSENT || STAT_KEY.test(key)))) return null;
  return { v: 1, root, entries: Object.fromEntries(list) as Record<string, string> };
}

const rememberedCaches = new Map<string, WorkingStatCache>();

/** Keeps a cache in this process under `key` (the pass uses `<source>:<incarnation>`); the least recently stored past 64 are dropped. */
export function rememberWorkingStatCache(key: string, cache: WorkingStatCache): void {
  rememberedCaches.delete(key);
  rememberedCaches.set(key, cache);
  while (rememberedCaches.size > WORKING_STAT_CACHES_REMEMBERED) rememberedCaches.delete(rememberedCaches.keys().next().value!);
}

/** The cache this process remembered under `key`, or null. */
export function recallWorkingStatCache(key: string): WorkingStatCache | null {
  return rememberedCaches.get(key) ?? null;
}