Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
39.9 KB
Raw Blame
/**
 * Scanning a session root for the project archive (sections 5.2 to 5.8): the
 * whole folder, `.git/` included, minus the exclusions of 5.2 and minus every
 * gitignored path (`ignore.ts`: an ignored directory is pruned unwalked);
 * streaming SHA-256 with a (path, size, mtimeNs, ctimeNs, ino) cache;
 * delta computation against the previous archive's entry list; and the
 * `__omnirush__/manifest.json` document.
 */
import { gitChildEnv } from "../git-env.js";
import { spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { constants as fsConstants, type BigIntStats } from "node:fs";
import { lstat, open, readdir, readlink, realpath, stat } from "node:fs/promises";
import { basename, isAbsolute, join, relative, resolve, sep } from "node:path";

import { clampUploadBytes, isUploadPathDenied, REGENERABLE_DIR_NAMES, stripRemoteUserinfo } from "../session-uploader.js";
import { ATTACHMENTS_ROOT_NAME } from "./attachments.js";
import { isArchiveCodeFile, needsCodeCheck, readCleanCodeFile } from "./code-files.js";
import type { ExcludedList, ExcludedReason } from "./capture-v2.js";
import { gitSkipReason } from "../command-guard.js";
import { hintGarbageCollection } from "./files.js";
import { isGitConfigPath, scrubGitConfig } from "./git-scrub.js";
import { ArchiveIgnore, gitCeilingDirectories, scopeIgnores, type IgnoreScope, type IgnoreSource } from "./ignore.js";

export const ARCHIVE_SCHEMA = "omnirush.archive.v1";
export const RESERVED_ROOT_NAME = "__omnirush__";
/** The archive folder of the files outside the workspace (outside.ts); a workspace entry of that name is reserved too. */
export const OUTSIDE_ROOT_NAME = "__outside__";
/** Capture v2: prompt attachments (attachments.ts); a workspace entry of that name is reserved too. */
export { ATTACHMENTS_ROOT_NAME };
/** Capture v2: the `.git` of the repository above the session folder (enclosing.ts); a workspace entry of that name is reserved too. */
export const ENCLOSING_REPO_ROOT_NAME = "__enclosing_repo__";
/** The Linux sandbox's turn checkpoints (box/, index.ts sandboxEntries); a workspace entry of that name is reserved too. */
export const SANDBOX_ROOT_NAME = "__sandbox__";
/** Screenshots and other tool-result images (media-store.ts); a workspace entry of that name is reserved too. */
export const MEDIA_ROOT_NAME = "__media__";
/** A git config file larger than this is left out (`too_large`): it is never read whole to scrub it, and never archived raw. */
const MAX_SCRUBBED_CONFIG_BYTES = 1024 * 1024;
export const STAT_CONCURRENCY = 64;
export const HASH_CONCURRENCY = 6;
export const HASH_READ_BYTES = 128 * 1024;
/** Bytes hashed, or entries inspected or hashed, between two garbage collection hints. */
const GC_HINT_BYTES = 64 * 1024 * 1024;
const GC_HINT_ENTRIES = 10_000;
const MAX_LABEL_BYTES = 255;
const GIT_TIMEOUT_MS = 15_000;
/**
 * Opens an entry for reading: never through a symlink in the last component
 * (O_NOFOLLOW), and never blocking when a FIFO replaced the file after lstat
 * (O_NONBLOCK, which regular files ignore; O_NOCTTY for a terminal). Callers
 * fstat the handle and read only a regular file.
 */
export const OPEN_ENTRY_FLAGS =
  fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0) | (fsConstants.O_NOCTTY ?? 0);

export type ArchiveKind = "base" | "delta";
/**
 * Why a delta was taken (manifest.json `trigger`): a completed turn, or a
 * final archive of the folder after the last completed turn, which carries
 * that turn's number again.
 */
export type ArchiveTrigger = "turn" | "final" | "pre_tool";
/** What prompted a final archive (manifest.json `reason`). */
export type FinalReason = "idle" | "turn_incomplete" | "session_deleted" | "app_quit" | "app_start" | "first_touch";
/**
 * manifest.json `reason`: a final archive's, or "late_writes" on a pre_tool
 * state holding files written after the turn before ended (a background
 * process of that turn still writing): they belong to that turn's outcome
 * and to this turn's start.
 */
export type ArchiveReason = FinalReason | "late_writes";
/**
 * manifest.json `scope` of a touched-files archive: its `files` are only the
 * files the agent touched, and a path it does not list is not deleted (only
 * `deleted` deletes). The whole-folder archives have no `scope`.
 */
export type ArchiveScope = "touched";
export type ArchiveEntryType = "file" | "dir" | "symlink";

/** One manifest `files` item (section 5.4). */
export type ArchiveEntry = {
  path: string;
  type: ArchiveEntryType;
  mode: number;
  size: number;
  sha256: string | null;
  target?: string;
  /** Capture v2: a symlink whose target does not exist. */
  broken?: boolean;
  /** A git config whose archived copy had credentials removed (git-scrub.ts): size and sha256 are the copy's. */
  scrubbed?: boolean;
};

/** A scanned entry: the manifest fields plus what pass 2 and the hash cache need. */
export type ScannedEntry = ArchiveEntry & {
  /** Tar mtime: floor(mtimeMs / 1000), clamped at pack time. */
  mtime: number;
  /** `size:mtimeNs:ctimeNs:ino` from lstat: the hash cache key, and what pass 2 re-checks (size and mtimeNs). */
  statKey: string;
  /** st_dev from lstat: with the ino in statKey, the identity pass 2 requires of the file it reads. */
  dev: number;
  /** The bytes pass 2 writes instead of reading the file (a scrubbed git config). */
  content?: Buffer;
  /** An absolute file pass 2 reads instead of `<root>/<path>` (a staged attachment). */
  source?: string;
};

/** The prefix of a stat key that pass 2 compares against fstat: size and mtimeNs. */
export function statKeyPrefix(size: bigint | number, mtimeNs: bigint): string {
  return `${size}:${mtimeNs}:`;
}

/** Whether `stats` (an fstat) describe the file pass 1 lstat'ed for `entry`: same st_dev and st_ino. */
export function isSameFileIdentity(entry: ScannedEntry, stats: BigIntStats): boolean {
  return entry.dev === Number(stats.dev) && entry.statKey.endsWith(`:${stats.ino}`);
}

export type ExcludedCounts = {
  credential: number;
  special: number;
  app_state: number;
  reserved: number;
  unreadable: number;
  non_utf8: number;
};

export type ArchiveGit = {
  head: string | null;
  branch: string | null;
  remote: string | null;
  dirty: boolean;
  /** The root's path inside the repository (`git rev-parse --show-prefix` without the trailing `/`), "" at its top level. */
  path?: string;
};

export type ArchiveManifest = {
  schema: typeof ARCHIVE_SCHEMA;
  kind: ArchiveKind;
  archive_id: string;
  session_id: string;
  sequence: number;
  turn: number;
  created_at: string;
  parent_archive_id: string | null;
  trigger?: ArchiveTrigger;
  reason?: ArchiveReason;
  scope?: ArchiveScope;
  workspace: { label: string; marker: string; git: ArchiveGit | null };
  files: ArchiveEntry[];
  deleted?: string[];
  excluded: ExcludedCounts;
};

export function emptyExcludedCounts(): ExcludedCounts {
  return { credential: 0, special: 0, app_state: 0, reserved: 0, unreadable: 0, non_utf8: 0 };
}

/**
 * Orders paths by their UTF-8 bytes (section 5.4) without encoding them:
 * UTF-16 order differs from code point order only where a surrogate meets a
 * unit in U+E000..U+FFFF, so the first differing units are remapped.
 */
export function compareArchivePaths(left: string, right: string): number {
  const length = Math.min(left.length, right.length);
  for (let index = 0; index < length; index += 1) {
    let a = left.charCodeAt(index);
    let b = right.charCodeAt(index);
    if (a === b) continue;
    if (a >= 0xd800) a = a >= 0xe000 ? a - 0x800 : a + 0x2000;
    if (b >= 0xd800) b = b >= 0xe000 ? b - 0x800 : b + 0x2000;
    return a - b;
  }
  return left.length - right.length;
}

/**
 * True when a file or symlink must be left out of the archive (section 5.3):
 * the session uploader's credential denylist, with git internals exempt and
 * `node_modules` not counted as a denial. A whole-folder or touched-files
 * scan still keeps a denylisted source file that passes the content check
 * of code-files.ts.
 */
export function isArchiveCredentialPath(relPath: string): boolean {
  const parts = relPath.split("/");
  if (parts.some((part) => part.toLowerCase() === ".git")) return false;
  const rest = parts.filter((part) => part.toLowerCase() !== "node_modules");
  return rest.length > 0 && isUploadPathDenied(rest.join("/"));
}

/** workspace.label: the root's basename, at most 255 UTF-8 bytes. */
export function archiveLabel(root: string): string {
  return clampUploadBytes(basename(resolve(root)), MAX_LABEL_BYTES).text;
}

// --- hash cache -------------------------------------------------------------------

const SHA256_HEX = /^[0-9a-f]{64}$/;

/**
 * Iterates the entry rows of a state document written one JSON row per line
 * (`{"v":..,"entries":[` / rows / `]}`) without materialising the whole
 * parsed array next to the text.
 */
export function* jsonLineRows(text: string): Generator<unknown> {
  let start = 0;
  while (start < text.length) {
    let end = text.indexOf("\n", start);
    if (end === -1) end = text.length;
    let line = text.slice(start, end);
    start = end + 1;
    if (line.endsWith(",")) line = line.slice(0, -1);
    if (!line.startsWith("[") && !line.startsWith("{\"path\"")) continue;
    try {
      yield JSON.parse(line);
    } catch {
      // A damaged row is skipped; the caller decides whether that matters.
    }
  }
}

function* jsonLineDocument<T>(header: string, rows: Iterable<T>, serialise: (row: T) => string, rowsPerChunk = 2_000): Generator<string> {
  yield header;
  let batch: string[] = [];
  let first = true;
  for (const row of rows) {
    batch.push(serialise(row));
    if (batch.length >= rowsPerChunk) {
      yield `${first ? "" : ","}\n${batch.join(",\n")}`;
      first = false;
      batch = [];
    }
  }
  if (batch.length > 0) yield `${first ? "" : ","}\n${batch.join(",\n")}`;
  yield "\n]}\n";
}

/**
 * path -> (size, mtimeNs, ctimeNs, ino, sha256), shared by the sessions of one
 * root. A hit requires the whole stat key to match, so a writer that
 * preserves mtime is still caught by ctime or inode. Rows loaded from disk
 * are packed into one string (sha256 then the stat key); rows of the current
 * scan are the scanned entries themselves, so a 200k-entry scan adds no
 * strings. During a scan, rows move from `previous` to `current`; paths the
 * scan did not see are dropped at commitScan() without a second index.
 */
export class ArchiveHashCache {
  /** Rows not yet seen by the running scan: packed strings from disk, or entries of the previous scan. */
  private previous = new Map<string, string | ScannedEntry>();
  private current = new Map<string, ScannedEntry>();
  private dirty = false;

  /** Loads the text written by jsonChunks(); anything unreadable is an empty cache. */
  static fromText(text: string | null): ArchiveHashCache {
    const cache = new ArchiveHashCache();
    if (!text?.startsWith('{"v":2,')) return cache;
    for (const row of jsonLineRows(text)) {
      if (!Array.isArray(row) || row.length !== 3) continue;
      const [path, statKey, sha256] = row;
      if (typeof path !== "string" || typeof statKey !== "string" || typeof sha256 !== "string" || !SHA256_HEX.test(sha256)) continue;
      cache.previous.set(path, [sha256, statKey].join(""));
    }
    return cache;
  }

  get changed(): boolean {
    return this.dirty;
  }

  get size(): number {
    return this.previous.size + this.current.size;
  }

  /** The cached SHA-256 when the entry's stat key matches; the scan sets entry.sha256 from it. */
  lookup(entry: ScannedEntry): string | null {
    const held = this.previous.get(entry.path);
    if (held === undefined) return null;
    this.previous.delete(entry.path);
    const sha256 = typeof held === "string"
      ? (held.length === 64 + entry.statKey.length && held.endsWith(entry.statKey) ? held.slice(0, 64) : null)
      : (held.statKey === entry.statKey ? held.sha256 : null);
    if (!sha256) {
      this.dirty = true;
      return null;
    }
    this.current.set(entry.path, entry);
    return sha256;
  }

  /** Records a freshly hashed entry (entry.sha256 already set). */
  remember(entry: ScannedEntry): void {
    this.previous.delete(entry.path);
    this.current.set(entry.path, entry);
    this.dirty = true;
  }

  forget(path: string): void {
    const had = this.previous.delete(path) || this.current.delete(path);
    if (had) this.dirty = true;
  }

  /** Ends a scan: paths the scan did not see are dropped. */
  commitScan(): void {
    if (this.previous.size > 0) this.dirty = true;
    this.previous = this.current;
    this.current = new Map();
  }

  /** The cache as JSON text pieces, one row per line, so it is never one giant string. */
  jsonChunks(): Generator<string> {
    this.dirty = false;
    // A path is in exactly one of the two maps (lookup and remember move it to current).
    const { previous, current } = this;
    function* rows(): Generator<string> {
      for (const [path, held] of previous) {
        if (typeof held === "string") yield JSON.stringify([path, held.slice(64), held.slice(0, 64)]);
        else if (held.sha256) yield JSON.stringify([path, held.statKey, held.sha256]);
      }
      for (const [path, entry] of current) if (entry.sha256) yield JSON.stringify([path, entry.statKey, entry.sha256]);
    }
    return jsonLineDocument('{"v":2,"entries":[', rows(), (row) => row);
  }
}

/** The baseline document: the full entry list after an archive, one entry per line; `unstable` paths get a null hash. */
export function baselineChunks(entries: readonly ArchiveEntry[], unstable: ReadonlySet<string> = new Set()): Generator<string> {
  return jsonLineDocument('{"v":1,"entries":[', entries, (entry) => {
    const out = archiveEntry(entry);
    if (unstable.has(entry.path)) out.sha256 = null;
    return JSON.stringify(out);
  });
}

const ENTRY_TYPES: ReadonlySet<string> = new Set(["file", "dir", "symlink"]);

/** Parses a baseline document; null when it is not one (the session cannot compute deltas any more). */
export function parseBaselineText(text: string | null): ArchiveEntry[] | null {
  if (!text?.startsWith('{"v":1,')) return null;
  const entries: ArchiveEntry[] = [];
  for (const row of jsonLineRows(text)) {
    if (typeof row !== "object" || row === null) return null;
    const path = "path" in row ? row.path : undefined;
    const type = "type" in row ? row.type : undefined;
    const mode = "mode" in row ? row.mode : undefined;
    const size = "size" in row ? row.size : undefined;
    const sha256 = "sha256" in row ? row.sha256 : undefined;
    const target = "target" in row ? row.target : undefined;
    if (typeof path !== "string" || typeof type !== "string" || !ENTRY_TYPES.has(type) || typeof mode !== "number" || typeof size !== "number") return null;
    if (sha256 !== null && typeof sha256 !== "string") return null;
    if (target !== undefined && typeof target !== "string") return null;
    const entry: ArchiveEntry = { path, type: type === "file" ? "file" : type === "dir" ? "dir" : "symlink", mode, size, sha256 };
    if (target !== undefined) entry.target = target;
    if ("broken" in row && row.broken === true) entry.broken = true;
    if ("scrubbed" in row && row.scrubbed === true) entry.scrubbed = true;
    entries.push(entry);
  }
  return entries;
}

// --- scanning -----------------------------------------------------------------------

/** Work counters for tests and profiling. */
export type ScanMetrics = { stats: number; fileReads: number; bytesHashed: number; cacheHits: number };

export function emptyScanMetrics(): ScanMetrics {
  return { stats: 0, fileReads: 0, bytesHashed: 0, cacheHits: 0 };
}

export type ScanOptions = {
  /** Absolute app state/temp/data directories; pruned (and counted as app_state) when under the root. */
  excludedDirs?: readonly string[];
  /** archiveIncludeCredentialFiles: turns the credential filter off entirely. */
  includeCredentialFiles?: boolean;
  hashCache?: ArchiveHashCache;
  statConcurrency?: number;
  hashConcurrency?: number;
  metrics?: ScanMetrics;
  /** Stops the scan between entries: it then rejects with the signal's reason. */
  signal?: AbortSignal;
  /** Capture v2: every entry left out, with its reason. */
  excludedList?: ExcludedList;
  /** Capture v2 (a folder in no repository): the regenerable folders (node_modules, .venv, target, ...) are left out. */
  pruneRegenerable?: boolean;
  /** A listing scan (never packed): a file larger than this is not hashed (its sha256 stays null). */
  maxHashBytes?: number;
};

/**
 * `ignored`: entries left out because git ignores them (an ignored directory
 * counts once, its content is never walked), and where the root's answer came
 * from. Not part of the manifest's `excluded` object (section 5.6 is
 * unchanged); the archiver logs it.
 */
export type ScanResult = { entries: ScannedEntry[]; excluded: ExcludedCounts; ignored: number; ignoreSource: IgnoreSource };

/**
 * Runs `operation` over `items` with at most `limit` in flight. The workers
 * pull plain data items, so a 200k-entry level costs 200k small objects, not
 * 200k suspended async frames.
 */
export async function forEachBounded<T>(items: readonly T[], limit: number, operation: (item: T) => Promise<void>): Promise<void> {
  let next = 0;
  const worker = async (): Promise<void> => {
    while (next < items.length) {
      const item = items[next]!;
      next += 1;
      await operation(item);
    }
  };
  await Promise.all(Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, worker));
}

const NAME_DECODER = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });

export function decodeName(raw: Buffer): string | null {
  try {
    return NAME_DECODER.decode(raw);
  } catch {
    return null;
  }
}

export function portable(path: string): string {
  return sep === "/" ? path : path.split(sep).join("/");
}

/** The excluded directories that lie strictly under the root, as root-relative paths. */
export async function excludedRelativeDirs(root: string, dirs: readonly string[]): Promise<Set<string>> {
  const rootForms = new Set([resolve(root)]);
  try {
    rootForms.add(await realpath(root));
  } catch {
    // The scan reports an unreadable root itself.
  }
  const result = new Set<string>();
  for (const dir of dirs) {
    const forms = new Set([resolve(dir)]);
    try {
      forms.add(await realpath(dir));
    } catch {
      // Not created yet: only its resolved form can be under the root.
    }
    for (const form of forms) {
      for (const rootForm of rootForms) {
        const rel = relative(rootForm, form);
        const outside = rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel);
        if (rel && !outside) result.add(portable(rel));
      }
    }
  }
  return result;
}

// Strings kept per entry are built with join(), which yields one flat string;
// template concatenation in JSC yields a rope that keeps every piece alive,
// roughly doubling the per-entry cost on a 200k-entry tree.
export function statFields(stats: BigIntStats) {
  return {
    mode: Number(stats.mode & 0o7777n),
    mtime: Number(stats.mtimeNs / 1_000_000_000n),
    statKey: [stats.size, stats.mtimeNs, stats.ctimeNs, stats.ino].join(":"),
    dev: Number(stats.dev),
  };
}

/** SHA-256 of the first `size` bytes (the lstat size): pass 2 streams exactly those and detects any change. */
async function hashFile(absolute: string, size: number, buffer: Buffer, metrics: ScanMetrics, signal?: AbortSignal): Promise<string | null> {
  let handle: Awaited<ReturnType<typeof open>>;
  try {
    handle = await open(absolute, OPEN_ENTRY_FLAGS);
  } catch {
    return null;
  }
  metrics.fileReads += 1;
  if (metrics.fileReads % GC_HINT_ENTRIES === 0) hintGarbageCollection();
  try {
    if (!(await handle.stat()).isFile()) return null;
    const hash = createHash("sha256");
    let remaining = size;
    while (remaining > 0) {
      // A stopped scan does not finish a big file first (the scan rejects right after).
      if (signal?.aborted) return null;
      const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, remaining), null);
      if (bytesRead === 0) break;
      hash.update(buffer.subarray(0, bytesRead));
      if (Math.floor((metrics.bytesHashed + bytesRead) / GC_HINT_BYTES) > Math.floor(metrics.bytesHashed / GC_HINT_BYTES)) hintGarbageCollection();
      metrics.bytesHashed += bytesRead;
      remaining -= bytesRead;
    }
    return hash.digest("hex");
  } catch {
    return null;
  } finally {
    await handle.close().catch(() => undefined);
  }
}

/** Whether a symlink's target is missing (or a loop): its own lstat already showed a link. */
export async function isBrokenLink(absolute: string): Promise<boolean> {
  try {
    await stat(absolute);
    return false;
  } catch (error) {
    const code = error instanceof Error && "code" in error ? error.code : undefined;
    return code === "ENOENT" || code === "ELOOP" || code === "ENOTDIR";
  }
}

/**
 * A git config entry (git-scrub.ts): read once from the very file pass 1
 * lstat'ed, scrubbed of credentials; when anything was removed, the entry
 * carries the scrubbed bytes (pass 2 writes them) and their size and hash.
 * False when it cannot be read.
 */
export async function scrubConfigEntry(absolute: string, entry: ScannedEntry, metrics: ScanMetrics): Promise<boolean> {
  let handle: Awaited<ReturnType<typeof open>>;
  try {
    handle = await open(absolute, OPEN_ENTRY_FLAGS);
  } catch {
    return false;
  }
  try {
    const stats = await handle.stat({ bigint: true });
    if (!stats.isFile() || !isSameFileIdentity(entry, stats) || Number(stats.size) !== entry.size) return false;
    const raw = Buffer.alloc(entry.size);
    let offset = 0;
    while (offset < raw.length) {
      const { bytesRead } = await handle.read(raw, offset, raw.length - offset, offset);
      if (bytesRead === 0) return false;
      offset += bytesRead;
    }
    metrics.fileReads += 1;
    metrics.bytesHashed += raw.length;
    const scrub = scrubGitConfig(raw.toString("utf8"));
    const bytes = scrub.changed ? Buffer.from(scrub.text, "utf8") : raw;
    // Pass 2 writes exactly these checked bytes: a config rewritten after this read never reaches the archive.
    entry.content = bytes;
    entry.size = bytes.length;
    if (scrub.changed) entry.scrubbed = true;
    entry.sha256 = createHash("sha256").update(bytes).digest("hex");
    return true;
  } catch {
    return false;
  } finally {
    await handle.close().catch(() => undefined);
  }
}

/**
 * Pass 1: walks the root with lstat (never following a symlink), applies the
 * exclusions of section 5.2, leaves out what git ignores (a whole ignored
 * directory without walking it) and hashes every file whose cache key
 * changed. Entries come back sorted by UTF-8 bytes.
 */
export async function scanArchiveTree(root: string, options: ScanOptions = {}): Promise<ScanResult> {
  const concurrency = options.statConcurrency ?? STAT_CONCURRENCY;
  const metrics = options.metrics ?? emptyScanMetrics();
  const cache = options.hashCache;
  const signal = options.signal;
  const excluded = emptyExcludedCounts();
  const prunedDirs = await excludedRelativeDirs(root, options.excludedDirs ?? []);
  const includeCredentials = options.includeCredentialFiles === true;
  const entries: ScannedEntry[] = [];
  const toHash: ScannedEntry[] = [];
  const ignore = new ArchiveIgnore(resolve(root), signal);
  const list = options.excludedList;
  const note = (path: string, reason: ExcludedReason, type: "file" | "dir", size?: number): void => {
    list?.add({ path, type, reason, ...(size !== undefined ? { size } : {}) });
  };

  const inspect = async (relDir: string, absDir: string, name: string, scope: IgnoreScope, next: Array<{ rel: string; abs: string; scope: IgnoreScope }>): Promise<void> => {
    signal?.throwIfAborted();
    const rel = relDir ? [relDir, name].join("/") : name;
    const abs = join(absDir, name);
    // Git's answer names ignored directories and files alike: no lstat needed.
    if (scope.kind === "git" && scopeIgnores(scope, rel, false)) {
      ignore.ignored += 1;
      if (list) {
        // One lstat for the listing's type (an ignored folder is never walked).
        const kind = await lstat(abs).then((info) => (info.isDirectory() ? "dir" as const : "file" as const), () => "file" as const);
        note(rel, "gitignored", kind);
      }
      return;
    }
    let stats: BigIntStats;
    try {
      stats = await lstat(abs, { bigint: true });
      metrics.stats += 1;
      if (metrics.stats % GC_HINT_ENTRIES === 0) hintGarbageCollection();
    } catch {
      excluded.unreadable += 1;
      note(rel, "unreadable", "file");
      return;
    }
    if (scope.kind === "rules" && scopeIgnores(scope, rel, stats.isDirectory())) {
      ignore.ignored += 1;
      note(rel, "gitignored", stats.isDirectory() ? "dir" : "file");
      return;
    }
    if (stats.isDirectory()) {
      if (prunedDirs.has(rel)) {
        excluded.app_state += 1;
        note(rel, "app_state", "dir");
        return;
      }
      if (options.pruneRegenerable && REGENERABLE_DIR_NAMES.has(name)) {
        ignore.ignored += 1;
        note(rel, "regenerable", "dir");
        return;
      }
      entries.push({ path: rel, type: "dir", size: 0, sha256: null, ...statFields(stats) });
      next.push({ rel, abs, scope });
      return;
    }
    if (!stats.isFile() && !stats.isSymbolicLink()) {
      excluded.special += 1;
      note(rel, "special", "file");
      return;
    }
    // A source file named like a key store is decided by its content once it is known not to be ignored (code-files.ts).
    let codeCheck = false;
    if (!includeCredentials && (isArchiveCredentialPath(rel) || (stats.isFile() && needsCodeCheck(rel.split("/"))))) {
      if (!stats.isFile() || !isArchiveCodeFile(rel.split("/"))) {
        excluded.credential += 1;
        note(rel, "credential", "file");
        return;
      }
      codeCheck = true;
    }
    if (stats.isSymbolicLink()) {
      let target: string | null;
      try {
        target = decodeName(await readlink(abs, { encoding: "buffer" }));
      } catch {
        excluded.unreadable += 1;
        note(rel, "unreadable", "file");
        return;
      }
      if (target === null) {
        excluded.non_utf8 += 1;
        note(rel, "non_utf8", "file");
        return;
      }
      const link: ScannedEntry = { path: rel, type: "symlink", size: 0, sha256: null, target, ...statFields(stats) };
      if (list && (await isBrokenLink(abs))) link.broken = true;
      entries.push(link);
      return;
    }
    const entry: ScannedEntry = { path: rel, type: "file", size: Number(stats.size), sha256: null, ...statFields(stats) };
    if (codeCheck) {
      const clean = await readCleanCodeFile(abs, rel, (opened) => isSameFileIdentity(entry, opened));
      if (!clean) {
        excluded.credential += 1;
        note(rel, "credential", "file");
        return;
      }
      // Pass 2 writes exactly the checked bytes.
      entry.content = clean.bytes;
      entry.size = clean.bytes.length;
      entry.sha256 = clean.sha256;
      entries.push(entry);
      return;
    }
    if (isGitConfigPath(rel) && entry.size > MAX_SCRUBBED_CONFIG_BYTES) {
      // Too large to scrub: never archived raw (it could carry credentials). Listed with its size.
      excluded.special += 1;
      note(rel, "too_large", "file", entry.size);
      return;
    }
    if (isGitConfigPath(rel)) {
      // Read and scrubbed in the hash pass; never from the cache (its bytes are needed for pass 2).
      toHash.push(entry);
      entries.push(entry);
      return;
    }
    if (options.maxHashBytes !== undefined && entry.size > options.maxHashBytes) {
      entries.push(entry);
      return;
    }
    const cached = cache?.lookup(entry) ?? null;
    if (cached) {
      entry.sha256 = cached;
      metrics.cacheHits += 1;
    } else {
      toHash.push(entry);
    }
    entries.push(entry);
  };

  // Breadth-first, one directory level at a time: list the level's
  // directories, then lstat every name found, both with bounded concurrency.
  // Each directory carries its parent's ignore scope; listing it works out
  // its own (a nested repository, a .gitignore under the rules fallback).
  let level: Array<{ rel: string; abs: string; scope: IgnoreScope }> = [{ rel: "", abs: resolve(root), scope: { kind: "none" } }];
  let ignoreSource: IgnoreSource = "rules";
  while (level.length > 0) {
    const names: Array<{ relDir: string; absDir: string; name: string; scope: IgnoreScope }> = [];
    await forEachBounded(level, concurrency, async (dir) => {
      signal?.throwIfAborted();
      let raw: Buffer[];
      try {
        raw = await readdir(dir.abs, { encoding: "buffer" });
      } catch (error) {
        if (!dir.rel) throw error;
        // The directory itself is recorded; its content is not.
        excluded.unreadable += 1;
        return;
      }
      const listed: string[] = [];
      let hasGit = false;
      let hasGitignore = false;
      for (const bytes of raw) {
        const name = decodeName(bytes);
        if (name === null) excluded.non_utf8 += 1;
        else if (!dir.rel && (name === RESERVED_ROOT_NAME || name === OUTSIDE_ROOT_NAME || name === ATTACHMENTS_ROOT_NAME || name === ENCLOSING_REPO_ROOT_NAME || name === SANDBOX_ROOT_NAME || name === MEDIA_ROOT_NAME)) {
          excluded.reserved += 1;
          note(name, "reserved", "dir");
        }
        else {
          listed.push(name);
          if (name.toLowerCase() === ".git") hasGit = true;
          else if (name === ".gitignore") hasGitignore = true;
        }
      }
      let scope: IgnoreScope;
      if (dir.rel) scope = await ignore.enter(dir.rel, dir.abs, dir.scope, { git: hasGit, gitignore: hasGitignore });
      else {
        scope = await ignore.rootScope(hasGitignore);
        ignoreSource = ignore.source;
      }
      for (const name of listed) names.push({ relDir: dir.rel, absDir: dir.abs, name, scope });
    });
    const next: Array<{ rel: string; abs: string; scope: IgnoreScope }> = [];
    await forEachBounded(names, concurrency, (item) => inspect(item.relDir, item.absDir, item.name, item.scope, next));
    level = next;
  }

  const unreadable = new Set<ScannedEntry>();
  const buffers: Buffer[] = [];
  await forEachBounded(toHash, options.hashConcurrency ?? HASH_CONCURRENCY, async (entry) => {
    signal?.throwIfAborted();
    if (isGitConfigPath(entry.path)) {
      if (!(await scrubConfigEntry(join(root, ...entry.path.split("/")), entry, metrics))) unreadable.add(entry);
      return;
    }
    const buffer = buffers.pop() ?? Buffer.allocUnsafe(HASH_READ_BYTES);
    try {
      const sha256 = await hashFile(join(root, ...entry.path.split("/")), entry.size, buffer, metrics, signal);
      if (sha256 === null) {
        unreadable.add(entry);
        cache?.forget(entry.path);
        return;
      }
      entry.sha256 = sha256;
      cache?.remember(entry);
    } finally {
      buffers.push(buffer);
    }
  });
  signal?.throwIfAborted();
  excluded.unreadable += unreadable.size;
  for (const entry of unreadable) note(entry.path, "unreadable", "file", entry.size);
  const kept = unreadable.size > 0 ? entries.filter((entry) => !unreadable.has(entry)) : entries;
  kept.sort((left, right) => compareArchivePaths(left.path, right.path));
  cache?.commitScan();
  return { entries: kept, excluded, ignored: ignore.ignored, ignoreSource };
}

// --- delta ----------------------------------------------------------------------

/** Whether `current` counts as modified against `previous` (section 5.8). */
export function isArchiveEntryModified(previous: ArchiveEntry, current: ArchiveEntry): boolean {
  if (previous.type !== current.type || previous.mode !== current.mode) return true;
  if (current.type === "file") return previous.sha256 !== current.sha256;
  if (current.type === "symlink") return previous.target !== current.target;
  return false;
}

export type ArchiveDelta<T extends ArchiveEntry> = { files: T[]; deleted: string[] };

function sortedByPath<T extends ArchiveEntry>(entries: readonly T[]): readonly T[] {
  for (let index = 1; index < entries.length; index += 1) {
    if (compareArchivePaths(entries[index - 1]!.path, entries[index]!.path) >= 0) {
      return [...entries].sort((left, right) => compareArchivePaths(left.path, right.path));
    }
  }
  return entries;
}

/**
 * Added and modified entries of the scan, plus every baseline path that is
 * gone, both sorted. A merge of two sorted lists: no index of either is built.
 */
export function computeArchiveDelta<T extends ArchiveEntry>(baseline: readonly ArchiveEntry[], current: readonly T[]): ArchiveDelta<T> {
  const previous = sortedByPath(baseline);
  const scanned = sortedByPath(current);
  const files: T[] = [];
  const deleted: string[] = [];
  let left = 0;
  let right = 0;
  while (left < previous.length || right < scanned.length) {
    const old = previous[left];
    const now = scanned[right];
    const order = old === undefined ? 1 : now === undefined ? -1 : compareArchivePaths(old.path, now.path);
    if (order < 0) {
      deleted.push(old!.path);
      left += 1;
    } else if (order > 0) {
      files.push(now!);
      right += 1;
    } else {
      if (isArchiveEntryModified(old!, now!)) files.push(now!);
      left += 1;
      right += 1;
    }
  }
  return { files, deleted };
}

export function isArchiveDeltaEmpty(delta: ArchiveDelta<ArchiveEntry>): boolean {
  return delta.files.length === 0 && delta.deleted.length === 0;
}

// --- manifest ---------------------------------------------------------------------

/** The manifest/baseline form of an entry: exactly the section 5.4 fields, in order. */
export function archiveEntry(entry: ArchiveEntry): ArchiveEntry {
  const out: ArchiveEntry = { path: entry.path, type: entry.type, mode: entry.mode, size: entry.size, sha256: entry.sha256 };
  if (entry.type === "symlink") out.target = entry.target ?? "";
  if (entry.type === "symlink" && entry.broken) out.broken = true;
  if (entry.type === "file" && entry.scrubbed) out.scrubbed = true;
  return out;
}

export type ManifestInput = {
  /** Capture v2 chains: ARCHIVE_SCHEMA_V2 (capture-v2.ts). */
  schema?: string;
  kind: ArchiveKind;
  archiveId: string;
  sessionId: string;
  sequence: number;
  turn: number;
  createdAt: Date;
  parentArchiveId: string | null;
  /** Deltas only: a completed turn, or a final archive (then with its reason). */
  trigger?: ArchiveTrigger;
  reason?: ArchiveReason;
  /** Touched-files archives only. */
  scope?: ArchiveScope;
  label: string;
  marker: string;
  git: ArchiveGit | null;
  files: readonly ArchiveEntry[];
  deleted?: readonly string[];
  excluded: ExcludedCounts;
};

/** A manifest that is serialised on demand: its size first, then its bytes while the tar is written. */
export type ManifestSource = { size: number; chunks: () => Iterable<Buffer> };

/**
 * `__omnirush__/manifest.json`. The entries are serialised a batch at a time,
 * once to learn the size the tar header needs and again while the member is
 * written, so a 200k-entry manifest (about 165 bytes per entry) is never held
 * in memory whole. The entries must not change in between.
 */
export function manifestSource(input: ManifestInput): ManifestSource {
  const head = JSON.stringify({
    schema: input.schema ?? ARCHIVE_SCHEMA,
    kind: input.kind,
    archive_id: input.archiveId,
    session_id: input.sessionId,
    sequence: input.sequence,
    turn: input.turn,
    created_at: input.createdAt.toISOString(),
    parent_archive_id: input.parentArchiveId,
    ...(input.trigger ? { trigger: input.trigger } : {}),
    ...(input.reason ? { reason: input.reason } : {}),
    ...(input.scope ? { scope: input.scope } : {}),
    workspace: { label: input.label, marker: input.marker, git: input.git },
  });
  const deleted = input.kind === "delta" ? `,"deleted":${JSON.stringify(input.deleted ?? [])}` : "";
  const tail = `]${deleted},"excluded":${JSON.stringify(input.excluded)}}`;
  function* chunks(): Generator<Buffer> {
    yield Buffer.from(`${head.slice(0, -1)},"files":[`, "utf8");
    for (let index = 0; index < input.files.length; index += 2_000) {
      const batch = input.files.slice(index, index + 2_000).map((entry) => JSON.stringify(archiveEntry(entry)));
      yield Buffer.from(`${index === 0 ? "" : ","}${batch.join(",")}`, "utf8");
    }
    yield Buffer.from(tail, "utf8");
  }
  let size = 0;
  for (const chunk of chunks()) size += chunk.length;
  return { size, chunks };
}

export function buildManifestBytes(input: ManifestInput): Buffer {
  return Buffer.concat([...manifestSource(input).chunks()]);
}

// --- git ------------------------------------------------------------------------

type GitRun = { ok: boolean; stdout: string; truncated: boolean };

/** Runs git in the root without locks, prompts or fsmonitor hooks, never above home; reads at most `maxBytes` of stdout. */
async function runGit(root: string, args: string[], maxBytes = 64 * 1024): Promise<GitRun> {
  // macOS without the Command Line Tools: /usr/bin/git would open the install dialog.
  if (await gitSkipReason()) return { ok: false, stdout: "", truncated: false };
  return runGitNow(root, args, maxBytes);
}

function runGitNow(root: string, args: string[], maxBytes: number): Promise<GitRun> {
  return new Promise((resolvePromise) => {
    const chunks: Buffer[] = [];
    let total = 0;
    let truncated = false;
    let settled = false;
    let child: ReturnType<typeof spawn> | null = null;
    const finish = (ok: boolean) => {
      if (settled) return;
      settled = true;
      clearTimeout(timer);
      resolvePromise({ ok, stdout: Buffer.concat(chunks).toString("utf8"), truncated });
    };
    const timer = setTimeout(() => {
      child?.kill("SIGKILL");
      finish(false);
    }, GIT_TIMEOUT_MS);
    timer.unref?.();
    try {
      child = spawn("git", ["-C", root, "-c", "core.fsmonitor=false", "-c", "core.quotePath=false", ...args], {
        stdio: ["ignore", "pipe", "ignore"],
        windowsHide: true,
        env: {
          ...gitChildEnv(),
          GIT_CEILING_DIRECTORIES: gitCeilingDirectories(),
          GIT_TERMINAL_PROMPT: "0",
          GIT_OPTIONAL_LOCKS: "0",
          GIT_PAGER: "cat",
          PAGER: "cat",
          LC_ALL: "C",
        },
      });
    } catch {
      finish(false);
      return;
    }
    child.stdout?.on("data", (chunk: Buffer) => {
      if (truncated) return;
      const room = maxBytes - total;
      if (chunk.length >= room) {
        chunks.push(chunk.subarray(0, room));
        total = maxBytes;
        truncated = true;
        child?.kill("SIGKILL");
        finish(true);
        return;
      }
      chunks.push(chunk);
      total += chunk.length;
    });
    child.on("error", () => finish(false));
    child.on("close", (code) => finish(code === 0));
  });
}

async function gitLine(root: string, args: string[]): Promise<string | null> {
  const result = await runGit(root, args);
  if (!result.ok || result.truncated) return null;
  return result.stdout.trim() || null;
}

/**
 * workspace.git (section 5.6): HEAD, branch, the origin (else first) remote
 * without userinfo, whether `git status --porcelain` prints anything, and the
 * root's path inside the repository ("" at its top level; a git_parent root
 * is a folder inside it).
 * Null when git cannot read the repository. GIT_OPTIONAL_LOCKS=0 keeps
 * `git status` from rewriting .git/index, which would otherwise show up as a
 * change in the next delta.
 */
export async function readArchiveGit(root: string): Promise<ArchiveGit | null> {
  if (!(await runGit(root, ["rev-parse", "--git-dir"])).ok) return null;
  const [head, branch, remotes, status, prefix] = await Promise.all([
    gitLine(root, ["rev-parse", "--verify", "-q", "HEAD"]),
    gitLine(root, ["symbolic-ref", "--short", "-q", "HEAD"]),
    gitLine(root, ["remote"]),
    runGit(root, ["status", "--porcelain", "--untracked-files=normal"], 1),
    runGit(root, ["rev-parse", "--show-prefix"]),
  ]);
  const names = remotes?.split(/\r?\n/).map((name) => name.trim()).filter(Boolean) ?? [];
  const remoteName = names.includes("origin") ? "origin" : names[0];
  const remoteUrl = remoteName ? await gitLine(root, ["remote", "get-url", remoteName]) : null;
  return {
    head: head && /^[0-9a-f]{40,64}$/.test(head) ? head : null,
    branch,
    remote: remoteUrl ? stripRemoteUserinfo(remoteUrl) : null,
    dirty: status.ok && status.stdout.length > 0,
    ...(prefix.ok && !prefix.truncated ? { path: prefix.stdout.replace(/\r?\n$/, "").replace(/\/$/, "") } : {}),
  };
}