Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.7 KB
Raw Blame
/**
 * GBRA-217 (A6, A7): page change reasons read back from Git. A commit gbrain makes, and an outside commit made with
 * `git commit --trailer "gbrain-reason: <path>: <text>"`, carries one `gbrain-reason` trailer per path. Sync mirrors
 * them into page history (`page_versions.change_reason` on the snapshot the import archives); purge finds the commits
 * whose reason quotes a purged claim and tombstones them (`reason_purge_tombstones`, text-free), so no later import
 * brings that reason back.
 *
 * One reader serves both: a single streaming `git log` over the first-parent line (merges diffed against their first
 * parent, so a merge's own trailer decides for a conflict resolution and a side-branch commit never decides), with no
 * pathspec and no grep on the command line, parsed as it streams and stopped at a commit cap.
 */
import { realpathSync } from 'node:fs';
import { basename, dirname, join, relative, sep } from 'node:path';
import { hardenedGitStream, hardenedGitSync, REASON_LOG_OPTIONS } from './hardened-git.ts';
import { canonicalChangeReason, CHANGE_REASON_MAX } from './persistence/change-reason.ts';
import { scanTextSpans } from './secret-scan.ts';
import type { BrainEngine } from './engine.ts';
import { engineKey } from './engine-key.ts';
import type { Notice } from './agent-output.ts';
import type { VersionHistory } from './page-state/versions.ts';

export const SYNC_REASON_SCAN_CAP = 10_000;
export const PURGE_REASON_SCAN_CAP = 200_000;
const SCAN_TIMEOUT_MS = 120_000;
/** The most bytes one `-z` token (a commit's trailer block, a path) is buffered for; a longer one is dropped unread. */
export const REASON_TOKEN_MAX_BYTES = 1024 * 1024;
const OID = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;

export interface TrailerCommit { sha: string; values: string[]; paths: string[] }
export type TrailerRead = { status: 'complete' | 'capped'; commits: number } | { status: 'failed'; commits: number; detail: string };

let caps = { sync: SYNC_REASON_SCAN_CAP, purge: PURGE_REASON_SCAN_CAP };
let gitStream = hardenedGitStream;
let scans = 0;
/** Test seams: smaller caps, a replacement Git runner, and the number of scans run (no argument restores each). */
export function __setReasonScanCapsForTests(next?: Partial<typeof caps>): void { caps = { sync: SYNC_REASON_SCAN_CAP, purge: PURGE_REASON_SCAN_CAP, ...next }; }
export function __setReasonGitStreamForTests(runner?: typeof hardenedGitStream): void { gitStream = runner ?? hardenedGitStream; }
export function __reasonScansForTests(): number { return scans; }
export function reasonScanCap(kind: 'sync' | 'purge'): number { return caps[kind]; }

class CapReached extends Error {}

/**
 * Streams `git log` over `revisions` (a range `a..b`, or one commit for its whole history), newest first along the
 * first parent, calling `visit` for each commit up to `cap`. Output is `-z`: each commit is an empty token, its OID,
 * its trailer values (one per line, folded lines joined), then its paths (the first prefixed by Git's blank line). A token
 * over REASON_TOKEN_MAX_BYTES is not buffered: a trailer block that long gives its commit no values, a path that long is
 * skipped, and the records after it still parse.
 */
export async function readReasonTrailers(gitRoot: string, revisions: string, cap: number, visit: (commit: TrailerCommit) => void): Promise<TrailerRead> {
  scans++;
  let state: 'sha' | 'trailers' | 'paths' = 'sha';
  let current: TrailerCommit | null = null;
  let commits = 0;
  let pending: Buffer[] = [];
  let pendingBytes = 0;
  const flush = () => { if (current) visit(current); current = null; };
  const token = (text: string | null) => {
    if (state === 'paths' && text === '') { flush(); state = 'sha'; return; }
    if (state === 'sha') {
      if (text === '') return;
      if (text === null || !OID.test(text)) throw new Error(`unexpected git log record ${text === null ? 'over the token bound' : JSON.stringify(text.slice(0, 60))}`);
      if (++commits > cap) throw new CapReached();
      current = { sha: text, values: [], paths: [] };
      state = 'trailers';
      return;
    }
    if (state === 'trailers') { current!.values = text === null ? [] : text.split('\n').map(line => line.trim()).filter(Boolean); state = 'paths'; return; }
    if (text !== null) current!.paths.push(current!.paths.length === 0 && text.startsWith('\n') ? text.slice(1) : text);
  };
  const take = (piece: Buffer) => token(pendingBytes + piece.length > REASON_TOKEN_MAX_BYTES ? null
    : (pending.length ? Buffer.concat([...pending, piece]) : piece).toString('utf8'));
  const onChunk = (chunk: Buffer) => {
    let start = 0;
    for (let end = chunk.indexOf(0); end >= 0; end = chunk.indexOf(0, start)) {
      take(chunk.subarray(start, end));
      pending = [];
      pendingBytes = 0;
      start = end + 1;
    }
    if (start === chunk.length) return;
    pendingBytes += chunk.length - start;
    if (pendingBytes > REASON_TOKEN_MAX_BYTES) pending = []; else pending.push(chunk.subarray(start));
  };
  try {
    const result = await gitStream(gitRoot, ['log', ...REASON_LOG_OPTIONS, '--end-of-options', revisions, '--'], { timeoutMs: SCAN_TIMEOUT_MS, onChunk });
    if (!result.ok) return { status: 'failed', commits, detail: `git log ${result.reason}${result.status === undefined ? '' : ` ${result.status}`}` };
    if (pendingBytes) take(Buffer.alloc(0));
    flush();
    return { status: 'complete', commits };
  } catch (error) {
    if (error instanceof CapReached) return { status: 'capped', commits: cap };
    return { status: 'failed', commits, detail: error instanceof Error ? error.message : String(error) };
  }
}

/** The spellings of a repository-relative path a trailer may use: itself, and (under a scoped source) relative to the scope. */
export function pathForms(scopePrefix: string): (path: string) => string[] {
  const prefix = scopePrefix && !scopePrefix.endsWith('/') ? `${scopePrefix}/` : scopePrefix;
  return path => prefix && path.startsWith(prefix) ? [path, path.slice(prefix.length)] : [path];
}

/**
 * Each trailer value of `commit` by the path it names (`<path>: <text>`). A path may itself hold ": ", so the longest
 * spelling of a path the commit touched that prefixes a value owns it; a value naming no such path is malformed.
 */
export function trailerTexts(commit: TrailerCommit, forms: (path: string) => string[]): { byPath: Map<string, string>; malformed: number } {
  const byPath = new Map<string, string>();
  let malformed = 0;
  for (const value of commit.values) {
    let best: { path: string; length: number } | null = null;
    for (const path of commit.paths) {
      for (const form of forms(path)) {
        if (value.startsWith(`${form}: `) && (!best || form.length > best.length)) best = { path, length: form.length };
      }
    }
    if (!best) { malformed++; continue; }
    if (!byPath.has(best.path)) byPath.set(best.path, value.slice(best.length + 2));
  }
  return { byPath, malformed };
}

export interface ReasonDecision { commit: string; reason: string | null; commits: number }
export interface SyncReasonScan {
  status: 'complete' | 'capped' | 'failed';
  /** The repository's top level, which the scanned paths are relative to. */
  gitRoot: string;
  /** The range as `git log` names it (`<from>..<to>`, or `<to>` for its whole history). */
  range: string;
  /** Repository-relative path -> the newest first-parent commit that touched it, its reason, and how many commits touched it. */
  byPath: Map<string, ReasonDecision>;
  malformed: number;
  secrets: Array<{ commit: string; path: string }>;
  detail?: string;
}


/**
 * A6: one scan per sync pass. For each path, the newest first-parent commit in the range that touched it decides: its
 * trailer for that path, or NULL when it has none, so an older reason is never stamped on newer text. Each reason is
 * canonicalised and secret-scanned (a secret, an over-long value or a tombstoned (commit, path) gives NULL); a value
 * naming no path its commit touched is malformed and counted. A failed or capped scan maps nothing.
 */
export async function scanSyncReasons(engine: BrainEngine, input: { sourceId: string; sourceRoot: string; gitRoot?: string; from: string | null; to: string }): Promise<SyncReasonScan | null> {
  const repo = input.gitRoot ? knownGitTopLevel(input.gitRoot, input.sourceRoot) : gitTopLevel(input.sourceRoot);
  if (!repo) return null;
  const range = input.from ? `${input.from}..${input.to}` : input.to;
  const byPath = new Map<string, ReasonDecision>();
  const secrets: SyncReasonScan['secrets'] = [];
  let malformed = 0;
  const tombstones = new Set((await engine.executeRaw<{ commit_sha: string; path: string }>(
    'SELECT commit_sha, path FROM reason_purge_tombstones WHERE source_id = $1', [input.sourceId])).map(row => `${row.commit_sha}\0${row.path}`));
  const forms = pathForms(repo.scopePrefix);
  const read = await readReasonTrailers(repo.gitRoot, range, caps.sync, commit => {
    const texts = trailerTexts(commit, forms);
    malformed += texts.malformed;
    for (const path of commit.paths) {
      const seen = byPath.get(path);
      if (seen) { seen.commits++; continue; }
      const raw = texts.byPath.get(path);
      let reason = raw === undefined ? null : canonicalChangeReason(raw);
      if (reason !== null && [...reason].length > CHANGE_REASON_MAX) { reason = null; malformed++; }
      if (reason !== null && scanTextSpans(reason).some(span => span.finding !== null)) { reason = null; secrets.push({ commit: commit.sha, path }); }
      if (reason !== null && tombstones.has(`${commit.sha}\0${path}`)) reason = null;
      byPath.set(path, { commit: commit.sha, reason, commits: 1 });
    }
  });
  if (read.status === 'complete') return { status: 'complete', gitRoot: repo.gitRoot, range, byPath, malformed, secrets };
  return { status: read.status, gitRoot: repo.gitRoot, range, byPath: new Map(), malformed: 0, secrets: [], ...(read.status === 'failed' ? { detail: read.detail } : {}) };
}

type GitTopLevel = { status: 'found'; gitRoot: string; scopePrefix: string } | { status: 'not_repository' } | { status: 'failed'; detail: string };

/**
 * Where `sourceRoot` sits in Git: its top level and the source's path under it (both real paths); `not_repository` only
 * when Git itself says the folder is in no repository (exit 128, "not a git repository"); anything else, a missing path,
 * no `git`, an unsafe or corrupt repository, a timeout, is `failed`.
 */
function locateGitTopLevel(sourceRoot: string): GitTopLevel {
  let root: string;
  try { root = realpathSync(sourceRoot); } catch (error) { return { status: 'failed', detail: `source path unreadable: ${(error as NodeJS.ErrnoException).code ?? 'error'}` }; }
  const top = hardenedGitSync(root, ['rev-parse', '--show-toplevel'], { timeoutMs: 10_000, maxBytes: 64 * 1024, stderr: true });
  if (top.ok) {
    const gitRoot = realpathSync(top.stdout.toString('utf8').trim());
    return { status: 'found', gitRoot, scopePrefix: relative(gitRoot, root).split(sep).join('/') };
  }
  if (top.reason === 'exit' && top.status === 128 && /not a git repository/i.test(top.stderr ?? '')) return { status: 'not_repository' };
  return { status: 'failed', detail: `git rev-parse --show-toplevel ${top.reason}${top.status === undefined ? '' : ` ${top.status}`}` };
}

/**
 * Sync's view: the Git top level holding `sourceRoot`, or null when it is in no Git repository (a plain folder source,
 * a repo-less brain) or Git cannot place it; such a pass runs no scan and imports as before.
 */
export function gitTopLevel(sourceRoot: string): { gitRoot: string; scopePrefix: string } | null {
  const top = locateGitTopLevel(sourceRoot);
  return top.status === 'found' ? { gitRoot: top.gitRoot, scopePrefix: top.scopePrefix } : null;
}

/**
 * GBRA-133: a classic sync pass already discovered its checkout's top level (real path) without blocking, so its scan places
 * `sourceRoot` under it and runs no synchronous `git rev-parse` on the resident's path; null when the source path is gone.
 */
function knownGitTopLevel(gitRoot: string, sourceRoot: string): { gitRoot: string; scopePrefix: string } | null {
  try { return { gitRoot, scopePrefix: relative(gitRoot, realpathSync(sourceRoot)).split(sep).join('/') }; } catch { return null; }
}

/** A file's path relative to the scan's Git top level, resolving its directory's real path (the file itself may be gone). */
function repoPathOf(scan: SyncReasonScan, filePath: string): string {
  let dir = dirname(filePath);
  try { dir = realpathSync(dir); } catch { /* the lexical path is the best guess */ }
  return relative(scan.gitRoot, join(dir, basename(filePath))).split(sep).join('/');
}

/**
 * What the snapshot an import of one file (absolute path) archives carries: the deciding reason with the commit and path it
 * came from (the snapshot insert rechecks that pair's tombstone, #8635), and a history gap when
 * the range held more than one commit for the file or the scan could not say. Nothing when the pass ran no scan (no Git
 * checkout).
 */
export function importHistoryFor(scan: SyncReasonScan | null | undefined, filePath: string): VersionHistory | undefined {
  if (!scan) return undefined;
  if (scan.status !== 'complete') return { reason: null, historyGap: true };
  const path = repoPathOf(scan, filePath);
  const decision = scan.byPath.get(path);
  return { reason: decision?.reason ?? null, ...(decision?.reason ? { reasonSource: { commit: decision.commit, path } } : {}), historyGap: (decision?.commits ?? 0) > 1 };
}

/** The pass's notices: a failed or capped scan (reasons stored as NULL, no backfill), and secrets found in pushed trailers. */
export function reasonScanNotices(scan: SyncReasonScan | null | undefined, sourceId: string): Notice[] {
  if (!scan) return [];
  const notices: Notice[] = [];
  if (scan.status !== 'complete') {
    const what = scan.status === 'capped' ? `reached its limit of ${caps.sync} commits` : `failed (${scan.detail ?? 'git error'})`;
    notices.push({ code: scan.status === 'capped' ? 'reason_scan_capped' : 'reason_scan_failed', kind: 'degraded',
      why: `Source ${sourceId} synced, but reading change reasons from Git for ${scan.range} ${what}. Content was imported; every reason in this pass is stored as empty and each imported page is marked as having a history gap. `
        + 'This release does not backfill reasons for that range, and a later sync will not rescan it. The reasons are still in Git.',
      fix: { argv: ['git', 'log', '--first-parent', '--format=%H %(trailers:key=gbrain-reason,valueonly)', scan.range], consent: [], actor: 'user', requires_exclusive: false,
        why: 'Lists the reasons Git holds for the range this pass could not read; read-only, nothing in the brain changes.' } });
  }
  if (scan.secrets.length) {
    const named = scan.secrets.slice(0, 10).map(hit => `${hit.commit.slice(0, 12)} ${hit.path}`).join(', ');
    const more = scan.secrets.length > 10 ? `, and ${scan.secrets.length - 10} more` : '';
    notices.push({ code: 'reason_trailer_secret', kind: 'safety',
      why: `A gbrain-reason trailer already in Git looks like it holds a secret (${named}${more}); that reason was not stored. Git history keeps the commit message, so rotate the credential.`,
      user_message: `A commit message in source ${sourceId} appears to contain a credential (${named}${more}). Rotate it: Git history keeps the commit text.` });
  }
  return notices;
}

export const SOURCE_PURGE_EPOCH_SQL = 'SELECT purge_epoch FROM sources WHERE id = $1';
let caches = new WeakMap<object, Map<string, { at: number; scan: Promise<SyncReasonScan | null> }>>();
const CACHE_TTL_MS = 10 * 60_000;

/**
 * Managed sync prepares one request at a time, each naming its pass's range: the pass's scan runs once per process and
 * its later requests reuse it. The key carries the source's purge epoch, which every fact purge bumps in the transaction
 * that writes its tombstones, so a purge from any process makes the next request scan again and read them; a request
 * prepared before the purge is caught when its snapshot is inserted (`createPageVersion`). The cache is per engine: the scan holds that brain's tombstones, so another brain on the same checkout never reuses it.
 */
export async function cachedSyncReasonScan(engine: BrainEngine, input: Parameters<typeof scanSyncReasons>[1]): Promise<SyncReasonScan | null> {
  const [source] = await engine.executeRaw<{ purge_epoch: string | number }>(SOURCE_PURGE_EPOCH_SQL, [input.sourceId]);
  const key = [input.sourceId, String(source?.purge_epoch ?? 0), input.sourceRoot, input.from ?? '', input.to].join('\0');
  const brain = engineKey(engine);
  const cache = caches.get(brain) ?? new Map();
  caches.set(brain, cache);
  const hit = cache.get(key);
  if (hit && Date.now() - hit.at < CACHE_TTL_MS) return hit.scan;
  const scan = scanSyncReasons(engine, input);
  cache.set(key, { at: Date.now(), scan });
  scan.catch(() => cache.delete(key));
  while (cache.size > 16) cache.delete(cache.keys().next().value!);
  return scan;
}
export function clearReasonScanCache(): void { caches = new WeakMap(); }

/** A7: what a purge's trailer scan found: the (commit, path) pairs whose reason quotes the claim, read up to HEAD. */
export type PurgeTrailerScan =
  | { status: 'no_repository' }
  | { status: 'complete'; gitRoot: string; head: string | null; hits: Array<{ commit: string; path: string }>; commits: number; limited: boolean }
  | { status: 'failed'; detail: string }
  | { status: 'capped'; cap: number };

/**
 * A7: the commits on the first-parent line of HEAD (`from..HEAD` for a recheck) whose `gbrain-reason` trailer quotes one of
 * `needles` (case-insensitive, both canonicalised as a stored reason is: whitespace runs and control characters folded),
 * each with the path its trailer names: the touched path it prefixes, else the text before its first ": ". Capped at
 * the purge cap, or at `limit` when the user chose one (then older commits are not read).
 */
export async function scanTrailersCarrying(sourceRoot: string, needles: readonly string[], opts: { from?: string | null; limit?: number } = {}): Promise<PurgeTrailerScan> {
  const repo = locateGitTopLevel(sourceRoot);
  if (repo.status === 'not_repository') return { status: 'no_repository' };
  if (repo.status === 'failed') return { status: 'failed', detail: repo.detail };
  const head = hardenedGitSync(repo.gitRoot, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], { timeoutMs: 10_000, maxBytes: 1024 });
  if (!head.ok) return head.reason === 'exit' && head.status === 1 ? { status: 'complete', gitRoot: repo.gitRoot, head: null, hits: [], commits: 0, limited: false }
    : { status: 'failed', detail: `git rev-parse ${head.reason}` };
  const tip = head.stdout.toString('utf8').trim();
  if (opts.from === tip) return { status: 'complete', gitRoot: repo.gitRoot, head: tip, hits: [], commits: 0, limited: false };
  const lowered = needles.map(needle => canonicalChangeReason(needle)?.toLowerCase() ?? '').filter(Boolean);
  const forms = pathForms(repo.scopePrefix);
  const hits: Array<{ commit: string; path: string }> = [];
  const cap = opts.limit ?? caps.purge;
  const read = await readReasonTrailers(repo.gitRoot, opts.from ? `${opts.from}..${tip}` : tip, cap, commit => {
    for (const value of commit.values) {
      const lower = canonicalChangeReason(value)?.toLowerCase() ?? '';
      if (!lowered.some(needle => lower.includes(needle))) continue;
      let best: { path: string; length: number } | null = null;
      for (const path of commit.paths) for (const form of forms(path)) {
        if (value.startsWith(`${form}: `) && (!best || form.length > best.length)) best = { path, length: form.length };
      }
      hits.push({ commit: commit.sha, path: best?.path ?? value.split(': ')[0]! });
    }
  });
  if (read.status === 'failed') return { status: 'failed', detail: read.detail };
  if (read.status === 'capped' && opts.limit === undefined) return { status: 'capped', cap };
  return { status: 'complete', gitRoot: repo.gitRoot, head: tip, hits, commits: read.commits, limited: read.status === 'capped' };
}