Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.5 KB
Raw Blame
/**
 * #6188 durable state of the fence census: what the candidate finder found,
 * where its scans stopped, and the per-source normalization trend. All of it
 * lives in `op_checkpoints`, so it needs no schema migration.
 *
 * - `fence-candidate`: one row per (source, incarnation, page slug) whose
 *   stored page (`page`) or working-tree file (`file`) fails the coordinated
 *   fence step. Each part is bound to the bytes it judged (sha256, page id,
 *   `updated_at`, file mtime) and carries location and reason codes only.
 * - `fence-scan`: one row per (source, incarnation): the DB watermark and
 *   one-time backfill cursor, the file-walk cursor and the last census.
 * - `fence-trend`: what Tier 1 normalized, per source: one row per sync run
 *   (a managed run's total commits with the checkpoint publication that
 *   completes the run; a legacy run writes its total when it ends) and one
 *   row per normalized page write (written once per request inside its
 *   publication, so a replayed publication never counts twice). Rows carry
 *   fix classes and writers (receipt principal kind or top directory), never
 *   a path or cell value.
 *
 * Candidate and scan rows survive the 7-day `op_checkpoints` purge while
 * their source incarnation lives (`purgeStaleCheckpoints`); trend rows age
 * out with it, which is all a 7-day trend needs.
 */
import type { BrainEngine } from '../engine.ts';
import type { FenceSection, FenceKind, FenceTier } from './types.ts';

type Exec = Pick<BrainEngine, 'executeRaw'>;

export const FENCE_CANDIDATE_OP = 'fence-candidate';
export const FENCE_SCAN_OP = 'fence-scan';
export const FENCE_TREND_OP = 'fence-trend';

/** Where a candidate's first issue sits: location only. */
export interface CandidateLocation { fence: FenceKind; section: FenceSection; rows: number[]; columns: string[]; line: number | null }

/** One judged copy of a page (stored body or working-tree file). */
export interface CandidateFinding {
  tier: FenceTier;
  /** Residual reasons, or the Tier 1 fix classes when the step normalizes it (tier deterministic). */
  reasons: string[];
  location: CandidateLocation | null;
  /** sha256 of the judged sections (pages) or file bytes (files). */
  sha256: string;
  checked_at: string;
}
export interface PageFinding extends CandidateFinding { page_id: number; updated_at: string; content_hash: string | null; source_path: string | null }
export interface FileFinding extends CandidateFinding { path: string; mtime_ms: number }

export interface CandidateRecord {
  version: 1;
  source_id: string;
  incarnation: string;
  /** The page slug the stored page or file maps to. */
  key: string;
  page?: PageFinding;
  file?: FileFinding;
}

/** The DB scan: incremental pages-updated watermark plus a one-time keyset backfill over `pages.id`. */
export interface PageScanState {
  /** Lower bound (timestamptz text) of the next incremental pass; null until the backfill starts it. */
  watermark: string | null;
  /** An unfinished incremental pass: rows with `updated_at >= lower`, after `(at, id)`; `next` becomes the watermark when it ends. */
  pass: { lower: string; next: string; after: { at: string; id: number } | null } | null;
  backfill: { after_id: number; done: boolean; started_at: string; completed_at?: string };
}

/** The working-tree walk: a full first walk, then only files changed since the last walked commit (or mtime) plus dirty files. */
export interface FileScanState {
  /** An unfinished pass: `full` lists every eligible file, `changed` only what changed since `walked`; `after` is the last path judged. */
  pass: { mode: 'full' | 'changed'; after: string | null; started_at: string; commit: string | null } | null;
  /** The last completed pass: HEAD when it started (null outside Git) and its start time (the mtime watermark). */
  walked: { commit: string | null; at: string } | null;
}

export interface ScanRecord {
  version: 1;
  source_id: string;
  incarnation: string;
  /** Fence rules version the stored findings were judged with; a newer one restarts both scans. */
  rules: string;
  pages: PageScanState;
  files: FileScanState | null;
  /** The last census: complete when the backfill is done, the incremental pass caught up and the file walk finished. */
  census: { complete: boolean; fresh_at: string | null; checked_at: string };
}

const candidateFingerprint = (sourceId: string, incarnation: string, key: string) => `${sourceId}:${incarnation}:${key}`;
const scanFingerprint = (sourceId: string, incarnation: string) => `${sourceId}:${incarnation}`;

export async function readScanRecord(engine: Exec, sourceId: string, incarnation: string): Promise<ScanRecord | null> {
  const [row] = await engine.executeRaw<{ record: ScanRecord }>('SELECT completed_keys->0 AS record FROM op_checkpoints WHERE op=$1 AND fingerprint=$2',
    [FENCE_SCAN_OP, scanFingerprint(sourceId, incarnation)]);
  return row?.record ?? null;
}

export async function writeScanRecord(engine: Exec, record: ScanRecord): Promise<void> {
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES($1,$2,$3::text::jsonb,now())
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=EXCLUDED.completed_keys,updated_at=now()`,
  [FENCE_SCAN_OP, scanFingerprint(record.source_id, record.incarnation), JSON.stringify([record])]);
}

/** Stores one judged part (`page` or `file`) of a candidate, keeping the other part. */
export async function upsertCandidatePart(engine: Exec, ids: { sourceId: string; incarnation: string; key: string },
  part: 'page' | 'file', finding: PageFinding | FileFinding): Promise<void> {
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at)
      VALUES($1,$2,jsonb_build_array(jsonb_build_object('version',1,'source_id',$3::text,'incarnation',$4::text,'key',$5::text,$6::text,$7::text::jsonb)),now())
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=jsonb_build_array((op_checkpoints.completed_keys->0)||jsonb_build_object($6::text,$7::text::jsonb)),updated_at=now()`,
  [FENCE_CANDIDATE_OP, candidateFingerprint(ids.sourceId, ids.incarnation, ids.key), ids.sourceId, ids.incarnation, ids.key, part, JSON.stringify(finding)]);
}

/** Drops the `part` of these candidates (they judged clean); a candidate with no part left is deleted. */
export async function clearCandidateParts(engine: Exec, ids: { sourceId: string; incarnation: string }, part: 'page' | 'file', keys: readonly string[]): Promise<void> {
  if (!keys.length) return;
  const fingerprints = keys.map(key => candidateFingerprint(ids.sourceId, ids.incarnation, key));
  const other = part === 'page' ? 'file' : 'page';
  await engine.executeRaw(`DELETE FROM op_checkpoints WHERE op=$1 AND fingerprint=ANY($2::text[]) AND (completed_keys->0->$3::text) IS NULL`,
    [FENCE_CANDIDATE_OP, fingerprints, other]);
  await engine.executeRaw(`UPDATE op_checkpoints SET completed_keys=jsonb_build_array((completed_keys->0)-$3::text),updated_at=now()
    WHERE op=$1 AND fingerprint=ANY($2::text[]) AND (completed_keys->0->$3::text) IS NOT NULL`, [FENCE_CANDIDATE_OP, fingerprints, part]);
}

/** Candidate records of live source incarnations (optionally one source). */
export async function readCandidateRecords(engine: Exec, sourceIds?: readonly string[]): Promise<CandidateRecord[]> {
  const rows = await engine.executeRaw<{ record: CandidateRecord }>(`SELECT c.completed_keys->0 AS record FROM op_checkpoints c
    JOIN sources s ON s.id=c.completed_keys->0->>'source_id' AND s.incarnation::text=c.completed_keys->0->>'incarnation'
    WHERE c.op=$1 AND ($2::text[] IS NULL OR s.id=ANY($2::text[])) ORDER BY c.fingerprint`, [FENCE_CANDIDATE_OP, sourceIds ? [...sourceIds] : null]);
  return rows.map(row => row.record);
}

/** One trend row: files or pages Tier 1 normalized, by fix class and by writer. */
export interface TrendEntry { day: string; count: number; by_class: Record<string, number>; writers: Record<string, number> }

/**
 * One sync run's total, keyed by run id (a rewrite replaces it, so a run is
 * counted once); the row keeps the UTC day of its first write.
 */
export async function recordSyncRunTrend(engine: Exec, input: { sourceId: string; runId: string; day: string; count: number;
  byClass: Record<string, number>; writers: Record<string, number> }): Promise<void> {
  if (!input.count) return;
  const record = { version: 1, source_id: input.sourceId, day: input.day, count: input.count, by_class: input.byClass, writers: input.writers };
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES($1,$2,$3::text::jsonb,now())
    ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=jsonb_build_array((EXCLUDED.completed_keys->0)||jsonb_build_object('day',op_checkpoints.completed_keys->0->'day')),updated_at=now()`,
  [FENCE_TREND_OP, `${input.sourceId}:run:${input.runId}`, JSON.stringify([record])]);
}

/** One normalized page write (put_page, put_pages, remember, takes and fact writers), written once per request. */
export async function recordWriteTrend(engine: Exec, input: { sourceId: string; requestKey: string; day: string; byClass: Record<string, number>; writer: string }): Promise<void> {
  const record = { version: 1, source_id: input.sourceId, day: input.day, count: 1, by_class: input.byClass, writers: { [input.writer]: 1 } };
  await engine.executeRaw(`INSERT INTO op_checkpoints(op,fingerprint,completed_keys,updated_at) VALUES($1,$2,$3::text::jsonb,now()) ON CONFLICT(op,fingerprint) DO NOTHING`,
    [FENCE_TREND_OP, `${input.sourceId}:write:${input.requestKey}`, JSON.stringify([record])]);
}

/**
 * Writes the trend row of a committed page write whose outcome reports a
 * normalized fence (the page-write `fences_normalized` object). Managed sync
 * outcomes (a per-file fix list) are skipped: the sync run flushes its own
 * total once per run.
 */
export async function recordPublicationFenceTrend(tx: Exec, row: { id: string; source_id: string; principal_kind: string }, outcome: Record<string, unknown>,
  now: Date = new Date()): Promise<void> {
  const report = outcome.fences_normalized as { count?: unknown; by_class?: Record<string, number> } | undefined;
  if (!report || Array.isArray(report) || typeof report !== 'object' || !Number(report.count)) return;
  await recordWriteTrend(tx, { sourceId: row.source_id, requestKey: row.id, day: now.toISOString().slice(0, 10), byClass: report.by_class ?? {}, writer: row.principal_kind });
}

/** Trend rows of these sources since `fromDay` (inclusive), summed per source and day. */
export async function readTrend(engine: Exec, sourceIds: readonly string[], fromDay: string): Promise<Map<string, TrendEntry[]>> {
  const out = new Map<string, TrendEntry[]>();
  if (!sourceIds.length) return out;
  const rows = await engine.executeRaw<{ record: TrendEntry & { source_id: string } }>(`SELECT completed_keys->0 AS record FROM op_checkpoints
    WHERE op=$1 AND completed_keys->0->>'source_id'=ANY($2::text[]) AND completed_keys->0->>'day'>=$3`, [FENCE_TREND_OP, [...sourceIds], fromDay]);
  for (const { record } of rows) {
    const days = out.get(record.source_id) ?? [];
    let day = days.find(entry => entry.day === record.day);
    if (!day) days.push(day = { day: record.day, count: 0, by_class: {}, writers: {} });
    day.count += Number(record.count ?? 0);
    for (const [cls, n] of Object.entries(record.by_class ?? {})) day.by_class[cls] = (day.by_class[cls] ?? 0) + Number(n);
    for (const [writer, n] of Object.entries(record.writers ?? {})) day.writers[writer] = (day.writers[writer] ?? 0) + Number(n);
    out.set(record.source_id, days);
  }
  for (const days of out.values()) days.sort((a, b) => a.day.localeCompare(b.day));
  return out;
}