Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
11.6 KB
Raw Blame
/**
 * #6712 PR 1 (L1): the content-addressed discovery prescreen and the drift baseline of each frozen import.
 *
 * Import provenance (sync-holds.ts) records, per page, the bytes the last publish left on disk (`disk_blob_oid`), the
 * page revision it wrote (`revision`) and the parser fingerprint (`parser`). Discovery compares them with the pinned
 * tree (one `ls-tree`) and the frozen page revision, for every Git import of an existing page:
 *
 * - **prescreened**: the file is the recorded disk blob, the page is still at the recorded revision and the fingerprint
 *   is this binary's. The entry is only a candidate: the run sends it to the guarded no-op kernel's bounded checks
 *   (`waivePrescreenedRun`), and only those skip it durably; any mismatch there imports it as before. Bypassed under
 *   `--full`.
 * - **no baseline concern**: the page is still at the recorded revision (only the file or the parser changed), so the
 *   import cannot overwrite a database-only edit.
 * - **drift** (`entry.drift`): the page moved since the record and the file did not (`database`), or both moved
 *   (`both`), or there is no usable record (`unknown`: none, one written before PR 0, or a partial one). `database` and
 *   `both` are guarded in every discovery: a Git delta proves the file changed, never that the page did not (#6712: an
 *   incremental sync published a committed edit over a database-only `put_page`). `unknown` is guarded in a tree walk
 *   (first sync, `--full`, an unreachable checkpoint, a re-discovery), where no Git delta says the file is the side that
 *   changed. A `both` entry whose page still holds what the recorded disk blob imports (the database changed only what an
 *   import keeps: tags, links) imports (sync-screen.ts `driftBaselineContent`). A page whose current revision a
 *   committed managed-sync request of this incarnation wrote, or whose writer's write-through wrote to its file
 *   (`pages.revision_write_request_id`, `fileBackedRevisions`), has no database-only edit to protect and is not guarded. A guarded entry is imported only when the freeze proves the import changes
 *   nothing (`sameContentAtOrigin`); otherwise it is held `file_drift` (sync-run.ts `driftHold`).
 * - Under `storage.keep_db` prefixes a drifted page is not held: its entry leaves the manifest (nothing imported) and is
 *   recorded as a drift candidate the keep_db worker reconciles from the database (`liveFileDrift` confirms the file
 *   and the database still differ; an entry where they agree stays, and its import is a no-op).
 */
import { realpathSync } from 'node:fs';
import { join, relative, sep } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import { loadActivePackForEngine } from '../schema-pack/engine-resolution.ts';
import { isKeepDbSlug, loadStorageConfig } from '../storage-config.ts';
import { readSyncImportProvenances, syncImportParserFor, type SyncImportParser, type SyncImportProvenance } from './sync-holds.ts';
import { pinnedWindowBlock, readTreeBlobs, seedPinnedWindow, type TreeBlob } from './sync-blobs.ts';
import { syncOriginPath } from './sync-origin.ts';
import { liveFileDrift, recordDriftSkip } from './drift-conflicts.ts';
import type { SyncDiscovery, SyncEntry } from './sync-discovery.ts';

/** Which side moved since the page's last recorded import (`unknown`: no usable record). */
export type SyncDriftSide = 'database' | 'both' | 'unknown';

/** Provenance rows read per statement. */
const PROVENANCE_CHUNK = 5000;

/** Whether a recorded parser fingerprint is this binary's for the page (the prescreen and its kernel run both check it). */
export const sameParser = (a: SyncImportParser | null | undefined, b: SyncImportParser) => !!a && a.parser_version === b.parser_version
  && a.markdown_chunker === b.markdown_chunker && (a.schema_pack ?? null) === b.schema_pack;

/** The drift side of one entry from its record and pinned blob, or null when nothing needs guarding (`prescreened`: a kernel candidate). */
export function classifyBaseline(entry: Pick<SyncEntry, 'revision' | 'sourcePath'>, record: SyncImportProvenance | undefined, blob: TreeBlob | undefined,
  expected: SyncImportParser, input: { full: boolean; treeWalk: boolean }): 'prescreened' | SyncDriftSide | null {
  const complete = !!record?.disk_blob_oid && record.revision != null;
  const fileSame = complete && !!blob && record!.disk_blob_oid === blob.oid;
  const databaseSame = complete && String(record!.revision) === String(entry.revision);
  if (fileSame && databaseSame && !input.full && sameParser(record!.parser, expected) && syncOriginPath(record!.origin) === syncOriginPath(entry.sourcePath)) return 'prescreened';
  if (databaseSame) return null;
  if (fileSame) return 'database';
  if (complete) return 'both';
  return input.treeWalk ? 'unknown' : null;
}

/**
 * Marks the frozen manifest's prescreen candidates (`prescreened`) and guarded imports (`drift`), and moves the drifted
 * entries under `storage.keep_db` prefixes to the keep_db worker. `held` paths (an active hold) are left to their
 * re-screen. Runs after identities are frozen; a company-brain manifest is never prescreened.
 */
export async function prescreenSyncEntries(engine: BrainEngine, discovered: SyncDiscovery,
  input: { full: boolean; treeWalk: boolean; noSchemaPack: boolean; dryRun: boolean; held: ReadonlyMap<string, { code: string }> }): Promise<void> {
  if (discovered.companyPlan) return;
  const candidates = discovered.entries.filter(entry => entry.action === 'import' && !entry.working && entry.pageId != null && entry.revision != null
    && !entry.renameFrom && !entry.renameHeld);
  if (!candidates.length) return;
  const nativeRoot = realpathSync.native(discovered.root), nativeGitRoot = realpathSync.native(discovered.gitRoot);
  const gitPathOf = (entry: SyncEntry) => relative(nativeGitRoot, join(nativeRoot, entry.path)).split(sep).join('/');
  // One listing answers the candidates and seeds the run's first pinned window (the manifest's first block of committed
  // imports), so the prescreen adds no `ls-tree` to the run.
  const block = pinnedWindowBlock(0), first = discovered.entries.slice(block.start, block.end).filter(entry => entry.action === 'import' && !entry.working).map(gitPathOf);
  const blobs = readTreeBlobs(discovered.gitRoot, discovered.target, [...new Set([...first, ...candidates.map(gitPathOf)])]);
  seedPinnedWindow(discovered.gitRoot, discovered.target, first, blobs);
  const records = new Map<number, SyncImportProvenance>();
  for (let i = 0; i < candidates.length; i += PROVENANCE_CHUNK) {
    const read = await readSyncImportProvenances(engine, discovered.sourceId, discovered.incarnation, candidates.slice(i, i + PROVENANCE_CHUNK).map(entry => Number(entry.pageId)));
    for (const [id, record] of read) records.set(id, record);
  }
  const pack = input.noSchemaPack ? null : await loadActivePackForEngine(engine, { remote: false, sourceId: discovered.sourceId }).catch(() => null);
  const guarded: SyncEntry[] = [];
  for (const entry of candidates) {
    const hold = input.held.get(entry.path);
    const side = classifyBaseline(entry, records.get(Number(entry.pageId)), blobs.get(gitPathOf(entry)), syncImportParserFor(entry.sourcePath, pack?.identity ?? null),
      { full: input.full || !!hold, treeWalk: input.treeWalk || hold?.code === 'file_drift' });
    if (side === 'prescreened') entry.prescreened = true;
    else if (side) { entry.drift = side; if (side === 'both') entry.driftBaseline = records.get(Number(entry.pageId))!.disk_blob_oid!; guarded.push(entry); }
  }
  if (!guarded.length) return;
  const backed = await fileBackedRevisions(engine, discovered, guarded.map(entry => Number(entry.pageId)));
  const drifted = guarded.filter(entry => backed.has(Number(entry.pageId)) ? (delete entry.drift, delete entry.driftBaseline, false) : true);
  await releaseKeepDbDrift(engine, discovered, drifted, input.dryRun);
}

/**
 * `file_drift` holds whose page moved since the hold recorded it (another database write, a reconcile, a repair): each
 * is re-screened like a hold whose file changed. One read for the source's drift holds.
 */
export async function movedDriftHolds(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string,
  holds: ReadonlyArray<{ path: string; code: string; page_id: number | null; meta: { page_revision?: string } }>): Promise<Set<string>> {
  const drift = holds.filter(hold => hold.code === 'file_drift' && hold.page_id !== null);
  if (!drift.length) return new Set();
  const rows = await engine.executeRaw<{ id: number | string; revision: string | null }>('SELECT id, knowledge_revision::text AS revision FROM pages WHERE source_id=$1 AND id=ANY($2::bigint[])',
    [sourceId, drift.map(hold => hold.page_id)]);
  const revisions = new Map(rows.map(row => [Number(row.id), row.revision]));
  return new Set(drift.filter(hold => (revisions.get(Number(hold.page_id)) ?? null) !== (hold.meta.page_revision ?? null)).map(hold => hold.path));
}

/**
 * Pages whose current revision has no database-only edit to protect: a committed managed-sync request of this
 * incarnation wrote it (the file was its input), or its writer's publication wrote that revision to the page's own
 * origin file (write-through: a `git` effect with the written bytes' hash, whatever its state). Either way the file held
 * the page at that revision, so a file that differs now changed in Git afterwards.
 */
async function fileBackedRevisions(engine: BrainEngine, discovered: Pick<SyncDiscovery, 'sourceId' | 'incarnation'>, pageIds: number[]): Promise<Set<number>> {
  const out = new Set<number>();
  for (let i = 0; i < pageIds.length; i += PROVENANCE_CHUNK) {
    const rows = await engine.executeRaw<{ id: number | string }>(`SELECT p.id FROM pages p JOIN persistence_requests r ON r.id=p.revision_write_request_id
      WHERE p.source_id=$1 AND p.id=ANY($2::bigint[]) AND r.state='committed' AND r.source_id=p.source_id AND (
        r.intent->>'kind' IN ('managed_sync_import','managed_sync_delete') AND r.source_incarnation::text=$3 AND (r.page_id IS NULL OR r.page_id=p.id)
        OR EXISTS (SELECT 1 FROM persistence_effects e WHERE e.request_id=r.id AND e.kind='git' AND e.revision=p.knowledge_revision
          AND e.data->>'expected_hash' IS NOT NULL AND (e.data->>'relative_path'=p.source_path OR right(e.data->>'relative_path', length(p.source_path)+1)='/'||p.source_path)))`, [discovered.sourceId, pageIds.slice(i, i + PROVENANCE_CHUNK), discovered.incarnation]);
    for (const row of rows) out.add(Number(row.id));
  }
  return out;
}

/**
 * `storage.keep_db`: a drifted page under its prefixes is reconciled from the database by the keep_db worker, never
 * imported or held by sync. Its entry leaves the manifest and is recorded as a drift candidate while the file and the
 * database still differ (or the check cannot tell); where they agree the entry stays and imports as a no-op.
 */
async function releaseKeepDbDrift(engine: BrainEngine, discovered: SyncDiscovery, drifted: SyncEntry[], dryRun: boolean): Promise<void> {
  if (!drifted.length) return;
  let keepDb: string[] = [];
  try { keepDb = loadStorageConfig(discovered.root)?.keep_db ?? []; } catch { return; }
  const released = new Set<SyncEntry>();
  for (const entry of drifted.filter(entry => keepDb.length && isKeepDbSlug(entry.slug!, keepDb))) {
    const live = await liveFileDrift(engine, discovered.sourceId, entry.slug!);
    if (live.still_present === false) { delete entry.drift; continue; }
    if (!dryRun) await recordDriftSkip(engine, discovered.sourceId, entry.slug!, { file_hash: live.file_hash ?? null, db_revision: live.db_revision ?? entry.revision ?? null });
    released.add(entry);
  }
  if (released.size) discovered.entries = discovered.entries.filter(entry => !released.has(entry));
}