Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
35.1 KB
Raw Blame
/**
 * v0.32.2 — markdown-first fact write path.
 *
 * The "system of record" invariant means new facts land in the entity
 * page's `## Facts` fence FIRST, then the DB index gets stamped via
 * engine.insertFacts. After this commit, every write path that wants
 * persistent fact storage routes through `writeFactsToFence`. The DB
 * single-row `engine.insertFact` stays in the surface for the
 * legacy / thin-client fallback only (when the brain has no
 * sources.local_path configured).
 *
 * Concurrency: reuses the v0.28 page-lock primitive
 * (`src/core/page-lock.ts`), an FS-level lockfile under
 * `~/.gbrain/page-locks/<sha256-of-slug>.lock` with heartbeat-recency
 * staleness (5-minute TTL; namespace-agnostic — no PID-liveness, #2840).
 * Multi-process safe — two `gbrain` invocations writing
 * to the same entity page serialize through the same kernel-visible
 * lockfile. 5-second timeout per the plan's "5s retry" failure mode.
 *
 * Atomicity: write the fence to `<file>.tmp`, re-parse the .tmp body,
 * THEN `renameSync` to the canonical file. If parse fails the .tmp
 * stays in place as quarantine evidence and the JSONL surface
 * (`facts.write_failures.jsonl`) records the failure for `gbrain
 * doctor` to surface. The on-disk markdown file is never corrupted
 * mid-write (renameSync is atomic on POSIX) and the DB is never
 * inserted when the fence isn't valid (Codex Q7 atomic-write
 * recovery).
 *
 * No re-entrancy needed: writeFactsToFence uses fs.writeFileSync +
 * renameSync directly — NOT engine.putPage — so no code path can
 * re-trigger runFactsBackstop on the markdown write. The architecture
 * self-prevents the recursion concern Codex Q7 raised; documenting
 * here so a future refactor that swaps writeFileSync for putPage
 * sees the constraint.
 */

import { execFileSync } from '../spawn.ts';
import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, appendFileSync, rmSync } from 'node:fs';
import { dirname, relative } from 'node:path';

import type { BrainEngine, NewFact, FactVisibility, FactKind, FactAttribution } from '../engine.ts';
import type { ResolutionSource } from '../entities/resolve.ts';
import { inferTypeFromPack, parseMarkdown } from '../markdown.ts';
import { yamlScalar } from '../frontmatter-inference.ts';
import { loadActivePackBestEffort } from '../schema-pack/best-effort.ts';
import { withPageLock } from '../page-lock.ts';
import { assertSourceFilesystemActive, hasSourceFilesystemLock, withSourceFilesystemLock } from '../minions/source-filesystem.ts';
import { gbrainPath } from '../config.ts';
import { isWriteThroughDisabled, resolvePageWriteTarget } from '../write-through.ts';
import { isDurabilityHardened, commitWriteThroughFile } from '../brain-repo-durability.ts';
import { upsertFactRow, parseFactsFence, formatFenceDate } from '../facts-fence.ts';
import { withHeldFactsFence } from './unfenced-facts.ts';
import { unparsedFenceRefusal } from '../fence-repair/refusal.ts';
import type { OperationError } from '../ops/contract.ts';
import { extractFactsFromFenceText } from './extract-from-fence.ts';
import { logStubGuardEvent } from './stub-guard-audit.ts';
import { isFactWithdrawn } from './withdrawal.ts';
import { managedPersistenceEnabled } from '../persistence/ownership.ts';
import { maintenanceTransaction } from '../persistence/attribution.ts';
import { withPageTierKept } from '../trust/fence-append.ts';
import { recordTaintEdges } from '../trust/taint.ts';
import { recordFlaggedRow, type GatedRowDecision } from '../write-gate-store.ts';
import type { TaintInput, WriteTrust } from '../trust/tier.ts';
import { gitChildEnv } from '../git-env.ts';
import { readOnlyCheckout } from '../persistence/mirror-read-only.ts';
import { bodyUnread } from '../page-body/page-shape.ts';
import { bestEffortStoredBodyMirror, fileBodyMatchesStored, fileDatabaseDriftToken, mirrorFileBody, readStoredBody } from './file-body-guard.ts';
import { relativeEscapesRoot } from '../path-confine.ts';

/** Resolved source binding for the entity page. */
export interface FenceTarget {
  /** Source primary key, e.g. 'default'. */
  sourceId: string;
  /** Filesystem root for this source. Null when the brain is read-only / thin-client. */
  localPath: string | null;
  /** Entity slug — also becomes source_markdown_slug + the file basename. */
  slug: string;
  /**
   * #4108: how `slug` was resolved. REQUIRED (not optional) so no caller can
   * silently skip provenance: 'fallback_slugify' and null (resolver returned
   * nothing / caller has no resolution step) are blocked from stub-creating a
   * page — a fallback-minted slug names an entity nothing verified exists.
   * 'exact_page' / 'alias_exact' / 'fuzzy_match' all verified a live page, so
   * stub-create (DB↔file drift repair) stays allowed for them.
   */
  resolutionSource: ResolutionSource | null;
  /** Automatic extraction (the facts backstop): a claim withdrawn at any visibility is skipped, not only at the fact's own. */
  automatic?: boolean;
}

/** Input fact prepared by runPipelineWithBody (post-dedup). */
export interface FenceInputFact {
  embedding_model?: string | null;
  fact: string;
  kind: NewFact['kind'];
  notability: NewFact['notability'];
  source: string;
  context?: string | null;
  visibility: FactVisibility;
  /** Defaults to 1.0 when undefined (matches engine.insertFact behavior). */
  confidence?: number;
  validFrom?: Date;
  /**
   * MEMORY_VERBS v1 (c5): remember's ttl → valid_until. Written to the
   * fence cell losslessly (formatFenceDate); the DB column derives from it on
   * the stamp step.
   * Undefined/null = never expires (pre-v1 behavior unchanged).
   */
  validUntil?: Date | null;
  embedding: Float32Array | null;
  sessionId: string | null;
  /** Speaker attribution; written to the fence's attributed_to cell. */
  attributedTo?: FactAttribution;
  /** #5575 B3: the deriver's write-gate decision (an `insert`); a flag records its receipt on the new row. */
  gate?: GatedRowDecision;
}

export interface FenceWriteResult {
  /** Number of new rows written + indexed. */
  inserted: number;
  /** DB ids assigned to the inserted rows, in input order. */
  ids: number[];
  /** True when the path fell through to DB-only: local_path unset, `sync.write_through` off, or a read-only checkout (`databaseOnlyReason`). */
  legacyFallback?: true;
  /** Why a `legacyFallback` is expected: `mirror_read_only` (#7098) or `url_clone` (#6946, a clone gbrain owns and pulls), so the checkout is never written. */
  databaseOnlyReason?: 'mirror_read_only' | 'url_clone' | 'file_database_drift';
  /** A held file mirror's typed diagnostic and repair instructions; facts take the DB-only fallback. */
  warning?: string;
  /** True when fence parse-validate failed; rows were NOT inserted, .tmp quarantined. */
  fenceWriteFailed?: true;
  /**
   * #6385 R12: set with `fenceWriteFailed` when the page's existing facts
   * fence does not parse cleanly, so appending (a re-render) would drop the
   * rows the parser skipped. Nothing was written (no .tmp); this is the typed
   * `invalid_fence` / `target_fence_malformed` refusal for the caller to throw.
   */
  fenceRefusal?: OperationError;
  /**
   * True when the stub-creation guard refused to spawn a phantom entity
   * page — either for an unprefixed bare slug (e.g. `jared` with no
   * `people/` directory), or (#4108) for a slug whose resolutionSource is
   * 'fallback_slugify'/null, i.e. a slug the resolver invented rather than
   * verified. Rows were NOT inserted; the caller is expected to route the
   * facts to the legacy DB-only path so they aren't silently dropped.
   *
   * The unprefixed arm is the v0.34.5 fix for the entity-resolution bug
   * where `"Jared"` fell through resolution and produced a top-level
   * `jared.md` stub.
   */
  stubGuardBlocked?: true;
  /**
   * Input facts dropped because their claim is withdrawn for this page's
   * entity (write-path audit B-10): a withdrawn claim is never appended to
   * the Markdown fence as an active row.
   */
  withdrawnSkipped?: number;
  /**
   * True when the shared page-target resolver could not produce a usable
   * fence file path (source tree missing / not a directory, or a hostile
   * recorded `source_path` escaping the tree). Rows were NOT inserted; the
   * caller is expected to route the facts to the legacy DB-only path so
   * they aren't silently dropped. Unlike the old blind `mkdir -p`, we do
   * NOT resurrect a deleted source tree just to hold a fence — the same
   * refusal writePageThrough applies (#2018 `repo_not_found`).
   */
  targetUnresolvable?: true;
}

const FAILURE_LOG_PATH = (): string => gbrainPath('facts.write_failures.jsonl');

function recordWriteFailure(slug: string, sourceId: string, warnings: string[], filePath: string): void {
  // Best-effort JSONL append — never throws back into the caller. The
  // log is the operator-visibility surface; `gbrain doctor` reads it
  // to surface facts.write_failures.
  try {
    const dir = dirname(FAILURE_LOG_PATH());
    mkdirSync(dir, { recursive: true });
    const line = JSON.stringify({
      ts: new Date().toISOString(),
      slug,
      source_id: sourceId,
      file_path: filePath,
      warnings,
    });
    appendFileSync(FAILURE_LOG_PATH(), `${line}\n`, 'utf-8');
  } catch (err) {
    // eslint-disable-next-line no-console
    console.warn(`[facts.write_failures] couldn't append: ${err instanceof Error ? err.message : String(err)}`);
  }
}

/**
 * #6385 R12: rows are appended by re-rendering the fence, which would drop
 * every row the parser skipped, so a fence that parses with warnings is
 * refused before anything is written (file, rows and DB stay as they are).
 * Returns the typed refusal (recorded to the failure log), or null.
 */
function refuseUnparsedFence(body: string, target: FenceTarget, filePath: string): OperationError | null {
  if (parseFactsFence(body).warnings.length === 0) return null;
  const refusal = unparsedFenceRefusal(parseMarkdown(body, `${target.slug}.md`), 'facts', target.slug, target.sourceId);
  recordWriteFailure(target.slug, target.sourceId, [`fence_input_unparsed: ${refusal.message}`], filePath);
  return refusal;
}

export type FactFenceGitPathState = 'clean' | 'self_dirty' | 'foreign_dirty' | 'unknown';

export function gitPathState(repoPath: string, filePath: string): FactFenceGitPathState {
  try {
    const rel = relative(repoPath, filePath);
    if (!rel || relativeEscapesRoot(rel)) return 'unknown';
    const status = execFileSync(
      'git',
      ['-C', repoPath, 'status', '--porcelain=v1', '--untracked-files=all', '--', rel],
      { encoding: 'utf-8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 10_000, env: gitChildEnv() },
    );
    const lines = status.split('\n').filter((l) => l.length > 0);
    if (lines.length === 0) return 'clean';
    // Distinguish dirt that IS the target fence file (safe for the
    // path-limited commit to sweep — a prior gbrain fence-commit that failed
    // leaves exactly this shape) from genuinely foreign dirt: unmerged
    // conflict states, rename entries naming a second path, or any entry the
    // parse can't positively attribute to `rel` (git quotes special chars).
    for (const line of lines) {
      const xy = line.slice(0, 2);
      if (xy.includes('U') || xy === 'AA' || xy === 'DD') return 'foreign_dirty';
      const pathField = line.slice(3);
      if (pathField !== rel && pathField !== `"${rel}"`) return 'foreign_dirty';
    }
    return 'self_dirty';
  } catch {
    return 'unknown';
  }
}

export async function commitFactFenceFile(
  repoPath: string,
  filePath: string,
  slug: string,
  sourceId: string,
  prewriteState: FactFenceGitPathState,
): Promise<void> {
  // Self-dirt does NOT block the commit: a prior fence-commit failure
  // (index.lock contention, kill mid-commit) leaves the fence file itself
  // dirty, and refusing on that shape latched the page's durability off
  // permanently. The locked read-modify-write already incorporated the
  // file's pre-write content, and the commit below is path-limited to this
  // one file, so sweeping self-dirt is the recovery — only genuinely foreign
  // dirt (or an unreadable state) keeps the audit-and-skip behavior.
  if (prewriteState === 'foreign_dirty' || prewriteState === 'unknown') {
    recordWriteFailure(
      slug,
      sourceId,
      [prewriteState === 'foreign_dirty'
        ? 'git_durability_preexisting_dirty'
        : 'git_durability_prewrite_state_unknown'],
      filePath,
    );
    return;
  }

  for (const delayMs of [0, 50, 200] as const) {
    if (delayMs > 0) {
      await new Promise(resolve => setTimeout(resolve, delayMs));
    }
    if (commitWriteThroughFile(repoPath, filePath, slug) && gitPathState(repoPath, filePath) === 'clean') {
      return;
    }
  }

  recordWriteFailure(slug, sourceId, ['git_durability_commit_failed'], filePath);
}

/**
 * Stub-create body for a new entity page. Minimum frontmatter so the
 * page validates as gbrain-canonical markdown and survives an
 * `importFromFile` round-trip. Type inferred from slug prefix
 * (e.g. `people/alice` → 'person'); unknown prefixes fall back to
 * 'concept' which is the most permissive PageType.
 */
export function stubEntityPage(
  slug: string,
  pack: Parameters<typeof inferTypeFromPack>[1] | null,
): string {
  // #4322: resolve the type through the ACTIVE PACK, not a hardcoded table.
  // The previous people/companies/deals/topics ternary shadowed every other
  // pack-declared prefix, so a stub under a declared prefix such as
  // `products/` was written as `concept` even though the pack maps that
  // prefix to `company` — manufacturing prefix/type mismatches in brains
  // that were otherwise fully pack-conformant, and (because `concept` skips
  // the facts backstop) silently opting those pages out of the very
  // subsystem that created them.
  //
  // A null pack means the load failed. Per best-effort.ts's contract we do
  // NOT substitute an ad-hoc table here; passing an empty pack routes
  // inferTypeFromPack to its own documented GBRAIN_BASE_PATH_PREFIXES
  // fallback, the same base behaviour every other ingest path degrades to.
  const type = inferTypeFromPack(slug, pack ?? { page_types: [] });
  const tail = slug.split('/').slice(1).join('/');
  const title = tail
    .replace(/[-_/]+/g, ' ')
    .replace(/\b\w/g, c => c.toUpperCase()) || slug;
  return `---\ntype: ${yamlScalar(type)}\ntitle: ${yamlScalar(title)}\nslug: ${yamlScalar(slug)}\n---\n\n# ${title}\n`;
}

/**
 * Run a markdown-first fence write for one entity. Acquires the page
 * lock, reads or stub-creates the file, appends each input fact to
 * the `## Facts` fence, atomically renames the .tmp into place, and
 * stamps the DB index via engine.insertFacts.
 *
 * Returns `legacyFallback: true` when `target.localPath` is null (also
 * under `sync.write_through` off and, with `databaseOnlyReason`, for a
 * read-only mirror (#7098) or a clone gbrain owns, #6946) —
 * the caller is responsible for falling through to the legacy
 * DB-only `engine.insertFact` path. We don't do the legacy fallback
 * here because the caller has the FactsBackstopCtx (visibility,
 * session, supersede policy) that the fence path doesn't need but
 * the legacy path does.
 *
 * Returns `fenceWriteFailed: true` when parse-validation of the
 * just-written .tmp fails. In that case the .tmp stays on disk as
 * quarantine evidence, the JSONL failure log records the warnings,
 * and the DB is NOT touched. The caller treats this as a hard
 * failure on the page (no rows inserted, no duplicate count, no
 * fact_ids). The same flag, with `fenceRefusal` and no .tmp, means the
 * page's existing fence does not parse cleanly (#6385 R12).
 */
export async function writeFactsToFence(
  engine: BrainEngine,
  target: FenceTarget,
  facts: FenceInputFact[],
  /** #5575 I2: a deriver's taint; the new rows get its tier and input edges. Unmanaged path only. */
  derivation?: { trust: WriteTrust; inputs: readonly TaintInput[] },
): Promise<FenceWriteResult> {
  if (await managedPersistenceEnabled(engine)) {
    // The coordinator owns the canonical file on a managed brain: publish the
    // fence rows through its fact intent instead of editing the file here.
    const kept: FenceInputFact[] = [];
    for (const f of facts) {
      if (!await isFactWithdrawn(engine, target.sourceId, target.automatic ? null : f.visibility, f.fact, target.slug)) kept.push(f);
    }
    const withdrawnSkipped = facts.length - kept.length ? { withdrawnSkipped: facts.length - kept.length } : {};
    if (!kept.length) return { inserted: 0, ids: [], ...withdrawnSkipped };
    const { publishManagedEntityFacts } = await import('./managed-fact-write.ts');
    const { inserted, ids } = await publishManagedEntityFacts(engine, target.sourceId, target.slug, kept);
    return { inserted, ids, ...withdrawnSkipped };
  }
  if (target.localPath === null) {
    return { inserted: 0, ids: [], legacyFallback: true };
  }
  if (facts.length === 0) {
    return { inserted: 0, ids: [] };
  }
  // `sync.write_through` off values make the brain DB-only by operator
  // choice: no fence file, no stub entity page, no git commit. Same
  // legacyFallback contract as a missing local_path — the caller's DB-only
  // path still records the facts.
  if (await isWriteThroughDisabled(engine)) return { inserted: 0, ids: [], legacyFallback: true };
  const readOnly = await readOnlyCheckout(engine, target.sourceId); if (readOnly) return { inserted: 0, ids: [], legacyFallback: true, databaseOnlyReason: readOnly };

  // #4204: compute the SAME path writePageThrough computes for this
  // (source, slug) — the fence appends to the page's file, so the two writers
  // must agree. The previous resolvePageFilePath routing nested any
  // non-default source under `<local_path>/.sources/<id>/` — but every
  // non-default source that reaches this line has its OWN `local_path`
  // (callers fall back to the legacy DB-only path when `sources.local_path`
  // is NULL), and write-through/scanOneSource put that topology's pages at
  // the tree ROOT. Sync's walker skips dot-directories, so a `.sources/`
  // fence was invisible to sync and the next extract_facts reconcile deleted
  // the fence-owned DB rows. The shared resolver also prefers the page's
  // recorded `source_path`, so the fence lands in the file of record instead
  // of minting a slug-derived twin beside a human-named vault file.
  const resolved = await resolvePageWriteTarget(engine, target.slug, target.sourceId);
  if (!resolved.ok) {
    // Target tree unusable (deleted dir, hostile source_path row, …) — the
    // caller routes the facts to the legacy DB-only path so they are
    // recorded, not dropped.
    return { inserted: 0, ids: [], targetUnresolvable: true };
  }
  const { filePath, writeRoot } = resolved;
  if (!hasSourceFilesystemLock(writeRoot)) {
    return withSourceFilesystemLock(engine, writeRoot, () => writeFactsToFence(engine, target, facts, derivation));
  }
  const tmpPath = `${filePath}.tmp`;
  const durabilityEnabled = isDurabilityHardened(writeRoot);

  return withPageLock(
    target.slug,
    async () => {
      const kept: FenceInputFact[] = [];
      for (const f of facts) {
        if (!await isFactWithdrawn(engine, target.sourceId, target.automatic ? null : f.visibility, f.fact, target.slug)) kept.push(f);
      }
      const withdrawnSkipped = facts.length - kept.length ? { withdrawnSkipped: facts.length - kept.length } : {};
      facts = kept;
      if (!facts.length) return { inserted: 0, ids: [], ...withdrawnSkipped };

      // 1. Read existing body or stub-create.
      let body: string;
      const fromFile = existsSync(filePath);
      const stored = fromFile ? await readStoredBody(engine, target.sourceId, target.slug) : null;
      const drift = (when: 'before' | 'during'): FenceWriteResult => {
        const warning = fileDatabaseDriftToken(target.slug, target.sourceId, when);
        console.warn(`[facts.fence] ${warning}`);
        return { inserted: 0, ids: [], legacyFallback: true, databaseOnlyReason: 'file_database_drift', warning, ...withdrawnSkipped };
      };
      if (fromFile) {
        body = readFileSync(filePath, 'utf-8');
        const refusal = refuseUnparsedFence(body, target, filePath);
        if (refusal) return { inserted: 0, ids: [], fenceWriteFailed: true, fenceRefusal: refusal, ...withdrawnSkipped };
        const page = await engine.getPage(target.slug, { sourceId: target.sourceId });
        if (page && bodyUnread(page)) {
          const warning = `${target.slug} (page_body_segmented: its file is not compared or rewritten here; facts are recorded database-only)`;
          console.warn(`[facts.fence] ${warning}`);
          return { inserted: 0, ids: [], legacyFallback: true, warning, ...withdrawnSkipped };
        }
        if (stored ? !fileBodyMatchesStored(body, target.slug, stored) : page !== null) return drift('before');
      } else {
        // Stub-creation guard, two arms:
        //
        // 1. Unprefixed slug (v0.34.5). Phantom entity pages at the brain
        //    root were being spawned when resolveEntitySlug fell through to
        //    a bare slugify because pg_trgm scored too low on short bare
        //    names. The resolver now has a prefix-expansion step that
        //    catches most of those, but this arm is the second wall: refuse
        //    to stub-create a page whose slug has no directory prefix
        //    (people/, companies/, deals/, topics/, etc.).
        //
        // 2. Fallback/absent resolution provenance (#4108). A PREFIXED
        //    fallback_slugify result (e.g. "companies/zeta-widgets" for an
        //    entity no page backs) sailed past arm 1 and materialized as a
        //    canonical stub page; after sync it resolved as exact_page,
        //    closing a fallback→stub→exact-match feedback loop. Blocklist
        //    shape on purpose: only 'fallback_slugify' and null are blocked,
        //    so future ResolutionSource members that verify a live page
        //    (like v0.46.15's 'alias_exact') fence without touching this.
        //
        // Either way the caller routes these facts to the legacy DB-only
        // path so they aren't silently dropped — the fact still gets
        // recorded (entity_slug retained), it just doesn't spawn a phantom
        // entity page on disk.
        //
        // Sunset target: v0.36, for arm 1 ONLY (the 'unprefixed' reason in
        // the audit log). Once `stub_guard_24h` (the gbrain doctor surface
        // backed by the audit log written here) reads <5 unprefixed
        // hits/week for 3 consecutive weeks on production brains, the
        // prefix-expansion in resolveEntitySlug is sufficient and arm 1 can
        // be removed. Arm 2 does NOT sunset: it is the only wall between
        // resolver-invented slugs and canonical page creation, and no
        // resolver improvement can retire it (the fallback floor is by
        // design). The audit log under
        // `~/.gbrain/audit/stub-guard-YYYY-Www.jsonl` is the operator
        // visibility surface, with per-arm `reason` fields.
        const fallbackResolved =
          target.resolutionSource === 'fallback_slugify' || target.resolutionSource == null;
        if (!target.slug.includes('/') || fallbackResolved) {
          logStubGuardEvent({
            slug: target.slug,
            source_id: target.sourceId,
            fact_count: facts.length,
            reason: !target.slug.includes('/') ? 'unprefixed' : 'fallback_resolution',
          });
          // eslint-disable-next-line no-console
          console.warn(
            !target.slug.includes('/')
              ? `[facts] refusing to stub-create unprefixed entity page slug=${target.slug} — routing to legacy DB-only path. Provide a directory prefix (people/, companies/, etc.) to opt into fence writes.`
              : `[facts] refusing to stub-create entity page slug=${target.slug} from a fallback-resolved reference (no live page verified) — routing to legacy DB-only path.`,
          );
          return { inserted: 0, ids: [], stubGuardBlocked: true, ...withdrawnSkipped };
        }
        const dbOnly = await dbOnlyPageRoute(engine, target, facts.length); if (dbOnly) return { ...dbOnly, ...withdrawnSkipped };
        // Stub-create the parent directory if it doesn't exist.
        mkdirSync(dirname(filePath), { recursive: true });
        const activePack = await loadActivePackBestEffort({ engine } as never);
        body = stubEntityPage(target.slug, activePack?.manifest ?? null);
      }

      const fileBefore = fromFile ? body : null;

      // 2. Upsert each fact onto the fence in input order. row_num
      //    monotonically increases (max-existing + 1 per call, append-only).
      //
      //    Seed the counter from the DB as well as the fence file. Uniqueness
      //    is enforced by idx_facts_fence_key on
      //    (source_id, source_markdown_slug, row_num) in Postgres, but
      //    upsertFactRow derives the next value from the fence in the markdown
      //    alone — and falls back to 1 when the file has no fence at all. Any
      //    write path that rewrites a page without preserving its facts fence
      //    (put_page write-through, sync, dream-cycle reverse-render) therefore
      //    resets the counter below what the DB already holds, and the next
      //    absorb re-issues a row_num that is already taken. That surfaces as
      //    "duplicate key value violates unique constraint idx_facts_fence_key"
      //    and the whole batch of facts is dropped.
      //
      //    Symptom in the wild: a page whose fence had been rewritten away had
      //    24 facts in the DB and none in the file, so every subsequent absorb
      //    on it failed permanently. Taking the max of both sources keeps the
      //    file as the readable mirror while the DB stays authoritative about
      //    which row_nums have been issued.
      //
      //    Degrades to the previous file-only behaviour if the lookup fails
      //    (pre-v51 brain without the fence columns, or a transient DB error):
      //    a fence write must not become impossible just because the counter
      //    hint is unavailable. The degradation is reported, never silent,
      //    because file-only numbering is the duplicate-key class above.
      let dbMaxRowNum = 0;
      try {
        const rows = await engine.executeRaw<{ max_row_num: number | null }>(
          `SELECT MAX(row_num) AS max_row_num FROM facts
            WHERE source_id = $1 AND source_markdown_slug = $2`,
          [target.sourceId, target.slug],
        );
        dbMaxRowNum = Number(rows[0]?.max_row_num ?? 0);
      } catch (err) {
        console.warn(`[facts.fence] FACTS_ROW_NUM_HINT_UNAVAILABLE: ${target.slug} (source ${target.sourceId}): ${err instanceof Error ? err.message : String(err)}; numbering from the file alone`);
      }
      body = (await withHeldFactsFence(engine, target.sourceId, target.slug, { compiled_truth: body })).compiled_truth;
      const fenceRefusal = refuseUnparsedFence(body, target, filePath);
      if (fenceRefusal) return { inserted: 0, ids: [], fenceWriteFailed: true, fenceRefusal, ...withdrawnSkipped };
      const { facts: existingFenceFacts } = parseFactsFence(body);
      const fileMaxRowNum = existingFenceFacts.length > 0
        ? Math.max(...existingFenceFacts.map(f => f.rowNum))
        : 0;
      let nextRowNum = Math.max(fileMaxRowNum, dbMaxRowNum) + 1;

      const assignedRowNums: number[] = [];
      for (const f of facts) {
        const { body: updated, rowNum } = upsertFactRow(body, {
          rowNum:      nextRowNum++,
          claim:       f.fact,
          kind:        (f.kind ?? 'fact') as FactKind,
          confidence:  f.confidence ?? 1.0,
          visibility:  f.visibility,
          notability:  f.notability ?? 'medium',
          validFrom:   formatFenceDate(f.validFrom ?? new Date()),
          // MEMORY_VERBS v1 (c5): remember's ttl threads through to the fence
          // cell — was hard-coded undefined, which silently dropped expiry on
          // this path. extractFactsFromFenceText derives the DB column from it.
          validUntil:  f.validUntil ? formatFenceDate(f.validUntil) : undefined,
          source:      f.source,
          context:     f.context ?? undefined,
          ...(f.attributedTo ? { attributedTo: f.attributedTo } : {}),
        });
        body = updated;
        assignedRowNums.push(rowNum);
      }

      // Snapshot the prewrite git state INSIDE the lock, immediately before
      // the write: an out-of-lock snapshot raced concurrent fence writers — a
      // waiter observed the holder's not-yet-committed rename as pre-existing
      // dirt and mis-attributed it in the audit.
      const durabilityPrewriteState: FactFenceGitPathState = durabilityEnabled
        ? gitPathState(writeRoot, filePath)
        : 'clean';

      // 3. Atomic write: .tmp first, then parse-validate, then rename.
      assertSourceFilesystemActive();
      writeFileSync(tmpPath, body, 'utf-8');

      // 4. Parse-before-rename: re-read the .tmp content and verify the
      //    fence is well-formed. Anything malformed → leave .tmp in
      //    place as quarantine, write JSONL, do NOT insert to DB.
      const tmpBody = readFileSync(tmpPath, 'utf-8');
      const parsed = parseFactsFence(tmpBody);
      if (parsed.warnings.length > 0) {
        recordWriteFailure(target.slug, target.sourceId, parsed.warnings, filePath);
        return { inserted: 0, ids: [], fenceWriteFailed: true };
      }

      const allExtracted = extractFactsFromFenceText(parsed.facts, target.slug, target.sourceId);
      const newRowSet = new Set(assignedRowNums);
      const toInsert = allExtracted.filter(r => newRowSet.has(r.row_num));

      // Carry per-input embedding + sessionId across — the fence
      // parser doesn't reconstruct embeddings (they're not in the
      // fence text) and source_session is runtime provenance that
      // isn't a fence column either. Stitch them back by row_num
      // index.
      const enriched = toInsert.map((row, i) => ({
        ...row,
        embedding:      facts[i].embedding,
        embedding_model: facts[i].embedding_model,
        source_session: facts[i].sessionId,
      }));

      const result = await maintenanceTransaction(engine, async tx => {
        if (fileBefore === null ? existsSync(filePath) : !existsSync(filePath) || readFileSync(filePath, 'utf-8') !== fileBefore) return null;
        if (stored && !await bestEffortStoredBodyMirror(tx, target.sourceId, target.slug, stored,
          savepoint => withPageTierKept(savepoint, target, () => mirrorFileBody(savepoint, target.sourceId, target.slug, tmpBody, stored)),
          err => console.warn(`[facts.fence] FACTS_PAGE_MIRROR_FAILED: ${target.slug} (source ${target.sourceId}): ${err instanceof Error ? err.message : String(err)}; page cache stale until gbrain sync --source ${target.sourceId}`))) return null;
        if (!stored && await readStoredBody(tx, target.sourceId, target.slug, true)) return null;
        const inserted = await insertFenceRows(tx, target.sourceId, enriched, facts, derivation);
        if (fileBefore === null ? existsSync(filePath) : !existsSync(filePath) || readFileSync(filePath, 'utf-8') !== fileBefore) {
          throw new FenceFileChanged();
        }
        assertSourceFilesystemActive();
        renameSync(tmpPath, filePath);
        return inserted;
      }).catch(err => {
        if (err instanceof FenceFileChanged) return null;
        throw err;
      });
      if (!result) {
        rmSync(tmpPath, { force: true });
        return drift('during');
      }
      // v0.46 (#3014) — an unresolvable `superseded by #N` reference (self
      // / dangling / struck target) leaves superseded_by NULL; log it rather
      // than swallow it. The row still lands (expired_at set for struck
      // rows), never a bad FK.
      for (const w of result.warnings) {
        // eslint-disable-next-line no-console
        console.warn(`[facts.supersession] ${w}`);
      }
      if (durabilityEnabled) {
        await commitFactFenceFile(
          writeRoot,
          filePath,
          target.slug,
          target.sourceId,
          durabilityPrewriteState,
        );
      }
      return { inserted: result.inserted, ids: result.ids, ...withdrawnSkipped };
    },
    { timeoutMs: 5_000 },
  );
}

class FenceFileChanged extends Error {}

/**
 * The DB stamp of new fence rows in one attributed transaction: at the
 * deriver's tier when it declared one, with input edges, and the flag receipt
 * of a row whose gate decision flagged it (`facts[i]` is the input of row i).
 */
async function insertFenceRows(engine: BrainEngine, sourceId: string, rows: Parameters<BrainEngine['insertFacts']>[0], facts: FenceInputFact[],
  derivation?: { trust: WriteTrust; inputs: readonly TaintInput[] }) {
  return maintenanceTransaction(engine, async tx => {
    const inserted = await tx.insertFacts(rows, { source_id: sourceId }); // gbrain-allow-direct-insert: writeFactsToFence is the markdown-first reconcile path; runs only after the atomic fence write commits
    for (const [i, id] of inserted.ids.entries()) {
      if (derivation) await recordTaintEdges(tx, { table: 'facts', id, sourceId }, derivation.inputs);
      if (facts[i]?.gate && inserted.ids.length === facts.length) await recordFlaggedRow(tx, facts[i].gate!, { table: 'facts', id, sourceId });
    }
    return inserted;
  }, derivation?.trust);
}

/**
 * #6398: no file is not no page. A DB-only page (a row with no file, a row
 * whose recorded source_path file vanished, or a tombstoned row) takes the
 * DB-only route: a stub would be mirrored over the DB body and timeline, or
 * resurrect a deleted page. An unreadable row writes nothing. Null: no row,
 * the stub may be created.
 */
async function dbOnlyPageRoute(engine: BrainEngine, target: FenceTarget, factCount: number): Promise<FenceWriteResult | null> {
  let live: Awaited<ReturnType<BrainEngine['getPage']>>;
  try {
    live = await engine.getPage(target.slug, { sourceId: target.sourceId, includeDeleted: true });
  } catch (err) {
    console.warn(`[facts.fence] FACTS_PAGE_READ_FAILED: ${target.slug} (source ${target.sourceId}): ${err instanceof Error ? err.message : String(err)}; nothing written`);
    return { inserted: 0, ids: [], fenceWriteFailed: true };
  }
  if (!live) return null;
  logStubGuardEvent({ slug: target.slug, source_id: target.sourceId, fact_count: factCount, reason: 'db_only_page' });
  console.warn(`[facts] page ${target.slug} (source ${target.sourceId}) has no file${live.deleted_at ? ' and is deleted' : ''} — routing facts to the legacy DB-only path instead of creating a stub that would overwrite it.`);
  return { inserted: 0, ids: [], stubGuardBlocked: true };
}

/**
 * Look up `sources.local_path` for a given source_id. Returns null
 * when the source has no local_path configured (thin-client / remote-
 * brain installs). Cached via the calling site is not necessary —
 * brains have at most a few sources and the lookup is a single
 * indexed query.
 *
 * Lives here (not in sources-ops.ts) so fence-write callers don't
 * need to thread the sources-ops module through the FactsBackstopCtx.
 */
export async function lookupSourceLocalPath(
  engine: BrainEngine,
  sourceId: string,
): Promise<string | null> {
  const rows = await engine.executeRaw<{ local_path: string | null }>(
    `SELECT local_path FROM sources WHERE id = $1 LIMIT 1`,
    [sourceId],
  );
  if (rows.length === 0) return null;
  return rows[0].local_path;
}