Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
31.6 KB
Raw Blame
/**
 * checkpoint-harvest.ts — Cathedral 5: the serve-side prompt harvest of a
 * compaction-boundary corpus segment. Fire-and-forget from the context_pack
 * IPC handler (the ack never waits on the LLM); the maintenance sweep's
 * corpus pass is the backstop by construction (segments are plain `.txt`
 * files in the corpus dir), so every skip below is safe.
 *
 * Discipline (sweep pass 3, shared fencing):
 *   claim (acquireCorpusClaim, O_EXCL) → `.ingested` re-check under claim →
 *   capability gate THEN kill switch → dream-output guard →
 *   window 1 of the turn-boundary windows (corpus-windows.ts; an existing
 *   `.progress` means window 1 is done) under a 60s abort → POST-check
 *   (the pipeline returns normally with partial results on abort — an
 *   aborted run writes NOTHING and releases the claim: retryable) →
 *   RECEIPT sidecar (link candidates) → idempotent manifest publish from
 *   the receipt (each slug verified via source-scoped getPage; a link that
 *   resolves to nothing is a lie and is skipped) → `.ingested` LAST, or, when
 *   windows remain, the receipt is released and the sweep finishes the tail.
 * A transient manifest failure leaves receipt + released claim; the retry
 * re-publishes from the receipt WITHOUT re-extracting (no duplicate spend,
 * no lost links).
 *
 * Serve lifecycle: the background-work registry's drain is CLI-exit-only by
 * documented contract, so serve calls `shutdownCheckpointHarvest()` before
 * `engine.disconnect()` — aborting the in-flight harvest and dropping the
 * queue (the #1762 hazard class: a fire-and-forget DB writer surviving
 * engine disconnect busy-loops the single-writer lock).
 *
 * Telemetry: `checkpoint-harvest` events on the hooks heartbeat JSONL —
 * counts + reason codes ONLY, never slugs/fact text [S3#7].
 */

import { readFile, rm, stat, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import type { BrainEngine } from '../engine.ts';
import type { CapabilityReport } from '../capability.ts';
import { acquireCorpusClaim, CORPUS_CLAIM_SUFFIX, CORPUS_INGESTED_SUFFIX } from '../sweep.ts';
import { appendCheckpointManifest } from './session-state.ts';
import { corpusFileSessionId, corpusTextForExtraction, parseSegmentFileName, readSegmentLedger, selfCaptureSidecarJson, HARVEST_RECEIPT_SUFFIX } from './corpus-segments.ts';
import { corpusFileStat, readCorpusProgress, runCorpusTurn, runCorpusWindows } from './corpus-windows.ts';
import { isClaudeCliSelfSessionId } from '../ai/providers/claude-cli-scratch.ts';
import { writeHeartbeat } from './hook-heartbeat.ts';
import { speakerOf } from './session-speaker.ts';
import { OperationError } from '../ops/contract.ts';

/** Bounded queue — overflow is a typed skip; the sweep backstop extracts later. */
export const HARVEST_QUEUE_CAP = 8;
/** Per-file abort so a hung provider can't wedge the FIFO. */
export const HARVEST_JOB_TIMEOUT_MS = 60_000;
/**
 * #5557: a writeback turn whose facts preflight still finds the canonical
 * writer busy after its own wait (`writer_lock_unavailable`, thrown before any
 * model call) is re-queued after each of these delays in turn; past the last
 * one it ends as an error. A Git effect holds that lock through its commit and
 * its push.
 */
export const HARVEST_WRITER_BUSY_RETRY_DELAYS_MS: readonly number[] = [5_000, 15_000, 45_000];
/** Receipt sidecar suffix — canonical home is corpus-segments (engine-free,
 * so the hook's GC can reap orphaned receipts); re-exported for callers. */
export { HARVEST_RECEIPT_SUFFIX };

export interface HarvestJob {
  engine: BrainEngine;
  /** #6268: the file's resolved source (its stamp or its session's frozen record), never a serve default. */
  sourceId: string;
  /** #6268: `sources.incarnation` the source resolved to; binds the receipt and window progress. */
  sourceIncarnation?: string;
  /** Harness session id — manifest lane + fact provenance. */
  sessionId: string;
  /** Absolute corpus dir (resolved by the CALLER the serve way). */
  corpusDir: string;
  /** Segment BASENAME (already validated by the IPC handler). */
  file: string;
  /**
   * Ambient-writeback lane (WP4): 'writeback' jobs carry a single gated user
   * turn (`.wb-` files) — extra serve-side gate on `memory.auto_writeback`
   * (a stale hook-side file-plane read never extracts against operator
   * intent: off ⇒ terminal `.ingested {skipped:'writeback_off'}`),
   * salient-mode notability filter, NO manifest publish (turn files have no
   * segment ledger), and the `writeback` heartbeat event. Default 'compact'.
   */
  lane?: 'compact' | 'writeback';
  /** TEST SEAM: capability report override (sweep PassCtx precedent). */
  capabilities?: CapabilityReport;
  /** TEST SEAM: per-job abort budget override (default HARVEST_JOB_TIMEOUT_MS). */
  timeoutMs?: number;
  /** Busy-writer re-queues already spent on this job; set by the pump only (#5557). */
  writerBusyRetries?: number;
}

interface HarvestReceipt {
  session_id: string;
  seg: string;
  links: string[];
  ts: string;
  /** #6268: the source + incarnation the links were extracted into; a replay under another is refused. */
  source_id?: string;
  source_incarnation?: string;
}

const queue: HarvestJob[] = [];
let inFlight = false;
let shuttingDown = false;
let currentAbort: AbortController | null = null;
/** Resolves when the pump goes idle — the shutdown join point. */
let idleResolve: (() => void) | null = null;
/** Busy-writer retries waiting on their delay (#5557); shutdown clears them. */
const pendingRetries = new Map<ReturnType<typeof setTimeout>, HarvestJob>();
let writerBusyRetryDelays: readonly number[] = HARVEST_WRITER_BUSY_RETRY_DELAYS_MS;
/** Error reasons already written to stderr by this serve run (#5557). */
const loggedErrorReasons = new Set<string>();

export type HarvestAck = { status: 'scheduled' } | { status: 'skipped'; reason: string };

/**
 * Enqueue a segment harvest. O(1), synchronous — the IPC ack never waits on
 * the LLM. Duplicate (same file already queued) and overflow are typed skips.
 */
export function scheduleCheckpointHarvest(job: HarvestJob): HarvestAck {
  if (shuttingDown) return { status: 'skipped', reason: 'shutting_down' };
  if ([...queue, ...pendingRetries.values()].some((q) => q.file === job.file && q.corpusDir === job.corpusDir)) {
    return { status: 'skipped', reason: 'already_queued' };
  }
  // F9/OV2-4 cost posture: per-session prompt-harvest cap for the writeback
  // lane, enforced HERE (the single serve process owns the queue; in-memory,
  // serve-restart resets — documented). Overflow is an ACK-only skip: the wb
  // file REMAINS on disk and the sweep's corpus pass batch-extracts it later
  // — freshness lost, nothing dropped.
  if (job.lane === 'writeback' && (wbSessionCounts.get(job.sessionId) ?? 0) >= WRITEBACK_SESSION_CAP) {
    return { status: 'skipped', reason: 'session_cap' };
  }
  if (queue.length >= HARVEST_QUEUE_CAP) return { status: 'skipped', reason: 'queue_full' };
  if (job.lane === 'writeback') {
    // Budget burns only on a real enqueue — a queue_full rejection retries
    // free via the sweep. Bounded map: evict the OLDEST entries (Map keeps
    // insertion order) rather than clear() — a wholesale clear would re-arm
    // the spend cap of every capped session under session churn (security
    // review, this wave). Serve restarts still reset (documented).
    if (wbSessionCounts.size >= WB_SESSION_MAP_MAX && !wbSessionCounts.has(job.sessionId)) {
      for (const key of wbSessionCounts.keys()) {
        if (wbSessionCounts.size < WB_SESSION_MAP_MAX) break;
        wbSessionCounts.delete(key);
      }
    }
    // delete-before-set makes the map LRU: a bare .set() on an existing key
    // keeps its ORIGINAL insertion slot, so the busiest long-lived session
    // would be evicted first — re-arming exactly the budget the eviction
    // policy exists to preserve (adversarial review, this wave).
    const burned = (wbSessionCounts.get(job.sessionId) ?? 0) + 1;
    wbSessionCounts.delete(job.sessionId);
    wbSessionCounts.set(job.sessionId, burned);
  }
  queue.push(job);
  void pump();
  return { status: 'scheduled' };
}

/** Per-session prompt-harvest budget for the writeback lane (F9/OV2-4). */
export const WRITEBACK_SESSION_CAP = 30;
/** Size bound for the per-session counter map — oldest-first eviction, so an
 * ACTIVE session's burned budget survives churn (a stale evicted session that
 * comes back gets a fresh budget, the accepted residual). */
const WB_SESSION_MAP_MAX = 200;
const wbSessionCounts = new Map<string, number>();

/** Shutdown grace: don't let a transport that ignores the abort signal hold
 * serve's exit — the claim's staleness window + the sweep backstop make an
 * abandoned in-flight job safe to orphan (pre-landing review, perf). */
export const HARVEST_SHUTDOWN_GRACE_MS = 5000;

/**
 * Abort the in-flight harvest, drop the queue, and wait for the pump to go
 * idle — bounded by HARVEST_SHUTDOWN_GRACE_MS. Serve's shutdown path calls
 * this BEFORE engine.disconnect().
 */
export async function shutdownCheckpointHarvest(): Promise<void> {
  shuttingDown = true;
  queue.length = 0;
  clearPendingRetries();
  try {
    currentAbort?.abort();
  } catch {
    /* noop */
  }
  if (inFlight) {
    await new Promise<void>((resolve) => {
      idleResolve = resolve;
      const t = setTimeout(resolve, HARVEST_SHUTDOWN_GRACE_MS);
      if (typeof (t as { unref?: () => void }).unref === 'function') {
        (t as unknown as { unref: () => void }).unref();
      }
    });
  }
}

/** TEST SEAM: reset module state between tests. */
export function __resetCheckpointHarvestForTests(): void {
  queue.length = 0;
  inFlight = false;
  shuttingDown = false;
  currentAbort = null;
  idleResolve = null;
  wbSessionCounts.clear();
  clearPendingRetries();
  writerBusyRetryDelays = HARVEST_WRITER_BUSY_RETRY_DELAYS_MS;
  loggedErrorReasons.clear();
}

/** TEST SEAM: busy-writer retry delays (null restores the defaults). */
export function __setWriterBusyRetryDelaysForTests(delays: readonly number[] | null): void {
  writerBusyRetryDelays = delays ?? HARVEST_WRITER_BUSY_RETRY_DELAYS_MS;
}

/** TEST SEAM: resolves once the queue is fully drained, pending retries included. */
export async function __drainCheckpointHarvestForTests(): Promise<void> {
  while (inFlight || queue.length > 0 || pendingRetries.size > 0) {
    await new Promise((r) => setTimeout(r, 10));
  }
}

async function pump(): Promise<void> {
  if (inFlight) return;
  const job = queue.shift();
  if (!job) {
    idleResolve?.();
    idleResolve = null;
    return;
  }
  inFlight = true;
  const t0 = Date.now();
  let outcome: 'ok' | 'degraded' | 'error' = 'ok';
  let reason: string | undefined;
  let inserted: number | undefined;
  let duplicate: number | undefined;
  let superseded: number | undefined;
  let links: number | undefined;
  try {
    const r = await runOne(job);
    outcome = r.outcome;
    reason = r.reason;
    inserted = r.inserted;
    duplicate = r.duplicate;
    superseded = r.superseded;
    links = r.links;
  } catch (e) {
    ({ outcome, reason } = classifyHarvestFailure(job, e));
  } finally {
    inFlight = false;
    await writeHeartbeat({
      ts: new Date().toISOString(),
      // The writeback lane heartbeats under its own event so doctor's
      // backstop counters never conflate the two lanes (OV-A11).
      event: job.lane === 'writeback' ? 'writeback' : 'checkpoint-harvest',
      outcome,
      ...(reason ? { reason } : {}),
      duration_ms: Date.now() - t0,
      ...(inserted !== undefined ? { inserted } : {}),
      ...(duplicate !== undefined ? { duplicate } : {}),
      ...(superseded !== undefined ? { superseded } : {}),
      ...(links !== undefined ? { links } : {}),
      // trim:false — serve is a long-lived high-frequency writer; its
      // read→tmp→rename trim would clobber concurrent hook appends.
    }, { trim: false });
    if (!shuttingDown) void pump();
    else {
      idleResolve?.();
      idleResolve = null;
    }
  }
}

function clearPendingRetries(): void {
  for (const timer of pendingRetries.keys()) clearTimeout(timer);
  pendingRetries.clear();
}

/**
 * A writeback turn refused because the canonical writer stayed busy is
 * re-queued; any other failure is an error that keeps its code. A compact
 * segment is never re-queued: its first window already wrote `.progress`, which
 * a retry reads as window 1 done, so the error stands and the sweep resumes it.
 */
function classifyHarvestFailure(job: HarvestJob, e: unknown): { outcome: 'degraded' | 'error'; reason: string } {
  if (job.lane === 'writeback' && isWriterBusy(e) && requeueWhileWriterBusy(job)) {
    return { outcome: 'degraded', reason: 'writer_busy_requeued' };
  }
  // #6091: writeback turned off while the provider call was in flight; admission
  // refused the facts and the next pass retires the file.
  if (e instanceof OperationError && e.code === 'ambient_capture_off') return { outcome: 'degraded', reason: 'writeback_off_inflight' };
  const reason = harvestErrorReason(e);
  logFirstHarvestError(job, reason, e);
  return { outcome: 'error', reason };
}

/** The facts preflight refused because the canonical writer stayed busy; nothing was extracted. */
function isWriterBusy(e: unknown): boolean {
  return e instanceof OperationError && e.code === 'writer_lock_unavailable';
}

/**
 * Schedules the job's next attempt after its busy-writer delay; false once the
 * delays are spent or serve is stopping. The retry skips the admission checks:
 * the job was admitted (and counted against its session cap) once already.
 */
function requeueWhileWriterBusy(job: HarvestJob): boolean {
  const spent = job.writerBusyRetries ?? 0;
  const delay = writerBusyRetryDelays[spent];
  if (delay === undefined || shuttingDown) return false;
  const timer = setTimeout(() => {
    pendingRetries.delete(timer);
    if (shuttingDown) return;
    queue.push({ ...job, writerBusyRetries: spent + 1 });
    void pump();
  }, delay);
  (timer as { unref?: () => void }).unref?.();
  pendingRetries.set(timer, job);
  return true;
}

/** A heartbeat reason is a short machine code, the shape hook.ts's reasonCode() enforces. */
const HEARTBEAT_REASON_CODE = /^[A-Za-z0-9_.:-]{1,48}$/;

/**
 * The error name plus its `code`, or else its `reason` (FactsExtractionError):
 * `operationerror:writer_lock_unavailable`, `factsextractionerror:provider_error`.
 * A composite past the reason-code shape falls back to the bare name.
 */
function harvestErrorReason(e: unknown): string {
  if (!(e instanceof Error)) return 'error';
  const name = (e.name || 'Error').toLowerCase();
  const { code, reason } = e as { code?: unknown; reason?: unknown };
  const detail = typeof code === 'string' ? code : typeof reason === 'string' ? reason : null;
  const composite = detail === null ? name : `${name}:${detail.toLowerCase()}`;
  if (HEARTBEAT_REASON_CODE.test(composite)) return composite;
  return HEARTBEAT_REASON_CODE.test(name) ? name : 'error';
}

/** The heartbeat keeps counts and codes only; the first failure of each reason also reaches serve's stderr. */
function logFirstHarvestError(job: HarvestJob, reason: string, e: unknown): void {
  if (loggedErrorReasons.has(reason)) return;
  loggedErrorReasons.add(reason);
  const message = e instanceof Error ? e.message : String(e);
  console.error(`[checkpoint-harvest] ${job.lane ?? 'compact'} harvest failed (${reason}): ${message.slice(0, 300)} `
    + '(logged once per serve run; later failures with this reason are counted in the hooks heartbeat only)');
}

/** A segment basename's content hash ('' when the name is not a segment). */
function segHashFromName(file: string): string {
  return parseSegmentFileName(file)?.hash ?? '';
}

async function runOne(job: HarvestJob): Promise<{
  outcome: 'ok' | 'degraded' | 'error';
  reason?: string;
  inserted?: number;
  duplicate?: number;
  superseded?: number;
  links?: number;
}> {
  const full = join(job.corpusDir, job.file);
  const claimPath = full + CORPUS_CLAIM_SUFFIX;
  const receiptPath = full + HARVEST_RECEIPT_SUFFIX;
  const ingestedPath = full + CORPUS_INGESTED_SUFFIX;

  if (!(await acquireCorpusClaim(claimPath))) {
    return { outcome: 'degraded', reason: 'claimed_elsewhere' };
  }
  try {
    // Re-check under the claim (sweep discipline — closes the double-spend gap).
    if (await stat(ingestedPath).then(() => true, () => false)) {
      return { outcome: 'ok', reason: 'already_ingested' };
    }

    if (job.lane === 'writeback') return await runWritebackTurn(job, full, ingestedPath);

    // #6091: the capture gate runs under the claim before any provider call or
    // receipt republication (serve compact lane and OpenClaw rung 2).
    const { applyCaptureGate, resolveCaptureGate } = await import('./capture-consent.ts');
    const gate = await applyCaptureGate(full, (await resolveCaptureGate(job.engine)).compact);
    if (gate.action === 'hold') return { outcome: 'degraded', reason: gate.reason };
    if (gate.action === 'retire') {
      await rm(receiptPath, { force: true }).catch(() => {});
      return { outcome: 'ok', reason: gate.reason };
    }

    // Receipt retry path (codex round 2): extraction already happened; the
    // manifest publish failed transiently. Re-publish WITHOUT re-extracting.
    let receipt = await readReceipt(receiptPath);
    if (receipt?.source_id !== undefined && (receipt.source_id !== job.sourceId
      || (job.sourceIncarnation !== undefined && receipt.source_incarnation !== job.sourceIncarnation))) {
      return { outcome: 'degraded', reason: 'receipt_source_mismatch' };
    }
    let counts: { inserted: number; duplicate: number } | null = null;
    // #5887: the harvest extracts window 1 only; when more remains, `.progress`
    // stands in for `.ingested` and the sweep finishes the tail.
    let windowsPending = receipt !== null && (await readCorpusProgress(full))?.finished === null;

    if (!receipt) {
      // An existing `.progress` means window 1 is done (an earlier harvest or
      // the sweep wrote it): never re-extract it here.
      if (await readCorpusProgress(full)) return { outcome: 'ok', reason: 'windows_pending' };
      // Gates in the pinned sweep order: capability THEN kill switch. A
      // gate-skip releases the claim and writes NO sidecar — when the gate
      // opens later, the sweep (which applies the same gates) extracts.
      const { extractionAvailableForEngine } = await import('../facts/extraction-availability.ts');
      if (!(await extractionAvailableForEngine(job.engine, job.capabilities))) {
        return { outcome: 'degraded', reason: 'keyless' };
      }
      const { isFactsExtractionEnabled } = await import('../facts/extract.ts');
      if (!(await isFactsExtractionEnabled(job.engine))) {
        return { outcome: 'degraded', reason: 'extraction_disabled' };
      }
      if (await writerRefused(job)) return { outcome: 'degraded', reason: 'writer_coordinator_required' };

      const fileStat = corpusFileStat(await stat(full));
      const raw = await readFile(full, 'utf-8');
      const { isDreamOutput } = await import('../cycle/transcript-discovery.ts');
      const { runFactsPipeline } = await import('../facts/backstop.ts');
      if (isDreamOutput(raw)) {
        await writeFile(
          ingestedPath,
          JSON.stringify({ ingested_at: new Date().toISOString(), skipped: 'dream_output' }) + '\n',
        );
        return { outcome: 'ok', reason: 'dream_output' };
      }

      const abort = new AbortController();
      const timer = setTimeout(() => abort.abort(), job.timeoutMs ?? HARVEST_JOB_TIMEOUT_MS);
      currentAbort = abort;
      let run: Awaited<ReturnType<typeof runCorpusWindows>>;
      try {
        // Paste stripping happens per turn inside the window planner (#5812).
        run = await runCorpusWindows({
          full, raw, fileStat, maxWindows: 1, overBudget: () => false, signal: abort.signal,
          failures: { fingerprint: await harvestFingerprint(job) },
          ...(job.sourceIncarnation ? { source: { id: job.sourceId, incarnation: job.sourceIncarnation } } : {}),
          extract: (text) => runFactsPipeline(text, {
            engine: job.engine,
            sourceId: job.sourceId,
            sessionId: job.sessionId,
            source: 'hook:compact',
            ...speakerOf(full),
            mode: 'inline',
            remote: false,
            abortSignal: abort.signal,
            // visibility deliberately unset → resolveDefaultVisibility [ENG-8]
          }),
        });
      } finally {
        clearTimeout(timer);
        currentAbort = null;
      }
      // POST-check (codex round 2): runFactsPipeline returns normally with
      // PARTIAL results on abort. An aborted run writes NOTHING (no receipt,
      // no .ingested, no progress) and releases the claim — fully retryable.
      if (abort.signal.aborted || run.status === 'aborted') return { outcome: 'degraded', reason: 'aborted' };
      if (run.status === 'contended') return { outcome: 'degraded', reason: 'claimed_elsewhere' };
      if (run.status === 'source_changed') return { outcome: 'degraded', reason: 'corpus_source_changed' };
      // #6786/#6785: nothing was recorded for window 1; the sweep retries it under the same gates.
      if (run.status === 'unavailable') return { outcome: 'degraded', reason: 'extract_skipped_chat_unavailable' };
      if (run.status === 'backoff' || run.status === 'dead_lettered') return { outcome: 'degraded', reason: `corpus_window_${run.status}` };
      windowsPending = run.status !== 'complete';
      const r = run.result;

      counts = { inserted: r.inserted, duplicate: r.duplicate };
      receipt = {
        session_id: job.sessionId,
        seg: segHashFromName(job.file),
        links: r.entity_slugs,
        ts: new Date().toISOString(),
        source_id: job.sourceId,
        ...(job.sourceIncarnation ? { source_incarnation: job.sourceIncarnation } : {}),
      };
      await writeFile(receiptPath, JSON.stringify(receipt) + '\n', { mode: 0o600 });
    }

    // Idempotent manifest publish from the receipt: bank ONLY links whose
    // entity page resolves via a SOURCE-SCOPED read (the existence check
    // doubles as the title lookup; null/throw ⇒ skip that link).
    const verified: Array<{ slug: string; title: string }> = [];
    for (const slug of receipt.links) {
      try {
        const page = await job.engine.getPage(slug, { sourceId: job.sourceId });
        if (page) verified.push({ slug, title: page.title || slug });
      } catch {
        /* transient or missing — a non-resolvable link is never banked */
      }
    }
    if (verified.length) {
      const ledger = readSegmentLedger(job.corpusDir, receipt.session_id);
      const n = Math.max(1, ledger.findIndex((e) => e.hash === receipt.seg) + 1);
      const published = await appendCheckpointManifest(
        job.engine, job.sourceId, null, receipt.session_id, verified,
        { seg: receipt.seg, n },
      );
      if (!published) {
        // Publish failed (returns false, never throws): keep the receipt —
        // the ONLY durable copy of the links — write NO `.ingested`, release
        // the claim. The retry re-publishes from the receipt without
        // re-extracting. Writing `.ingested` here would delete the links.
        return { outcome: 'degraded', reason: 'manifest_failed' };
      }
    }

    if (windowsPending) {
      // Window 1's links are banked and `.progress` is written: release the
      // receipt; the sweep extracts the remaining windows and writes `.ingested`.
      await rm(receiptPath, { force: true }).catch(() => {});
      return {
        outcome: 'ok',
        reason: 'windows_pending',
        ...(counts ? { inserted: counts.inserted, duplicate: counts.duplicate } : {}),
        links: verified.length,
      };
    }

    // `.ingested` LAST — the sweep now skips this segment; the receipt has
    // served its purpose and is removed (GC would otherwise orphan-reap it).
    await writeFile(
      ingestedPath,
      JSON.stringify({
        ingested_at: new Date().toISOString(),
        ...(counts ? { facts_inserted: counts.inserted, facts_duplicate: counts.duplicate } : { from_receipt: true }),
        links_banked: verified.length,
      }) + '\n',
    );
    await rm(receiptPath, { force: true }).catch(() => {});

    return {
      outcome: 'ok',
      ...(counts ? { inserted: counts.inserted, duplicate: counts.duplicate } : {}),
      links: verified.length,
    };
  } finally {
    await rm(claimPath, { force: true }).catch(() => {});
  }
}

/**
 * Writeback-lane extraction (WP4): one gated user turn. Distinct from the
 * compact lane in exactly four ways — the AUTHORITATIVE serve-side
 * `memory.auto_writeback` gate (off ⇒ TERMINAL `.ingested` sidecar, so a
 * stale hook-side file-plane read never extracts against operator intent —
 * and the sweep respects the same sidecar), the salient-mode notability
 * filter, `source: 'hook:writeback'`, and NO manifest publish (turn files
 * have no segment ledger). Claim/ingested lifecycle is the caller's
 * (runOne's) shared discipline. wb-file state machine (OV2-10): a
 * genuinely-resolved gate-off ⇒ terminal sidecar; keyless /
 * extraction_disabled / abort / plane-drift / invalid-mode / non-transport
 * extraction skip ⇒ NO sidecar (claim released, file remains — the sweep
 * retries when the gate opens or the config re-coheres); success ⇒
 * `.ingested` with persisted counts.
 */
async function runWritebackTurn(job: HarvestJob, full: string, ingestedPath: string): Promise<{
  outcome: 'ok' | 'degraded' | 'error';
  reason?: string;
  inserted?: number;
  duplicate?: number;
  superseded?: number;
}> {
  // #5820: a turn banked from gbrain's own claude-cli session (an older
  // binary's Stop hook, or a child that still ran user hooks) is terminal
  // here exactly as in the sweep — extracting it would spawn another
  // claude-cli call that banks again.
  if (isClaudeCliSelfSessionId(corpusFileSessionId(job.file))) {
    await writeFile(ingestedPath, selfCaptureSidecarJson());
    return { outcome: 'ok', reason: 'self_capture' };
  }
  // Gate semantics (captureGateDecision, resolved {gate:true}): a config READ
  // FAILURE, PLANE DRIFT (DB row absent while the file mirror says enabled — a
  // failed dual-write is not operator intent) and an UNRECOGNIZED mode value
  // hold: no sidecar, the claim releases and the sweep retries once the config
  // is coherent. Only a genuinely-resolved OFF (or unset: this lane is opt-in)
  // writes the terminal sidecar. The gate resolves BEFORE the
  // capability/kill-switch checks: an operator's OFF must retire the banked
  // turn even on a keyless or extraction-disabled brain — otherwise the file
  // lingers eligible and a later re-enable would extract turns the operator
  // already revoked (codex re-review, this wave).
  const { applyCaptureGate, resolveCaptureGate } = await import('./capture-consent.ts');
  const gate = await resolveCaptureGate(job.engine);
  const applied = await applyCaptureGate(full, gate.writeback);
  if (applied.action === 'hold') return { outcome: 'degraded', reason: applied.reason };
  if (applied.action === 'retire') return { outcome: 'ok', reason: applied.reason };
  const { extractionAvailableForEngine } = await import('../facts/extraction-availability.ts');
  if (!(await extractionAvailableForEngine(job.engine, job.capabilities))) {
    return { outcome: 'degraded', reason: 'keyless' };
  }
  const { isFactsExtractionEnabled } = await import('../facts/extract.ts');
  if (!(await isFactsExtractionEnabled(job.engine))) {
    return { outcome: 'degraded', reason: 'extraction_disabled' };
  }
  if (await writerRefused(job)) return { outcome: 'degraded', reason: 'writer_coordinator_required' };

  const raw = await readFile(full, 'utf-8');
  const { runFactsPipeline } = await import('../facts/backstop.ts');
  const abort = new AbortController();
  const timer = setTimeout(() => abort.abort(), job.timeoutMs ?? HARVEST_JOB_TIMEOUT_MS);
  currentAbort = abort;
  let turn: Awaited<ReturnType<typeof runCorpusTurn>>;
  try {
    // #6785: the same failure cap as the sweep's turn files (one record per file).
    turn = await runCorpusTurn({
      full, signal: abort.signal, incarnation: job.sourceIncarnation ?? null,
      failures: { fingerprint: await harvestFingerprint(job) },
      extract: () => runFactsPipeline(corpusTextForExtraction(job.file, raw), {
        engine: job.engine,
        sourceId: job.sourceId,
        sessionId: job.sessionId,
        source: 'hook:writeback',
        ...speakerOf(full),
        mode: 'inline',
        remote: false,
        abortSignal: abort.signal,
        notabilityFilter: gate.mode === 'salient' ? 'medium-and-up' : 'all',
        // visibility deliberately unset → resolveDefaultVisibility [ENG-8] —
        // the backstop inherits extract_facts' contract, never widened (req 6).
      }),
    });
  } finally {
    clearTimeout(timer);
    currentAbort = null;
  }
  if (abort.signal.aborted) return { outcome: 'degraded', reason: 'aborted' };
  if (turn.status === 'unavailable') return { outcome: 'degraded', reason: 'extract_skipped_chat_unavailable' };
  if (turn.status !== 'complete') return { outcome: 'degraded', reason: `corpus_window_${turn.status}` };
  const r = turn.result;
  if (r.skipped_reason) {
    // Non-transport extraction skip (refusal / content_filter /
    // malformed_output / non_terminal_stop / chat_unavailable): NOT terminal
    // — no sidecar, so the sweep's corpus pass retries the file exactly once
    // more and records the final outcome in ITS sidecar. Sidecaring here
    // would turn a transient moderation/format hiccup into silent permanent
    // loss of the banked turn (adversarial review, this wave). Transport
    // failures already throw (FactsExtractionError) and land in runOne's
    // catch with the same no-sidecar result.
    return { outcome: 'degraded', reason: `extract_skipped_${r.skipped_reason}` };
  }

  await writeFile(
    ingestedPath,
    JSON.stringify({
      ingested_at: new Date().toISOString(),
      facts_inserted: r.inserted,
      facts_duplicate: r.duplicate,
      facts_superseded: r.superseded,
      lane: 'writeback',
    }) + '\n',
  );
  return { outcome: 'ok', inserted: r.inserted, duplicate: r.duplicate, superseded: r.superseded };
}

/**
 * #6785: the facts-absorb owner-readiness preflight (#5362) before any model
 * call. A claimed but not activated canonical worktree refuses the fence
 * write after extraction; the turn stays banked (no sidecar) for the sweep,
 * which holds it under the same check until `gbrain sources writer activate`.
 */
/** #6785: the extraction-config fingerprint a capped window re-arms on (null: never re-arms). */
async function harvestFingerprint(job: HarvestJob): Promise<string | null> {
  const { extractionConfigFingerprint } = await import('../facts/extraction-availability.ts');
  return extractionConfigFingerprint(job.engine).catch(() => null);
}

async function writerRefused(job: HarvestJob): Promise<boolean> {
  const { assertSourceLegacyWriter } = await import('../persistence/filesystem-guard.ts');
  try {
    await assertSourceLegacyWriter(job.engine, job.sourceId);
    return false;
  } catch (e) {
    if ((e as { code?: unknown }).code !== 'writer_coordinator_required') throw e;
    logFirstHarvestError(job, 'writer_coordinator_required', new Error(
      `${e instanceof Error ? e.message : String(e)} Check the writer: gbrain sources writer status ${job.sourceId}`));
    return true;
  }
}

async function readReceipt(path: string): Promise<HarvestReceipt | null> {
  try {
    const raw = await readFile(path, 'utf-8');
    const parsed = JSON.parse(raw) as HarvestReceipt;
    if (
      parsed && typeof parsed === 'object' &&
      typeof parsed.session_id === 'string' &&
      typeof parsed.seg === 'string' &&
      Array.isArray(parsed.links)
    ) {
      return { ...parsed, links: parsed.links.filter((l): l is string => typeof l === 'string') };
    }
    return null;
  } catch {
    return null;
  }
}