Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
49.5 KB
Raw Blame
/**
 * `gbrain repair failed-writes` (#5983): resubmits caller writes that the
 * managed writer guard refused while it misread page children's source
 * (`writer_coordinator_required`, stored before #5982 as `storage_error`
 * "Publication failed (P0001)"), and (#6429) caller writes that ended
 * `conflict` with `source_changed` because the page's file and database
 * differed, such as the writes refused while a database-only grandfather
 * stamp made every file look edited. The
 * terminal receipt keeps its full intent until receipt compaction
 * (`persistence.receipt_retention_days`), so the write can be submitted again
 * once the guard or the drift check is fixed.
 *
 * Only writes a caller made directly are replayed: `put_page` without an
 * owner-internal `kind`, `add_timeline_entry` and `remember`. Writes an owner
 * process produced (sync and file imports, reconcile, relink, maintenance
 * pages, jobs) are counted as residuals with the command that produces them
 * again, because re-running the producer uses current content.
 *
 * A candidate is kept, never replayed, when:
 *   - `already_written`: a later committed request carries the same intent, or
 *     an earlier apply's replay committed;
 *   - `duplicate`: a later failed or pending request carries the same intent
 *     (that one is the candidate);
 *   - `superseded`: a later request wrote or deleted the page (committed or
 *     still pending; for `put_page`, also a later failed `put_page`, which is
 *     the newer content), or the page changed after the revision the caller
 *     read;
 *   - `unpinned_target`: a `remember` whose original target was inferred as
 *     unattributed; replaying would infer again and could pick another page;
 *   - `file_database_drift` (W4.11): its last replay was refused
 *     `source_changed` because the page's file and database differ, and the
 *     page has not been written since; the listing names the reconcile
 *     command, and the write is a candidate again once the page changes.
 *
 * Explicit-only and preview-bound: the preview lists every candidate with its
 * class and saves the replay set, with each target page's revision, under its
 * hash; `--apply --expect <hash>` replays exactly that set. Each item is
 * reclassified (`changed_since_preview`) and its original authority re-checked
 * (`authority_revoked`), then submitted through the operation's normal path on
 * the original trust lane (a remote caller's write is prepared as a remote
 * write, with its take-holder and delegated-namespace limits), bound to the
 * previewed page revision (`force` is dropped), under a request id derived
 * from the failed request id. The failed receipt stays as history.
 */
import { OperationError } from '../ops/contract.ts';
import { isTimelineSection } from '../persistence/timeline-omission.ts';
import type { OperationContext } from '../ops/contract.ts';
import type { BrainEngine } from '../engine.ts';
import { digest } from '../persistence/digest.ts';
import { isTerminal, principalKey, type WriteAuthority, type WriteRequest } from '../persistence/model.ts';
import { authorizeStoredRequest } from '../persistence/authority.ts';
import { initializeLocalPersistence, requestPrincipalForContext, submitPageMutation } from '../persistence/page-mutations.ts';
import { submitRememberMutation } from '../persistence/memory-mutations.ts';
import { getWorktreeBinding } from '../persistence/ownership.ts';
import { prepareFileTarget } from '../persistence/page-prepare.ts';
import { PENDING_IMPORT, SYNC_HELD } from '../persistence/pending-import.ts';
import { FACTS_ENTITY_MOVED_MESSAGES, isEntityMovedRefusal } from '../persistence/verb-errors.ts';
import { MAX_FILE_REFUSAL_RETRIES } from '../persistence/facts-maintenance.ts';
import { factsAbsorbWriteKey, factsRetryDeadCommand } from '../facts/dead-absorb-retry.ts';
import { readFactsBackstopJobPage } from '../persistence/effect-facts.ts';
import { clearApprovedSet, loadApprovedSet, previewChangedError, previewHash, saveApprovedSet } from '../persistence/preview-approval.ts';
import { resolveEntitySlugWithSource } from '../entities/resolve.ts';
import { CALLER_REPLAYABLE_OPERATIONS, dispositionOf, DRIFT_REFUSAL_SQL, GUARD_REFUSAL_SQL, PAGE_REMOVAL_OPERATIONS, supersessionOrderSql, supersessionSql } from '../persistence/terminal-classify.ts';
import { isStatementTimeout } from '../persistence/bounded-reads.ts';
import { readRequestIndexStates } from '../persistence/checkpoint-validation.ts';
import { PERSISTENCE_REQUEST_REPLAY_ID_INDEX, PERSISTENCE_REQUEST_SLUG_SEQUENCE_INDEX } from '../persistence/schema.ts';
import { StatementDeadlineError, withStatementDeadline } from '../statement-deadline.ts';
import { afterCursor, type RepairHandler, type RepairItem, type RepairItemOutcome, type RepairPlan, type RepairScope } from './core.ts';

const REPLAYABLE = [...CALLER_REPLAYABLE_OPERATIONS];
const REMEMBER_PREPARED_KEYS = ['entity_slug', 'fence', 'valid_from', 'valid_until', 'entity_inferred', 'entity_warning', 'visibility_defaulted'];
const REMOTE_PREPARED_KEYS = ['source_kind', 'source_uri', 'ingested_via'];
const PAGE_REMOVALS = PAGE_REMOVAL_OPERATIONS;
const GUARD_REFUSAL = GUARD_REFUSAL_SQL('r');
const DRIFT_REFUSAL = DRIFT_REFUSAL_SQL('r');
/** #7602: a managed fact request refused because its entity page moved; extraction re-admits these since, but older ones stayed refused. */
const FACTS_MOVED_REFUSAL = `(r.operation='extract_facts' AND r.state='conflict' AND r.error_code='revision_conflict'
  AND r.error_message IN (${FACTS_ENTITY_MOVED_MESSAGES.map(message => `'${message.replaceAll("'", "''")}'`).join(',')}))`;
/** #7602: a refused fact request whose retained facts a later request of the same extraction committed on the same page. */
const FACTS_RECOVERED = `EXISTS (SELECT 1 FROM persistence_requests c WHERE r.operation='extract_facts' AND c.operation='extract_facts'
  AND c.source_id=r.source_id AND c.slug=r.slug AND c.state='committed' AND c.sequence>r.sequence
  AND COALESCE(c.intent->>'inputDigest', c.outcome->>'input_digest')=r.intent->>'inputDigest'
  AND COALESCE(c.intent->>'batchKey', c.outcome->>'batch_key') IS DISTINCT FROM r.intent->>'batchKey')`;

/** One managed fact entity request of an extraction (every request sharing its source and input digest). */
interface ExtractionRequest { source_id: string; input_digest: string; slug: string; state: string; error_code: string | null; error_message: string | null; batch_key: string | null }

const extractionKey = (sourceId: string, inputDigest: unknown) => `${sourceId}\n${String(inputDigest)}`;

/** Every entity request of the listed fact rows' extractions, read in one query and grouped by extraction. */
async function factExtractions(engine: BrainEngine, sourceIds: string[], rows: FailedWrite[]): Promise<Map<string, ExtractionRequest[]>> {
  const extractions = new Map<string, ExtractionRequest[]>();
  const digests = [...new Set(rows.filter(row => row.operation === 'extract_facts').map(row => String(row.intent.inputDigest)))];
  if (!digests.length) return extractions;
  for (const row of await engine.executeRaw<ExtractionRequest>(`SELECT source_id, intent->>'inputDigest' AS input_digest, slug, state, error_code, error_message,
      intent->>'batchKey' AS batch_key FROM persistence_requests WHERE operation='extract_facts' AND source_id = ANY($1::text[])
      AND NOT compacted AND intent IS NOT NULL AND intent->>'kind'='managed_facts_entity' AND intent->>'inputDigest' = ANY($2::text[])`, [sourceIds, digests])) {
    const key = extractionKey(row.source_id, row.input_digest);
    extractions.set(key, [...(extractions.get(key) ?? []), row]);
  }
  return extractions;
}

interface FailedWrite {
  id: string; request_id: string; sequence: string; operation: string; source_id: string; slug: string; digest: string; state: string;
  intent: Record<string, unknown>; authority: WriteAuthority; created_at: string; producer: string | null; error_code: string | null; page_id: string | null;
  /** #6693: the refusal's recorded origin (`pending_import`, `sync_held`), when the receipt carries one. */
  origin?: string | null;
}
type Disposition = 'replay' | 'already_written' | 'duplicate' | 'superseded' | 'unpinned_target' | 'file_database_drift';
/** A replay bound to the page revision the preview saw (null: no live page). */
interface ApprovedWrite extends FailedWrite { revision: string | null; selection: string[] }
interface FailedWriteItem extends RepairItem { failed: ApprovedWrite; hash: string; last: boolean }

function previewCommand(scope: RepairScope): string {
  return `gbrain repair failed-writes${scope.source_ids.length === 1 ? ` --source ${scope.source_ids[0]}` : ''}`;
}

/**
 * #7602: a refused fact request keeps its extracted facts. A page-triggered one (its job is keyed by the page write) is
 * recovered by resetting that job once it is dead: its retry re-admits them at the entity page's current revision with
 * no model call, once the cause clears. Only an extraction whose open entity requests are all moved-entity refusals (the
 * entity walk) or all held-file refusals (the file walk) is re-admitted by one retry. A mixed extraction, a batch whose
 * follow-ups were refused too, an entity page replaced since, a job that completed or is gone, a page write the job's
 * own page check now skips (written again since, or a narrowed writer grant), a writer-guard refusal and a batch with
 * no page write would replay the refusal or have no job to retry: those are extracted again. While the job still runs (a walk in
 * progress leaves follow-ups open), an entity page that is still the admitted one is listed `pending:`.
 */
async function factsRecovery(engine: BrainEngine, write: FailedWrite, extractions: Map<string, ExtractionRequest[]>): Promise<string> {
  const origin = (write.intent.origin as { slug?: string } | null | undefined)?.slug;
  const job = typeof write.intent.originalRequestId === 'string' ? write.intent.originalRequestId : null;
  const extractAgain = `re-extract: gbrain call extract_facts again on the current text of ${origin ?? 'the source turn'} with a new request_id (one model call)`;
  if (!job || write.state !== 'conflict') return extractAgain;
  const [page] = await engine.executeRaw<{ id: string }>('SELECT id::text FROM pages WHERE source_id=$1 AND slug=$2 AND deleted_at IS NULL', [write.source_id, write.slug]);
  if (!page && write.page_id !== null) return `not recoverable: entity page ${write.slug} was deleted after these facts were admitted`;
  if ((page?.id ?? null) !== write.page_id) return `${extractAgain}; entity page ${write.slug} was replaced after these facts were admitted, so a job retry replays the refusal`;
  const key = factsAbsorbWriteKey(job);
  const [run] = await engine.executeRaw<{ status: string; data: Record<string, unknown> }>("SELECT status, data FROM minion_jobs WHERE name='facts-absorb' AND idempotency_key=$1 ORDER BY id DESC LIMIT 1", [key]);
  if (run && run.status !== 'dead' && run.status !== 'completed' && run.status !== 'cancelled') return `pending: its facts-absorb job (${key}) is ${run.status}; preview again once it ends`;
  const open = (extractions.get(extractionKey(write.source_id, write.intent.inputDigest)) ?? []).filter(row => row.state !== 'committed');
  if (!open.every(isEntityMovedRefusal) && !open.every(row => row.error_code === 'source_changed')) {
    return `${extractAgain}; its extraction also holds refusals one job retry does not re-admit together`;
  }
  if (new Set(open.filter(row => row.slug === write.slug).map(row => row.batch_key)).size > MAX_FILE_REFUSAL_RETRIES) {
    return `${extractAgain}; its follow-up batches were refused as well, so a job retry replays the refusal`;
  }
  if (run?.status !== 'dead') return `${extractAgain}; its facts-absorb job ${run ? `is ${run.status}` : 'is gone'}, so gbrain facts retry-dead has no dead job to reset`;
  const source = await readFactsBackstopJobPage(engine, run.data);
  if ('skipped' in source) {
    return source.skipped === 'superseded' || source.skipped === 'page_missing'
      ? `${extractAgain}; ${origin ?? 'the source page'} changed after the page write its facts-absorb job extracts, so a job retry skips it`
      : `${extractAgain}; a job retry would skip the page write (${source.skipped}), so gbrain facts retry-dead cannot recover it`;
  }
  return `recover: ${factsRetryDeadCommand('--reset', key)} (the next facts_drain pass retries the page's facts-absorb job, which re-admits the retained facts without a model call${write.error_code === 'source_changed' ? ' once the file check passes' : ''})`;
}

/** The command that produces an owner-internal write again, from current content. */
function producerCommand(producer: string, sourceId: string): string {
  const parts = producer.split(':');
  if (parts.some(part => part.startsWith('managed_sync') || part === 'managed_file_import')) return `gbrain sync --source ${sourceId} --no-pull --json`;
  if (parts.includes('canonical_reconcile')) return `gbrain sources reconcile ${sourceId} --audit`;
  if (parts.includes('managed_file_repair')) return `gbrain repair frontmatter --source ${sourceId} (a frontmatter repair) or gbrain repair fences --source ${sourceId} (a fence repair)`;
  if (parts.includes('relink_facts')) return `gbrain facts relink --source ${sourceId} --dry-run`;
  if (parts.includes('managed_maintenance_expire_captured_facts')) return `gbrain repair captured-facts --source ${sourceId}`;
  if (parts.includes('managed_maintenance_agent_session_facts')) return `gbrain repair agent-session-facts --source ${sourceId}`;
  if (parts.includes('managed_maintenance_restore_extractor_facts')) return `gbrain repair extractor-facts --source ${sourceId}`;
  if (parts.includes('managed_maintenance_restore_unwithdrawn_facts')) return `gbrain repair unwithdrawn-facts --source ${sourceId}`;
  if (parts.includes('managed_maintenance_restore_redacted_facts')) return `gbrain repair redacted-facts --source ${sourceId}`;
  if (parts.includes('managed_maintenance_conversation_facts')) return `gbrain extract-conversation-facts --source-id ${sourceId} --dry-run`;
  return 'the next `gbrain dream` cycle (or autopilot) produces it again';
}

/** `facts` (the preview only) adds refused fact requests, without the ones a later request of the same extraction recovered. */
async function failedWrites(engine: BrainEngine, sourceIds: string[], opts: { id?: string; facts?: boolean } = {}): Promise<FailedWrite[]> {
  const { id, facts } = opts;
  const rows = await engine.executeRaw<FailedWrite>(`SELECT r.id::text, r.request_id::text, r.sequence::text, r.operation, r.source_id, r.slug, r.digest, r.state, r.intent, r.authority,
      to_char(r.created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') AS created_at, r.error_detail->>'origin' AS origin, r.error_code, r.page_id::text AS page_id,
      CASE WHEN r.operation = ANY($2::text[]) AND NOT (r.operation='put_page' AND r.intent ? 'kind') THEN NULL
        ELSE r.operation || COALESCE(':' || (r.intent->>'kind'), '') END AS producer
    FROM persistence_requests r
    WHERE r.source_id = ANY($1::text[]) AND NOT r.compacted AND r.intent IS NOT NULL AND (${GUARD_REFUSAL} OR ${DRIFT_REFUSAL}${facts ? ` OR ${FACTS_MOVED_REFUSAL}` : ''})
      ${id ? 'AND r.id=$3::uuid' : ''}
    ORDER BY r.sequence`, id ? [sourceIds, REPLAYABLE, id] : [sourceIds, REPLAYABLE]);
  if (id) return rows;
  // #8707: the recovery check reads only the refused fact requests, in one statement. Correlated into the selection above it
  // ran for every candidate, and the planner walked the request table from each candidate's sequence to the end.
  const factIds = rows.filter(row => row.operation === 'extract_facts').map(row => row.id);
  const recovered = new Set(factIds.length ? (await engine.executeRaw<{ id: string }>(
    `SELECT r.id::text FROM persistence_requests r WHERE r.id = ANY($1::uuid[]) AND ${FACTS_RECOVERED}`, [factIds])).map(row => row.id) : []);
  const live = rows.filter(row => !recovered.has(row.id));
  // #6429: a replay this command submitted that was refused source_changed is itself a conflict row; the original
  // write stays the candidate (its attempts are read through replayAttempt), so the attempt rows are not listed twice.
  const replays = new Set<string>();
  const requestIds = new Set(live.map(row => row.request_id));
  for (const row of live) for (let attempt = 0; requestIds.has(replayId(row.id, attempt)); attempt++) replays.add(replayId(row.id, attempt));
  return live.filter(row => !replays.has(row.request_id));
}

/**
 * #6429: the caller writes this command would still replay, counted per source, operation and refusal reason for
 * doctor's `lost_caller_writes`, plus the refused receipts compaction already emptied (unreplayable) and (#6527) the
 * retained ones the preview keeps for a reason SQL can decide (`not_replayable`). Distinct
 * intents, so a caller's retry counts its write once; a refused replay attempt is the original write (its rows are
 * dropped and its attempts walked as `disposition` does); a write whose replay or retry committed or whose page was
 * later deleted (for put_page: written again) is not lost. Owner-produced writes are not counted: their producer
 * runs them again. Reads no intents, and every read is one pass over the table, never one probe per candidate:
 * doctor runs this on every call.
 */
export async function lostCallerWrites(engine: BrainEngine, sourceIds: string[]):
Promise<{ lost: { source_id: string; operation: string; reason: 'source_changed' | 'writer_guard'; count: number }[]; compacted: number;
  not_replayable: { unpinned_target: number; revision_superseded: number } }> {
  type Lost = { id: string; request_id: string; source_id: string; slug: string; operation: string; digest: string; sequence: string; reason: 'source_changed' | 'writer_guard';
    unpinned: boolean; stale_revision: boolean };
  const caller = `r.operation = ANY($2::text[]) AND NOT (r.operation='put_page' AND r.intent ? 'kind')`;
  const refused = `(${GUARD_REFUSAL} OR ${DRIFT_REFUSAL})`;
  const [rows, [compacted]] = await Promise.all([
    engine.executeRaw<Lost>(
      `SELECT r.id::text, r.request_id::text, r.source_id, r.slug, r.operation, r.digest, r.sequence::text,
        CASE WHEN r.state='conflict' THEN 'source_changed' ELSE 'writer_guard' END AS reason,
        (r.operation='remember' AND jsonb_typeof(r.intent->'entity_slug') IS DISTINCT FROM 'string') AS unpinned,
        (r.operation='put_page' AND jsonb_typeof(r.intent->'expected_revision')='string' AND r.intent->>'expected_revision' IS DISTINCT FROM
          (SELECT p.knowledge_revision::text FROM pages p WHERE p.source_id=r.source_id AND p.slug=r.slug AND p.deleted_at IS NULL)) AS stale_revision
      FROM persistence_requests r
      WHERE r.source_id = ANY($1::text[]) AND NOT r.compacted AND r.intent IS NOT NULL AND ${refused} AND ${caller}
      ORDER BY r.sequence`, [sourceIds, REPLAYABLE]),
    // A compacted receipt keeps no intent, so an owner-produced put_page cannot be told from a caller's; remember and
    // add_timeline_entry are always caller writes.
    engine.executeRaw<{ count: number }>(
      `SELECT count(*)::int AS count FROM persistence_requests r
      WHERE r.source_id = ANY($1::text[]) AND r.compacted AND (${GUARD_REFUSAL} OR ${DRIFT_REFUSAL}) AND r.operation = ANY($2::text[])`,
      [sourceIds, REPLAYABLE.filter(operation => operation !== 'put_page')]),
  ]);
  if (!rows.length) return { lost: [], compacted: Number(compacted?.count ?? 0), not_replayable: { unpinned_target: 0, revision_superseded: 0 } };
  // Attempt rows this command submitted belong to their original (walked below), not to the listing.
  const requestIds = new Set(rows.map(row => row.request_id));
  const replays = new Set<string>();
  for (const row of rows) for (let attempt = 0; requestIds.has(replayId(row.id, attempt)); attempt++) replays.add(replayId(row.id, attempt));
  const originals = rows.filter(row => !replays.has(row.request_id));
  // Supersession, set-based: per slug the newest committed removal and write, per (slug, digest) the newest committed same intent.
  const slugs = [...new Set(originals.map(row => row.slug))];
  const [bySlug, byDigest] = await Promise.all([
    engine.executeRaw<{ source_id: string; slug: string; removal: string | null; write: string | null }>(
      `SELECT c.source_id, c.slug, max(c.sequence) FILTER (WHERE c.operation = ANY($3::text[]))::text AS removal, max(c.sequence)::text AS write
      FROM persistence_requests c WHERE c.source_id = ANY($1::text[]) AND c.slug = ANY($2::text[]) AND c.state='committed' GROUP BY c.source_id, c.slug`,
      [sourceIds, slugs, PAGE_REMOVALS]),
    engine.executeRaw<{ source_id: string; slug: string; digest: string; same: string }>(
      `SELECT c.source_id, c.slug, c.digest, max(c.sequence)::text AS same FROM persistence_requests c
      WHERE c.source_id = ANY($1::text[]) AND c.slug = ANY($2::text[]) AND c.digest = ANY($3::text[]) AND c.state='committed' GROUP BY c.source_id, c.slug, c.digest`,
      [sourceIds, slugs, [...new Set(originals.map(row => row.digest))]]),
  ]);
  const later = (a: string | null | undefined, b: string) => a != null && BigInt(a) > BigInt(b);
  const slugKey = (row: { source_id: string; slug: string }) => `${row.source_id}\0${row.slug}`;
  const removalBy = new Map(bySlug.map(row => [slugKey(row), row.removal]));
  const writeBy = new Map(bySlug.map(row => [slugKey(row), row.write]));
  const sameBy = new Map(byDigest.map(row => [`${slugKey(row)}\0${row.digest}`, row.same]));
  let pending = originals.filter(row => !later(sameBy.get(`${slugKey(row)}\0${row.digest}`), row.sequence)
    && !later(removalBy.get(slugKey(row)), row.sequence) && !(row.operation === 'put_page' && later(writeBy.get(slugKey(row)), row.sequence)));
  // Replay attempts whose row is not a refusal (committed, or still live): one read per attempt round, rounds are few.
  const lost: Lost[] = [];
  for (let attempt = 0; pending.length; attempt++) {
    const states = new Map((await engine.executeRaw<{ request_id: string; state: string }>(
      'SELECT request_id::text, state FROM persistence_requests WHERE request_id = ANY($1::uuid[])', [pending.map(row => replayId(row.id, attempt))])).map(row => [row.request_id, row.state]));
    const next: Lost[] = [];
    for (const row of pending) {
      const state = states.get(replayId(row.id, attempt));
      if (state === 'committed') continue;
      if (state && isTerminal({ state: state as WriteRequest['state'] })) next.push(row);
      else lost.push(row);
    }
    pending = next;
  }
  // #6527: the two classes the preview keeps out of its replay set that SQL decides (as `disposition` does): a remember
  // whose subject was inferred (entity_slug absent or not a string) and a put_page whose page moved past the revision
  // the caller read. The rest (duplicate, still-drifted file) need per-row or filesystem reads; the preview decides.
  const unpinned = new Set<string>(), stale = new Set<string>();
  const replayable = lost.filter(row => {
    if (row.stale_revision) stale.add(`${row.source_id}\0${row.digest}`);
    else if (row.unpinned) unpinned.add(`${row.source_id}\0${row.digest}`);
    else return true;
    return false;
  });
  const groups = new Map<string, { source_id: string; operation: string; reason: Lost['reason']; digests: Set<string> }>();
  for (const row of replayable) {
    const key = `${row.source_id}\0${row.operation}\0${row.reason}`;
    const group = groups.get(key) ?? { source_id: row.source_id, operation: row.operation, reason: row.reason, digests: new Set<string>() };
    group.digests.add(row.digest);
    groups.set(key, group);
  }
  const sorted = [...groups.values()].sort((a, b) => a.source_id.localeCompare(b.source_id) || a.operation.localeCompare(b.operation) || a.reason.localeCompare(b.reason));
  return { lost: sorted.map(group => ({ source_id: group.source_id, operation: group.operation, reason: group.reason, count: group.digests.size })), compacted: Number(compacted?.count ?? 0),
    not_replayable: { unpinned_target: unpinned.size, revision_superseded: stale.size } };
}

async function pageRevision(engine: BrainEngine, sourceId: string, slug: string): Promise<string | null> {
  const [page] = await engine.executeRaw<{ revision: string | null }>(
    'SELECT knowledge_revision::text AS revision FROM pages WHERE source_id=$1 AND slug=$2 AND deleted_at IS NULL', [sourceId, slug]);
  return page?.revision ?? null;
}

function replayId(failedId: string, attempt: number): string {
  const h = digest(['failed-writes-replay-v1', failedId, attempt]);
  return `${h.slice(0, 8)}-${h.slice(8, 12)}-4${h.slice(13, 16)}-a${h.slice(17, 20)}-${h.slice(20, 32)}`;
}

/**
 * The replay's request id: one per failed request and attempt, so a rerun after
 * a crash resumes the same admission and a terminal failure gets a new id.
 * `prior` is that attempt's request when it was already admitted.
 */
async function replayAttempt(engine: BrainEngine, failedId: string): Promise<{ id: string; prior: WriteRequest | null }> {
  for (let attempt = 0; ; attempt++) {
    const id = replayId(failedId, attempt);
    const [prior] = await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE request_id=$1::uuid', [id]);
    if (!prior || prior.state === 'committed' || !isTerminal(prior)) return { id, prior: prior ?? null };
  }
}

/**
 * #8707: each refused write's replay attempts, walked as `replayAttempt` walks them, with one read per attempt number across
 * every write: `committed` when the walk stopped at a committed attempt, `lastFailed` the newest attempt before the stop
 * that ended without committing.
 */
async function replayWalks(engine: BrainEngine, failedIds: string[]): Promise<Map<string, { committed: boolean; lastFailed: WriteRequest | null }>> {
  const walks = new Map<string, { committed: boolean; lastFailed: WriteRequest | null }>();
  const last = new Map<string, WriteRequest>();
  let pending = failedIds;
  for (let attempt = 0; pending.length; attempt++) {
    const found = new Map<string, WriteRequest>();
    for (const row of await engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE request_id = ANY($1::uuid[])', [pending.map(id => replayId(id, attempt))])) {
      if (!found.has(String(row.request_id))) found.set(String(row.request_id), row);
    }
    const next: string[] = [];
    for (const id of pending) {
      const prior = found.get(replayId(id, attempt));
      if (!prior || prior.state === 'committed' || !isTerminal(prior)) walks.set(id, { committed: prior?.state === 'committed', lastFailed: last.get(id) ?? null });
      else { last.set(id, prior); next.push(id); }
    }
    pending = next;
  }
  return walks;
}

/** W4.11: the reconcile step for a page whose file and database differ. */
function driftDetail(sourceId: string, slug: string): string {
  return `the page's file and database differ, so the replay was refused (source_changed); reconcile first: gbrain sources reconcile ${sourceId} ${slug} --preview, then run gbrain repair failed-writes again`;
}

/**
 * #6429: whether the page's canonical file still differs from its database copy, the same check the replay's
 * preparation applies. A reconcile that rewrites only the file leaves the page row untouched, so the row's
 * `updated_at` alone cannot tell that the drift is gone. A page that no longer exists has no file to drift (the
 * supersession rules below decide); without the owner binding on this host the drift is assumed to remain.
 */
async function stillDrifted(engine: BrainEngine, write: FailedWrite): Promise<boolean> {
  const snapshot = await engine.readPageSnapshot(write.slug, { sourceId: write.source_id });
  if (!snapshot) return false;
  const binding = await getWorktreeBinding(engine, write.source_id);
  if (!binding) return true;
  try {
    await prepareFileTarget(engine, { source_id: write.source_id, worktree_id: binding.worktree_id, slug: write.slug }, snapshot, null);
    return false;
  } catch (error) {
    return !(error instanceof OperationError) || error.code === 'source_changed';
  }
}

type Verdict = { class: Disposition; detail?: string };

/** Later rows of the pages one supersession read may walk; a page's later rows bound each of its writes' walk. */
const SUPERSESSION_ROWS_PER_READ = 100_000;

/**
 * The open writes in groups whose supersession reads stay small: each write's read walks the later rows of its page, so
 * the groups are packed by that count (read once per page), and a write on a page longer than the bound is read alone.
 */
async function supersessionBatches(engine: BrainEngine, open: FailedWrite[]): Promise<string[][]> {
  if (!open.length) return [];
  const spans = new Map((await engine.executeRaw<{ source_id: string; slug: string; later: number }>(
    `SELECT p.source_id, p.slug, count(*)::int AS later FROM (SELECT source_id, slug, min(sequence) AS sequence FROM persistence_requests
      WHERE id = ANY($1::uuid[]) GROUP BY 1, 2) p JOIN persistence_requests c ON c.source_id=p.source_id AND c.slug=p.slug AND c.sequence>p.sequence GROUP BY 1, 2`,
    [open.map(write => write.id)])).map(row => [`${row.source_id}\0${row.slug}`, row.later]));
  const batches: string[][] = [];
  let rows = Infinity;
  for (const write of open) {
    const walk = 1 + (spans.get(`${write.source_id}\0${write.slug}`) ?? 0);
    if (rows + walk > SUPERSESSION_ROWS_PER_READ) { batches.push([]); rows = 0; }
    batches.at(-1)!.push(write.id);
    rows += walk;
  }
  return batches;
}

/**
 * Why each caller write must not be replayed now, or `replay`. #8707: the request-table reads (replay attempts, later
 * requests of the page) are made for all the writes together, never once per write (the later requests in groups bounded
 * by their pages' history, `supersessionBatches`), and the file check runs once per page.
 */
async function dispositions(engine: BrainEngine, writes: FailedWrite[]): Promise<Map<string, Verdict>> {
  const verdicts = new Map<string, Verdict>();
  const walks = await replayWalks(engine, writes.map(write => write.id));
  const drift = new Map<string, Promise<boolean>>();
  const open: FailedWrite[] = [];
  for (const write of writes) {
    const walk = walks.get(write.id)!;
    if (walk.committed) { verdicts.set(write.id, { class: 'already_written', detail: 'replayed by gbrain repair failed-writes' }); continue; }
    const drifted = walk.lastFailed;
    if (drifted?.error_code === 'source_changed') {
      const [written] = await engine.executeRaw<{ later: boolean }>(
        'SELECT updated_at > $3::timestamptz AS later FROM pages WHERE source_id=$1 AND slug=$2', [write.source_id, write.slug, drifted.updated_at]);
      const page = `${write.source_id}\0${write.slug}`;
      if (!written?.later && !drift.has(page)) drift.set(page, stillDrifted(engine, write));
      if (!written?.later && await drift.get(page)) { verdicts.set(write.id, { class: 'file_database_drift', detail: driftDetail(write.source_id, write.slug) }); continue; }
    }
    open.push(write);
  }
  const later = new Map<string, { state: string; operation: string; same_intent: boolean }>();
  for (const ids of await supersessionBatches(engine, open)) {
    for (const row of await engine.executeRaw<{ id: string; state: string; operation: string; same_intent: boolean }>(
      `SELECT r.id::text, c.state, c.operation, c.same_intent FROM persistence_requests r CROSS JOIN LATERAL (SELECT c.state, c.operation, c.digest=r.digest AS same_intent
        FROM persistence_requests c WHERE ${supersessionSql('r', 'c')} ORDER BY ${supersessionOrderSql('r', 'c')} LIMIT 1) c WHERE r.id = ANY($1::uuid[])`, [ids])) later.set(row.id, row);
  }
  for (const write of open) {
    const row = later.get(write.id);
    const found = dispositionOf(row);
    const read = write.intent.expected_revision;
    if (found === 'already_written') verdicts.set(write.id, { class: 'already_written' });
    else if (found === 'duplicate') verdicts.set(write.id, { class: 'duplicate', detail: `a later request with the same intent is ${row!.state}` });
    else if (found === 'superseded') verdicts.set(write.id, { class: 'superseded', detail: `a later ${row!.operation} of this page is ${row!.state}` });
    else if (write.operation === 'put_page' && typeof read === 'string' && read !== await pageRevision(engine, write.source_id, write.slug)) {
      verdicts.set(write.id, { class: 'superseded', detail: 'the page changed after the revision the caller read' });
    } else if (write.operation === 'remember' && typeof write.intent.entity_slug !== 'string') {
      verdicts.set(write.id, { class: 'unpinned_target', detail: 'the fact was saved unattributed; replaying would infer its subject again' });
    } else verdicts.set(write.id, { class: 'replay' });
  }
  return verdicts;
}

async function disposition(engine: BrainEngine, write: FailedWrite): Promise<Verdict> {
  return (await dispositions(engine, [write])).get(write.id)!;
}

function item(write: ApprovedWrite, hash: string, last: boolean): FailedWriteItem {
  return { cursor: { phase: 1, id: Number(write.sequence) }, source_id: write.source_id, slug: write.slug,
    chars: typeof write.intent.content === 'string' ? write.intent.content.length : 0, action: `replay:${write.operation}`, failed: write, hash, last };
}

/**
 * #8707: a preview read the server ended at its statement timeout (or a statement deadline refused) becomes a refusal
 * with the next step instead of `internal_error`. Nothing was saved: the approval checkpoint is written only after every
 * candidate is classed. When an index the preview reads is not valid, building it is the fix.
 */
async function previewTimeoutRefusal(engine: BrainEngine, error: unknown, scope: RepairScope): Promise<unknown> {
  if (!isStatementTimeout(error) && !(error instanceof StatementDeadlineError)) return error;
  const command = previewCommand(scope);
  const argv = ['gbrain', 'repair', 'failed-writes', ...(scope.source_ids.length === 1 ? ['--source', scope.source_ids[0]!] : []), '--json'];
  const unbuilt = (await withStatementDeadline(null, () => readRequestIndexStates(engine, [PERSISTENCE_REQUEST_REPLAY_ID_INDEX, PERSISTENCE_REQUEST_SLUG_SEQUENCE_INDEX])).catch(() => []))
    .filter(index => index.state !== 'valid').map(index => `${index.name} (${index.state})`);
  const refusal = new OperationError('repair_preview_timeout',
    'The failed-writes preview stopped: a read of the write journal ran past the database statement timeout. Nothing was saved and no page changed.',
    unbuilt.length ? `Build the request indexes the preview reads (${unbuilt.join(', ')}) with gbrain repair request-indexes --apply, then run ${command} again.`
      : `Run ${command} again; if it stops again, raise the statement timeout for that one command on the brain host.`,
    'docs/guides/troubleshooting.md#repair-preview-timeout');
  refusal.retryable = false;
  refusal.why = unbuilt.length
    ? `Without ${unbuilt.join(' and ')} a lookup of the preview reads the whole request table.`
    : 'Every preview read is a set read over the request table; on this brain one of them took longer than the statement timeout allows.';
  refusal.fix = unbuilt.length
    ? { argv: ['gbrain', 'repair', 'request-indexes', '--json'], consent: [], actor: 'agent', requires_exclusive: false,
      why: 'Previews the request-table indexes a rebuild would build, without changing anything; its apply command builds them online.',
      verify: { argv: ['gbrain', 'doctor', '--only', 'persistence_request_indexes', '--json'] }, docs: 'docs/guides/repair.md#request-indexes' }
    : { argv, consent: [], actor: 'host_admin', requires_exclusive: false, why: 'Re-runs the read-only preview.',
      user_message: `The preview of refused writes did not finish within the database statement timeout. On the brain host, run ${command} with a longer timeout for that one command (GBRAIN_STATEMENT_TIMEOUT=15min; behind a transaction-mode pooler that setting is dropped, so raise the role's default instead).`,
      verify: { argv }, docs: 'docs/guides/troubleshooting.md#repair-preview-timeout' };
  return refusal;
}

export const failedWritesRepair: RepairHandler = {
  kind: 'failed-writes',
  embeds: false,
  async plan(engine, scope, after, opts): Promise<RepairPlan> {
    const command = previewCommand(scope);
    if (!opts?.apply) {
      try {
        const rows = await failedWrites(engine, scope.source_ids, { facts: true });
        const extractions = await factExtractions(engine, scope.source_ids, rows);
        const verdicts = await dispositions(engine, rows.filter(write => !write.producer));
        const residuals: Record<string, number> = {};
        const listing: NonNullable<RepairPlan['listing']> = [];
        const producers = new Map<string, string>();
        const replay: ApprovedWrite[] = [];
        for (const write of rows) {
          const label = `${write.source_id}:${write.slug} ${write.operation} (${write.state} ${write.created_at}, request ${write.id})`;
          if (write.producer) {
            residuals[`producer_owned:${write.producer}`] = (residuals[`producer_owned:${write.producer}`] ?? 0) + 1;
            const facts = write.operation === 'extract_facts';
            // GST-2517: only a caller submits a forget, so no gbrain process produces it again.
            const forget = write.operation === 'forget' || write.operation === 'forget_fact';
            const run = facts ? await factsRecovery(engine, write, extractions)
              : forget ? `a caller made this ${write.operation} and nothing produces it again; read the fact, then submit the ${write.operation} again with a new request_id if it is still wanted`
                : producerCommand(write.producer, write.source_id);
            producers.set(`${write.producer} in ${write.source_id}`, facts ? 'produced by gbrain itself, not replayed here; each listed request names the command that recovers its facts'
              : `${forget ? 'made by a caller' : 'produced by gbrain itself'}, not replayed here; ${run}`);
            listing.push({ item: label, class: 'producer_owned', detail: facts ? run : `${forget ? 'resubmit' : 're-run'}: ${run}` });
            continue;
          }
          const verdict = verdicts.get(write.id)!;
          if (verdict.class === 'replay') replay.push({ ...write, revision: await pageRevision(engine, write.source_id, write.slug), selection: scope.source_ids });
          else residuals[verdict.class] = (residuals[verdict.class] ?? 0) + 1;
          listing.push({ item: label, class: verdict.class, ...(verdict.detail ? { detail: verdict.detail } : {}) });
        }
        const sources = await engine.executeRaw<{ id: string; incarnation: string }>(
          'SELECT id, incarnation::text AS incarnation FROM sources WHERE id=ANY($1::text[]) ORDER BY id', [scope.source_ids]);
        const hash = previewHash({ kind: 'failed-writes-v1', brain_id: scope.brain_id, sources, selection: { source_ids: scope.source_ids },
          replay: replay.map(write => ({ id: write.id, digest: write.digest, revision: write.revision })) });
        if (replay.length) await saveApprovedSet<ApprovedWrite>(engine, { command: 'failed-writes', hash }, replay);
        return { items: replay.map((write, index) => item(write, hash, index === replay.length - 1)), preview_hash: hash, residuals, listing,
          warnings: [...[...producers].map(([what, note]) => `${what}: ${note}`),
            ...(replay.length ? [`This preview recorded an approval checkpoint for its replay set (preview ${hash.slice(0, 12)}), which --apply --expect reads; no page content changed.`] : [])] };
      } catch (error) {
        throw await previewTimeoutRefusal(engine, error, scope);
      }
    }
    if (!opts.expect) {
      throw new OperationError('invalid_params', 'gbrain repair failed-writes --apply replays only the set a preview printed.',
        `Preview first: ${command} — show the user the listing, then run the apply command it prints: ${command} --apply --expect <preview-hash>`,
        'docs/guides/repair.md#failed-writes');
    }
    const approved = await loadApprovedSet<ApprovedWrite>(engine, { command: 'failed-writes', hash: opts.expect, previewCommand: command,
      emptyHash: async () => previewHash({ kind: 'failed-writes-v1', brain_id: scope.brain_id, sources: (await engine.executeRaw<{ id: string; incarnation: string }>('SELECT id, incarnation::text AS incarnation FROM sources WHERE id=ANY($1::text[]) ORDER BY id', [scope.source_ids])), selection: { source_ids: scope.source_ids }, replay: [] }) });
    if (approved.items.some(write => JSON.stringify(write.selection) !== JSON.stringify(scope.source_ids))) throw previewChangedError(opts.expect, command);
    const items = approved.items.map((write, index) => item(write, opts.expect!, index === approved.items.length - 1));
    return { items: items.filter(entry => afterCursor(entry.cursor, after)), preview_hash: opts.expect, residuals: {} };
  },
  async apply(ctx, entry): Promise<RepairItemOutcome> {
    const { failed, hash, last } = entry as FailedWriteItem;
    const outcome = await replay(ctx, failed);
    if (last) await clearApprovedSet(ctx.engine, { command: 'failed-writes', hash });
    return outcome;
  },
};

/** The subagent job a stored restricted authority names: wave 10's delegatedJobId, else a legacy `wiki/agents/<id>/*` namespace. */
function recordedSubagentId(authority: WriteAuthority): number | undefined {
  if (authority.delegatedJobId !== undefined) return authority.delegatedJobId;
  const [only, ...rest] = authority.delegatedPrefixes ?? [];
  const id = rest.length ? undefined : /^wiki\/agents\/([1-9]\d*)\/\*$/.exec(only ?? '')?.[1];
  return id && Number.isSafeInteger(Number(id)) ? Number(id) : undefined;
}

/**
 * The original caller's trust lane: a remote caller's write is prepared as a
 * remote write, within its holder and namespace limits, under its stored
 * authority (#5994: `replayAuthority`, re-authorized live). A restricted
 * write gets back the subagent identity it recorded; a legacy sandboxed
 * subagent write (no allow-list) replays on the legacy namespace path.
 */
function laneContext(ctx: OperationContext, authority: WriteAuthority): OperationContext {
  if (!authority.remote) return { ...ctx, remote: false, replayAuthority: authority };
  const lane: OperationContext = { ...ctx, remote: true, replayAuthority: authority, takesHoldersAllowList: authority.takesHolders ? [...authority.takesHolders] : ['world'] };
  if (!authority.restrictedNamespace) return lane;
  const subagentId = recordedSubagentId(authority);
  if (authority.databaseOnlyReason === 'subagent_sandbox' && subagentId !== undefined) return { ...lane, viaSubagent: true, subagentId };
  return { ...lane, viaSubagent: true, allowedSlugPrefixes: [...(authority.delegatedPrefixes ?? [])], ...(subagentId === undefined ? {} : { subagentId }) };
}

/** The caller's params, without what preparation added, bound to the previewed page revision. */
async function replayParams(ctx: OperationContext, write: ApprovedWrite, requestId: string): Promise<{ params: Record<string, unknown> } | { refused: RepairItemOutcome }> {
  const params: Record<string, unknown> = { ...write.intent, request_id: requestId };
  if (write.authority.remote) for (const key of REMOTE_PREPARED_KEYS) delete params[key];
  if (write.operation === 'remember') {
    const target = String(write.intent.entity_slug);
    for (const key of REMEMBER_PREPARED_KEYS) delete params[key];
    if ((await resolveEntitySlugWithSource(ctx.engine, write.source_id, target))?.slug !== target) {
      return { refused: { applied: false, outcome: 'changed_since_preview', reason: `the fact's subject ${target} no longer resolves to itself` } };
    }
    params.entity = target;
    // #6693: the fact keeps the dates its first admission gave it (valid_from defaults to the admission time, a ttl to an absolute expiry).
    if (typeof write.intent.valid_from === 'string') params.valid_from = write.intent.valid_from;
    if (typeof write.intent.valid_until === 'string') params.ttl = write.intent.valid_until;
    else delete params.ttl;
    return { params };
  }
  if (write.operation === 'put_page') {
    delete params.force;
    delete params.expected_revision;
    if (write.revision) params.expected_revision = write.revision;
  }
  return { params };
}

async function replay(ctx: OperationContext, failed: ApprovedWrite): Promise<RepairItemOutcome> {
  const [live] = await failedWrites(ctx.engine, [failed.source_id], { id: failed.id });
  if (!live || live.digest !== failed.digest) return { applied: false, outcome: 'changed_since_preview', reason: 'the failed request was compacted or changed' };
  const lane = laneContext(ctx, live.authority);
  await initializeLocalPersistence(lane);
  const attempt = await replayAttempt(ctx.engine, failed.id);
  if (attempt.prior?.state === 'committed') return { applied: false, outcome: 'already_written' };
  if (attempt.prior && principalKey({ kind: attempt.prior.principal_kind, id: attempt.prior.principal_id }) !== principalKey(await requestPrincipalForContext(lane))) {
    return { applied: false, outcome: 'pending_elsewhere', reason: `replay request ${attempt.id} is ${attempt.prior.state} under another writer` };
  }
  if (!attempt.prior) {
    const verdict = await disposition(ctx.engine, live);
    if (verdict.class !== 'replay') return { applied: false, outcome: 'changed_since_preview', reason: verdict.detail ?? verdict.class };
    if (live.operation === 'put_page' && await pageRevision(ctx.engine, failed.source_id, failed.slug) !== failed.revision) {
      return { applied: false, outcome: 'changed_since_preview', reason: 'the page changed since the preview' };
    }
  }
  const [stored] = await ctx.engine.executeRaw<WriteRequest>('SELECT * FROM persistence_requests WHERE id=$1::uuid', [failed.id]);
  try { await authorizeStoredRequest(ctx.engine, stored); } catch (error) {
    if (error instanceof OperationError) return { applied: false, outcome: 'authority_revoked', reason: `${error.code}: ${error.message}` };
    throw error;
  }
  const prepared = await replayParams(lane, failed, attempt.id);
  if ('refused' in prepared) return prepared.refused;
  const { params } = prepared;
  try {
    // #5969 (D3): the Timeline section the first admission read from the caller's content, before normalization.
    const { timeline_section: timelineSection, ...replayed } = params;
    if (live.operation === 'remember') await submitRememberMutation(lane, params);
    else await submitPageMutation(lane, { operation: live.operation, params: replayed, ...(isTimelineSection(timelineSection) ? { timelineSection } : {}) });
  } catch (error) {
    if (error instanceof OperationError && error.code === 'source_changed') {
      return { applied: false, outcome: 'file_database_drift', reason: driftDetail(failed.source_id, failed.slug) };
    }
    if (error instanceof OperationError && !['write_pending', 'owner_unavailable', 'writer_lock_unavailable', 'writer_busy'].includes(error.code)) {
      return { applied: false, outcome: ['revision_conflict', 'revision_required'].includes(error.code) ? 'conflict' : 'refused', reason: `${error.code}: ${error.message}` };
    }
    throw error;
  }
  return { applied: true, outcome: 'replayed' };
}

/** #6693: the most caller remembers one sync run replays, and the time it may spend on them. */
export const IMPORT_REPLAY_LIMIT = 50;
export const IMPORT_REPLAY_BUDGET_MS = 30_000;

/** #6693: what one post-sync replay pass did; counts only, never a slug or claim. */
export interface ImportReplayReport {
  /** Refused remembers of the source whose page awaited import or was held by sync, not yet written. */
  examined: number;
  replayed: number;
  /** Their page still differs from its file (not imported yet, or still held): kept for a later sync. */
  waiting: number;
  /** Not replayed: written or superseded since, unpinned, or the replay was refused (authority, conflict); `gbrain repair failed-writes` lists each. */
  skipped: number;
  /** Left for the next sync by the per-run limit or time budget. */
  remaining: number;
}

/**
 * #6693 (D7): after a managed sync, replay once each caller `remember` that was
 * refused because its page's file held a committed change sync had not imported
 * (`pending_import`) or was held by sync (`sync_held`), now that the page matches
 * its file again. Same path as `gbrain repair failed-writes --apply`: the
 * disposition rules, the original trust lane, a live authority re-check and a
 * request id derived from the refused request, so two passes (or a pass and a
 * manual apply) admit one request. A page still drifted is skipped before any
 * admission, so a pass adds no receipt for it. `put_page` is never replayed here:
 * its full body would replace the change the sync just imported.
 */
export async function replayImportedRemembers(ctx: OperationContext, sourceId: string,
  opts: { limit?: number; budgetMs?: number; now?: () => number } = {}): Promise<ImportReplayReport> {
  const now = opts.now ?? Date.now;
  const deadline = now() + (opts.budgetMs ?? IMPORT_REPLAY_BUDGET_MS);
  const limit = opts.limit ?? IMPORT_REPLAY_LIMIT;
  const candidates = (await failedWrites(ctx.engine, [sourceId])).filter(write => write.operation === 'remember' && write.producer === null
    && (write.origin === PENDING_IMPORT || write.origin === SYNC_HELD));
  const report: ImportReplayReport = { examined: 0, replayed: 0, waiting: 0, skipped: 0, remaining: 0 };
  for (const [index, write] of candidates.entries()) {
    if (report.replayed >= limit || now() >= deadline) { report.remaining = candidates.length - index; break; }
    const verdict = await disposition(ctx.engine, write);
    // A replay that committed (or a later write of the same fact) settles the receipt; it stays as history only.
    if (verdict.class === 'already_written') continue;
    report.examined++;
    if (verdict.class !== 'replay') { report.skipped++; continue; }
    if (await stillDrifted(ctx.engine, write)) { report.waiting++; continue; }
    const outcome = await replay(ctx, { ...write, revision: await pageRevision(ctx.engine, write.source_id, write.slug), selection: [sourceId] });
    if (outcome.applied) report.replayed++;
    else if (outcome.outcome === 'file_database_drift') report.waiting++;
    else report.skipped++;
  }
  return report;
}