Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
59.2 KB
Raw Blame
import type { BrainEngine } from '../engine.ts';
import { isQuarantined } from '../quarantine.ts';
import type { ParsedPage } from '../import-file.ts';
import type { PageSnapshot } from '../page-state/types.ts';
import { pipelined } from '../page-state/transactions.ts';
import { FACTS_FENCE_BEGIN, FACTS_FENCE_END, parseFactsFence } from '../facts-fence.ts';
import { TAKES_FENCE_BEGIN, TAKES_FENCE_END, parseTakesFence } from '../takes-fence.ts';
import { extractFactsFromFenceText, pageFactDate } from '../facts/extract-from-fence.ts';
import { takesPreparation } from '../takes-write.ts';
import { LINK_EXTRACTOR_VERSION_TS, parseTimelineEntries } from '../link-extraction.ts';
import { eachStoredTimelineRow, extractTimelineFromContent, hasExtractorDetail, supersededTimelineReadings, TimelineKeySet,
  type ExtractedTimelineEntry } from '../timeline-extract.ts';
import { sanitizeRemoteBody } from '../remote-body.ts';
import { sanitizeForJsonb } from '../batch-rows.ts';
import { bodyUnread, isSegmentedPage } from '../page-body/page-shape.ts';
import { bodyBlocks, detachedValue, type BodyBlock } from '../page-body/segment-blocks.ts';
import { TakesFenceScan, takesFromRuns } from '../page-body/segment-takes.ts';
import { openFenceRuns } from '../page-body/segment-facts.ts';
import { derivationScope, readRuns, SegmentDerivationRaced, streamRuns, type DerivationScope, type DeriveRun } from '../page-body/segment-derive.ts';
import type { BodyRegion } from '../page-body/types.ts';
import { createHash } from 'node:crypto';
import { materializedMarker, materializedMarkerHash, timelineKey, timelineKeyHash } from '../timeline-marker.ts';
import type { OperationError } from '../ops/contract.ts';
import { protectedRegions } from '../fence-scan.ts';
import { FENCE_ROWS_MAX, fenceOperationError, scanCanonicalFences } from '../fence-repair/refusal.ts';
import type { FenceSection } from '../fence-repair/types.ts';
import { contestHeldRows, factsAboveWriter, guardFenceRows, guardFenceTakes, heldRows } from '../trust/fence-guard.ts';
import { propagateFactLowering } from '../trust/propagation.ts';
import { loadWriteGateConfig } from '../trust/gate-outcomes.ts';
import { fenceOwnerKey, readFenceBlocks, TAKE_FENCE_KIND } from '../generations/fence.ts';
import { projectFactFenceGeneration } from '../generations/adapters/fact-rows.ts';
import { openedFactRowSql } from './context.ts';
import { fenceRowKeys, moveKeptRows, pageFenceRowKeys } from '../generations/fence-evidence.ts';
import { projectTakeFenceGeneration } from '../generations/adapters/take-rows.ts';
import { linksFenceProjectionRows, projectLinksFence } from '../links-fence-projection.ts';

const FENCE_PAIRS=[{begin:FACTS_FENCE_BEGIN,end:FACTS_FENCE_END},{begin:TAKES_FENCE_BEGIN,end:TAKES_FENCE_END}];

/**
 * Rows held by fences whose begin sits in markdown code, keyed `row_num:text`,
 * with the section each sits in. Readers treat such a fence as an example and
 * project nothing from it, so a real fence wrapped in code would silently
 * delete the rows it holds.
 */
function quotedFenceRows(fields:Array<[FenceSection,string]>):{facts:Map<string,FenceSection>,takes:Map<string,FenceSection>} {
  const facts=new Map<string,FenceSection>(),takes=new Map<string,FenceSection>();
  for(const [section,field] of fields) for(const region of protectedRegions(field,FENCE_PAIRS).regions) {
    if(region.read) continue;
    const text=field.slice(region.start,region.end);
    if(region.pair===0) for(const f of parseFactsFence(text).facts) facts.set(`${f.rowNum}:${f.claim}`,section);
    else for(const t of parseTakesFence(text).takes) takes.set(`${t.rowNum}:${t.claim}`,section);
  }
  return {facts,takes};
}

type CanonicalBody = Pick<ParsedPage, 'compiled_truth' | 'timeline'>;

/**
 * The caller's authority over the prior canonical content (#5567):
 * `editing` writers are bound to the observed revision and render from the
 * database, `preserving` writers regenerate or overwrite without that binding,
 * `file` writers import file edits bound to the file preimage and never
 * rewrite the user's file, and `immutable` imports publish approved bytes they
 * may never extend.
 */
export type ProjectionWriter = 'editing' | 'preserving' | 'file' | 'immutable';

/**
 * How one stored timeline row relates to the write, judged at preparation:
 * `in_body` exactly matches a new bullet, `drifted` matches one only after
 * normalization, `removed` / `removed_marked` had an unmarked / materialized
 * bullet in the prior body that the new body dropped, `superseded` is what
 * an older reading (supersededTimelineReadings) filed for text either body
 * still or once carried (with no detail beyond what the extractor writes) and
 * the current reading replaces, `superseded_annotated` is such a row someone gave
 * its own detail, and `database_only` has no bullet in either body.
 */
export type TimelineRowState = 'in_body' | 'drifted' | 'removed' | 'removed_marked' | 'superseded' | 'superseded_annotated' | 'database_only';
export type TimelineRowAction = 'refresh_detail' | 'delete' | 'keep' | 'materialize';

/**
 *   row state      | editing        | preserving     | file           | immutable
 *   ---------------+----------------+----------------+----------------+---------------
 *   in_body        | refresh_detail | refresh_detail | refresh_detail | refresh_detail
 *   drifted        | delete         | delete         | delete         | delete
 *   removed        | delete         | delete         | delete         | delete
 *   removed_marked | delete         | materialize    | delete         | keep
 *   superseded     | delete         | delete         | delete         | delete
 *   superseded_ann.| keep           | keep           | keep           | keep
 *   database_only  | materialize    | materialize    | keep           | keep
 *
 * A coordinated write deletes only rows whose bullet the writer can see in the
 * prior or new body. A materialized bullet is removed only by a writer bound to
 * a revision or file preimage that contained it; a preserving writer renders it
 * again. Writers that render from the database write bullet-less rows back into
 * the page (`materialize`); rows that fail the render round trip, and every
 * `materialize` row a caller did not render, are kept. A superseded row is
 * never written back (it would re-file the older, misdated reading); one
 * carrying its own detail stays in the database only (#6226, taste T3). `put_page` with the
 * current revision is the supported way to delete a materialized row. Deletes
 * and detail refreshes also require the row id and detail pinned at
 * preparation, so rows that change afterwards are left alone.
 */
const TIMELINE_DECISIONS: Record<ProjectionWriter, Record<TimelineRowState, TimelineRowAction>> = {
  editing: { in_body: 'refresh_detail', drifted: 'delete', removed: 'delete', removed_marked: 'delete', superseded: 'delete', superseded_annotated: 'keep', database_only: 'materialize' },
  preserving: { in_body: 'refresh_detail', drifted: 'delete', removed: 'delete', removed_marked: 'materialize', superseded: 'delete', superseded_annotated: 'keep', database_only: 'materialize' },
  file: { in_body: 'refresh_detail', drifted: 'delete', removed: 'delete', removed_marked: 'delete', superseded: 'delete', superseded_annotated: 'keep', database_only: 'keep' },
  immutable: { in_body: 'refresh_detail', drifted: 'delete', removed: 'delete', removed_marked: 'keep', superseded: 'delete', superseded_annotated: 'keep', database_only: 'keep' },
};

/**
 * #5969 (D3): what an ordinary put_page / put_pages body says about its
 * Timeline section (`timeline-omission.ts`); other writers pass none and keep
 * the table above. Only rows the writer saw (`removed`, `removed_marked`)
 * change: an explicitly empty section deletes them for every writer class; a
 * body with no section at all keeps them for a local preserving writer
 * (rendered back like a marked row) and, with `drop_timeline`, deletes them.
 */
export interface TimelineWritePolicy { section: 'present' | 'emptied' | 'omitted'; drop: boolean; preserveOmitted: boolean }

export function timelineRowAction(writer: ProjectionWriter, state: TimelineRowState, policy?: TimelineWritePolicy): TimelineRowAction {
  const action = TIMELINE_DECISIONS[writer][state];
  if (!policy || (state !== 'removed' && state !== 'removed_marked')) return action;
  if (policy.section === 'emptied' || (policy.section === 'omitted' && policy.drop)) return 'delete';
  return policy.section === 'omitted' && policy.preserveOmitted ? 'materialize' : action;
}

function exactTimelineKey(entry: { date: string; source?: string | null; summary: string }): string {
  return JSON.stringify([entry.date.slice(0, 10), sanitizeForJsonb(entry.source ?? ''), sanitizeForJsonb(entry.summary)]);
}

function safeBody(body: CanonicalBody): string {
  return sanitizeRemoteBody([body.compiled_truth, body.timeline ?? ''].join('\n'), { keepMaterializedMarkers: true });
}

function extractTimeline(safe: string, slug: string): Map<string, ExtractedTimelineEntry> {
  const timeline = new Map(extractTimelineFromContent(safe, slug).map(t => [timelineKey(t), t]));
  for (const t of parseTimelineEntries(safe)) timeline.set(timelineKey({ ...t, source: t.source ?? 'markdown' }), { ...t, source: t.source ?? 'markdown', slug });
  return timeline;
}

function canonicalTimeline(body: CanonicalBody, slug: string): Map<string, ExtractedTimelineEntry> {
  return extractTimeline(safeBody(body), slug);
}

/** Tuples of a sanitized text whose bullet is introduced by a marker that still matches it. */
function markedTimeline(safe: string, slug: string, marked = new Set<string>()): Set<string> {
  const lines = safe.split('\n');
  lines.forEach((line, i) => {
    const hash = materializedMarkerHash(line);
    if (!hash || i + 1 >= lines.length) return;
    for (const key of extractTimeline(lines[i + 1], slug).keys()) if (timelineKeyHash(key) === hash) marked.add(key);
  });
  return marked;
}

/** What a body says about timeline rows: its canonical timeline, the tuples a marker still matches, the superseded older readings. */
interface TimelineView { entries: Map<string, ExtractedTimelineEntry>; marked: Set<string>; superseded: Set<string> }

function timelineView(body: CanonicalBody, slug: string): TimelineView {
  const safe = safeBody(body);
  return { entries: extractTimeline(safe, slug), marked: markedTimeline(safe, slug), superseded: new Set(supersededTimelineReadings(safe).map(t => timelineKey(t))) };
}

/**
 * What a prior body says about the stored rows a write classifies, by the row's index in the rows read and its key:
 * a bullet (`entry`), a bullet a marker still matches (`marked`), a superseded older reading (`superseded`).
 */
export interface PriorRows { entry(i: number, key: string): boolean; marked(i: number, key: string): boolean; superseded(i: number, key: string): boolean }

/** The views a prior snapshot contributes, plus its takes rows per field and whether either field's takes fence warns. */
export interface PriorView { rows: PriorRows; takeRows: Set<number>; takesWarn: boolean }

function viewRows(view: TimelineView): PriorRows {
  return { entry: (_, key) => view.entries.has(key), marked: (_, key) => view.marked.has(key), superseded: (_, key) => view.superseded.has(key) };
}

const PRIOR_ENTRY = 1, PRIOR_MARKED = 2, PRIOR_SUPERSEDED = 4;

/**
 * The segmented prior view per snapshot page object, for the stored rows it was asked about (their keys' SHA-256): a
 * write asks for it at preparation, materialization and removal reporting, and each call reads the same revision.
 */
const segmentedPriors = new WeakMap<object, { digest: string; view: PriorView }>();

/**
 * The prior view of a stored segmented page whose body is unread, in one pass block by block (page-body/segment-blocks.ts)
 * and cached for the snapshot (segmentedPriors). Each block is sanitized alone, as the segmenter scans it; every bullet
 * and heading entry and parseTimelineEntries' readings of it count as the body's entries. Only the stored rows asked
 * about are kept, one flag byte each, never the body's entries. A `###` dated entry's detail ends at its block. The
 * takes rows are parsed per field, as canonicalTakeRows does (page-body/segment-takes.ts), in the same pass.
 */
export async function segmentedPriorView(engine: BrainEngine, page: { id: number; slug: string; source_id: string }, slug: string, keys: readonly string[]): Promise<PriorView> {
  const digest = createHash('sha256').update(keys.join('\0')).digest('hex');
  const cached = segmentedPriors.get(page);
  if (cached?.digest === digest) return cached.view;
  const index = new Map<string, number[]>();
  keys.forEach((key, i) => { const at = index.get(key); if (at) at.push(i); else index.set(key, [i]); });
  const flags = new Uint8Array(keys.length);
  const mark = (key: string, bit: number) => { for (const i of index.get(key) ?? []) flags[i]! |= bit; };
  const takes = new Map<BodyRegion, TakesFenceScan>((['truth', 'timeline'] as const).map(region => [region, new TakesFenceScan(page)]));
  let region: BodyRegion | null = null;
  let regionAt = 0;
  for await (const block of bodyBlocks(engine, page)) {
    if (block.region && block.region !== region) { region = block.region; regionAt = block.at; }
    if (block.region) takes.get(block.region)!.push({ text: block.text, at: block.at - regionAt });
    const safe = sanitizeRemoteBody(block.text, { keepMaterializedMarkers: true });
    for (const t of extractTimelineFromContent(safe, slug)) mark(timelineKey(t), PRIOR_ENTRY);
    for (const t of parseTimelineEntries(safe)) mark(timelineKey({ ...t, source: t.source ?? 'markdown' }), PRIOR_ENTRY);
    for (const key of markedTimeline(safe, slug)) mark(key, PRIOR_MARKED);
    for (const t of supersededTimelineReadings(safe)) mark(timelineKey(t), PRIOR_SUPERSEDED);
  }
  const fields = [...takes.values()].map(scan => scan.result());
  const view: PriorView = {
    rows: { entry: i => (flags[i]! & PRIOR_ENTRY) !== 0, marked: i => (flags[i]! & PRIOR_MARKED) !== 0, superseded: i => (flags[i]! & PRIOR_SUPERSEDED) !== 0 },
    takeRows: new Set(fields.flatMap(field => field.parsed.takes.map(t => t.rowNum))), takesWarn: fields.some(field => field.parsed.warnings.length > 0),
  };
  segmentedPriors.set(page, { digest, view });
  return view;
}

/** The timeline keys one block of a segmented body yields, as segmentedPriorView and segmentedCanonicalCollector read them. */
function blockTimeline(text: string, slug: string) {
  const safe = sanitizeRemoteBody(text, { keepMaterializedMarkers: true });
  return {
    extracted: extractTimelineFromContent(safe, slug).map(t => [timelineKey(t), t] as const),
    parsed: parseTimelineEntries(safe).map(t => [timelineKey({ ...t, source: t.source ?? 'markdown' }), { ...t, source: t.source ?? 'markdown', slug }] as const),
    marked: markedTimeline(safe, slug),
    superseded: supersededTimelineReadings(safe).map(t => timelineKey(t)),
  };
}

/**
 * What one run of a segmented body holds for the timeline (spec 8.2): the timelineDigest of the timelineKey of each
 * entry its blocks yield (extracted, parsed; each in block order, repeats kept), of each materialized marker and of each
 * superseded older reading. Numbers, not entries, so a row and the read that builds it stay a small fraction of the run's
 * text even when forced cuts join a whole timeline into one run.
 */
interface TimelineRunOutput { x: number[]; p: number[]; m: number[]; s: number[] }

/** The first 52 bits of a timeline key's SHA-256, as an exact JS number. */
const timelineDigest = (key: string) => Number.parseInt(createHash('sha256').update(key).digest('hex').slice(0, 13), 16);

/**
 * Each run's timeline row output, in page order, handed to `visit` (page-body/segment-derive.ts streamRuns); every block
 * a due run is read for also goes to `onBlock`. Returns the scope, or null when the page is not segmented.
 */
async function eachTimelineRun(engine: BrainEngine, page: { id: number; slug: string; source_id: string }, slug: string,
  visit: (run: DeriveRun, output: TimelineRunOutput) => void,
  opts: { scope?: DerivationScope; onBlock?: (run: DeriveRun | null, block: BodyBlock) => void } = {}): Promise<DerivationScope | null> {
  const scope = opts.scope ?? await derivationScope(engine, { id: Number(page.id), slug: page.slug, source_id: page.source_id });
  if (!scope) return null;
  const inputs = () => ({ slug, extractor: LINK_EXTRACTOR_VERSION_TS });
  await streamRuns<TimelineRunOutput>(engine, 'timeline', scope, inputs, {
    start: () => ({ x: [], p: [], m: [], s: [] }),
    block: (out, block) => {
      const safe = sanitizeRemoteBody(block.text, { keepMaterializedMarkers: true });
      for (const t of extractTimelineFromContent(safe, slug)) out.x.push(timelineDigest(timelineKey(t)));
      for (const t of parseTimelineEntries(safe)) out.p.push(timelineDigest(timelineKey({ ...t, source: t.source ?? 'markdown' })));
      for (const key of markedTimeline(safe, slug)) out.m.push(timelineDigest(key));
      for (const t of supersededTimelineReadings(safe)) out.s.push(timelineDigest(timelineKey(t)));
    },
  }, visit, opts.onBlock);
  return scope;
}

const raced = (error: unknown): null => {
  if (error instanceof SegmentDerivationRaced) return null;
  throw error;
};

/**
 * segmentedPriorView from the page's timeline and fences rows: only runs changed since their rows were written are read.
 * Null (the caller takes the full pass) when the page is not segmented, a takes fence runs past the run it begins in,
 * or the body moves during the read.
 */
export async function incrementalPriorView(engine: BrainEngine, page: { id: number; slug: string; source_id: string }, slug: string, keys: readonly string[]): Promise<PriorView | null> {
  const index = new Map<number, number[]>();
  keys.forEach((key, i) => { const digest = timelineDigest(key); const at = index.get(digest); if (at) at.push(i); else index.set(digest, [i]); });
  const flags = new Uint8Array(keys.length);
  const mark = (digest: number, bit: number) => { for (const i of index.get(digest) ?? []) flags[i]! |= bit; };
  // One read serves both kinds: a run due for its fences row is scanned while the timeline reads it.
  const fences = async () => {
    const scope = await derivationScope(engine, { id: Number(page.id), slug: page.slug, source_id: page.source_id });
    if (!scope) return null;
    const runs = await openFenceRuns(engine, page, scope);
    await eachTimelineRun(engine, page, slug, (_, out) => {
      for (const digest of out.x) mark(digest, PRIOR_ENTRY);
      for (const digest of out.p) mark(digest, PRIOR_ENTRY);
      for (const digest of out.m) mark(digest, PRIOR_MARKED);
      for (const digest of out.s) mark(digest, PRIOR_SUPERSEDED);
    }, { scope, onBlock: runs.push });
    return runs.finish();
  };
  const runs = await fences().catch(raced);
  if (!runs) return null;
  const fields = (['truth', 'timeline'] as const).map(region => takesFromRuns(runs.filter(({ run }) => run.region === region).map(({ output }) => output.takes)));
  if (fields.some(field => field === null)) return null;
  return {
    rows: { entry: i => (flags[i]! & PRIOR_ENTRY) !== 0, marked: i => (flags[i]! & PRIOR_MARKED) !== 0, superseded: i => (flags[i]! & PRIOR_SUPERSEDED) !== 0 },
    takeRows: new Set(fields.flatMap(field => field!.parsed.takes.map(t => t.rowNum))), takesWarn: fields.some(field => field!.parsed.warnings.length > 0),
  };
}

/**
 * segmentedCanonicalTimeline from the page's timeline rows: the entries no stored row records are found from the
 * digests, and only the runs that last yield one of them are read for its entry (plus runs changed since their row was
 * written). Null (the caller takes the full pass) when the page is not segmented or its body moves during the read.
 */
async function incrementalCanonicalTimeline(engine: BrainEngine, page: { id: number; slug: string; source_id: string }, slug: string): Promise<Map<string, ExtractedTimelineEntry> | null> {
  const recorded = new TimelineKeySet();
  await eachStoredTimelineRow(engine, Number(page.id), row => {
    canonicalTimelineWork.storedRows++;
    recorded.add(String(timelineDigest(timelineKey({ date: row.date_text, source: row.source, summary: row.summary }))));
  });
  // Each unrecorded digest's run of last occurrence, in first-occurrence order (the full pass's Map semantics).
  const extracted = new Map<number, number>(), parsed = new Map<number, number>();
  const scope = await eachTimelineRun(engine, page, slug, (run, out) => {
    for (const digest of out.x) if (!recorded.has(String(digest))) extracted.set(digest, run.index);
    for (const digest of out.p) if (!recorded.has(String(digest))) parsed.set(digest, run.index);
  });
  if (!scope) return null;
  const entries = new Map<number, { x: Map<number, readonly [string, ExtractedTimelineEntry]>; p: Map<number, readonly [string, ExtractedTimelineEntry]> }>();
  const wanted = new Set([...extracted.values(), ...parsed.values()]);
  await readRuns(engine, scope, wanted, (run, block) => {
    if (!run) return;
    const into = entries.get(run.index) ?? { x: new Map(), p: new Map() };
    entries.set(run.index, into);
    const found = blockTimeline(block.text, slug);
    for (const [key, entry] of found.extracted) if (extracted.get(timelineDigest(key)) === run.index) into.x.set(timelineDigest(key), [key, detachedValue(entry)]);
    for (const [key, entry] of found.parsed) if (parsed.get(timelineDigest(key)) === run.index) into.p.set(timelineDigest(key), [key, detachedValue(entry)]);
  });
  const kept = new Map<string, ExtractedTimelineEntry>();
  for (const [digest, at] of extracted) { const [key, entry] = entries.get(at)!.x.get(digest)!; kept.set(key, entry); }
  for (const [digest, at] of parsed) { const [key, entry] = entries.get(at)!.p.get(digest)!; kept.set(key, entry); }
  return kept;
}

/** The prior snapshot's view of `rows` (its stored timeline rows): from its strings, or streamed when its segmented body is unread. */
async function priorView(engine: BrainEngine, page: PageSnapshot['page'], slug: string, rows: readonly StoredTimelineRow[]): Promise<PriorView> {
  if (bodyUnread(page)) {
    const keys = rows.map(row => timelineKey(row));
    return await incrementalPriorView(engine, page, slug, keys) ?? segmentedPriorView(engine, page, slug, keys);
  }
  const fields = [page.compiled_truth, page.timeline ?? ''].map(field => parseTakesFence(field));
  return { rows: viewRows(timelineView(page, slug)),
    takeRows: new Set(fields.flatMap(field => field.takes.map(t => t.rowNum))), takesWarn: fields.some(field => field.warnings.length > 0) };
}

/**
 * The canonical timeline of a segmented body fed one block at a time (`add`), extractTimeline's two passes merged in its
 * order (every bullet and heading entry, then parseTimelineEntries' readings over them). An entry whose timelineKey is in
 * `recorded` (the page's stored rows) is not kept: unrecordedCanonicalTimeline drops it, and once a page is extracted
 * its recorded entries are nearly all of it.
 */
export function segmentedCanonicalCollector(slug: string, recorded?: { has(key: string): boolean }): { add(text: string): void; result(): Map<string, ExtractedTimelineEntry> } {
  const extracted = new Map<string, ExtractedTimelineEntry>();
  const parsed = new Map<string, ExtractedTimelineEntry>();
  return {
    add(text) {
      const safe = sanitizeRemoteBody(text, { keepMaterializedMarkers: true });
      for (const t of extractTimelineFromContent(safe, slug)) {
        const key = timelineKey(t);
        if (!recorded?.has(key)) extracted.set(key, detachedValue(t));
      }
      for (const t of parseTimelineEntries(safe)) {
        const key = timelineKey({ ...t, source: t.source ?? 'markdown' });
        if (!recorded?.has(key)) parsed.set(key, detachedValue({ ...t, source: t.source ?? 'markdown', slug }));
      }
    },
    result() {
      for (const [key, entry] of parsed) extracted.set(key, entry);
      return extracted;
    },
  };
}

/**
 * @internal Work of the managed timeline extraction over a page (`storedRows`: stored rows walked by
 * unrecordedCanonicalTimeline and segmentedCanonicalTimeline; `kept`: canonical entries a segmented body's read kept).
 * Counts, not time.
 */
export const canonicalTimelineWork = { storedRows: 0, kept: 0 };

/**
 * The canonical timeline of a stored segmented page whose body is unread, block by block (segmentedCanonicalCollector),
 * less the entries a stored row of the page records: the rows are walked in pages and held as key digests
 * (TimelineKeySet), so the read keeps what an insert-only caller may add, never every entry the body yields.
 */
async function segmentedCanonicalTimeline(engine: BrainEngine, page: { id: number; slug: string; source_id: string }, slug: string): Promise<Map<string, ExtractedTimelineEntry>> {
  const incremental = await incrementalCanonicalTimeline(engine, page, slug).catch(raced);
  if (incremental) {
    canonicalTimelineWork.kept += incremental.size;
    return incremental;
  }
  const recorded = new TimelineKeySet();
  await eachStoredTimelineRow(engine, Number(page.id), row => {
    canonicalTimelineWork.storedRows++;
    recorded.add(timelineKey({ date: row.date_text, source: row.source, summary: row.summary }));
  });
  const collector = segmentedCanonicalCollector(slug, recorded);
  for await (const block of bodyBlocks(engine, page)) collector.add(block.text);
  const kept = collector.result();
  canonicalTimelineWork.kept += kept.size;
  return kept;
}

/**
 * The canonical timeline of a stored page (a snapshot's page) for an insert-only caller: a segmented body is read
 * block by block and keeps only entries no stored row records. Read it before a transaction; unrecordedCanonicalTimeline
 * filters it inside one.
 */
export async function storedCanonicalTimeline(engine: BrainEngine, page: PageSnapshot['page'], slug: string): Promise<Map<string, ExtractedTimelineEntry>> {
  return bodyUnread(page) ? segmentedCanonicalTimeline(engine, page, slug) : canonicalTimeline(page, slug);
}

/**
 * Character ranges of materialized timeline history in a raw body: each marker
 * line whose hash matches the bullet after it, that bullet, and its indented
 * detail lines. This is database history written back into the page, not
 * newly authored text.
 */
export function materializedHistoryRanges(body: string): Array<[number, number]> {
  const lines = body.split('\n');
  const starts: number[] = [];
  let offset = 0;
  for (const line of lines) { starts.push(offset); offset += line.length + 1; }
  const ranges: Array<[number, number]> = [];
  lines.forEach((line, i) => {
    const hash = materializedMarkerHash(line);
    if (!hash || i + 1 >= lines.length) return;
    if (![...extractTimeline(lines[i + 1], '').keys()].some(key => timelineKeyHash(key) === hash)) return;
    let last = i + 1;
    while (last + 1 < lines.length && /^[ \t]+\S/.test(lines[last + 1]) && !/^[ \t]*[-*+] /.test(lines[last + 1])) last++;
    ranges.push([starts[i], starts[last] + lines[last].length]);
  });
  return ranges;
}

interface StoredTimelineRow { id: number; date: string; source: string; summary: string; detail: string }

/** The page's stored takes a fence's new rows (`$2`) would collide with: the same row number, another claim, kind or holder. */
export const TAKE_COLLISIONS_SQL = `SELECT k.row_num FROM takes k
    JOIN jsonb_to_recordset($2::text::jsonb) AS n(row_num integer,claim text,kind text,holder text) ON n.row_num=k.row_num
    WHERE k.page_id=$1 AND (k.claim,k.kind,k.holder) IS DISTINCT FROM (n.claim,n.kind,n.holder) ORDER BY k.row_num LIMIT $3`;

/** A page's own (non-event) timeline rows, in date order. */
export const STORED_TIMELINE_SQL = 'SELECT id,date::text AS date,source,summary,detail FROM timeline_entries WHERE page_id=$1 AND event_page_id IS NULL ORDER BY date,id';

function storedTimeline(engine: BrainEngine, pageId: number): Promise<StoredTimelineRow[]> {
  return engine.executeRaw<StoredTimelineRow>(STORED_TIMELINE_SQL, [pageId]);
}

/**
 * Timeline tuples the coordinator projects from a canonical page body that
 * have no stored row on the page under the same normalized key. Insert-only
 * callers (managed `extract --stale`) add exactly these, so a stored row that
 * differs only by whitespace is not duplicated. `storedRows` is the caller's
 * own read of the page's non-event rows (a batched walk); without it the rows
 * are walked in pages (eachStoredTimelineRow), never held as one array.
 */
export async function unrecordedCanonicalTimeline(engine: BrainEngine, pageId: number, body: CanonicalBody | ReadonlyMap<string, ExtractedTimelineEntry>,
  slug: string, storedRows?: ReadonlyArray<Pick<StoredTimelineRow, 'date' | 'source' | 'summary'>>): Promise<ExtractedTimelineEntry[]> {
  const entries = body instanceof Map ? body : canonicalTimeline(body as CanonicalBody, slug);
  if (storedRows) {
    const stored = new Set(storedRows.map(row => timelineKey(row)));
    return [...entries].filter(([key]) => !stored.has(key)).map(([, entry]) => entry);
  }
  const unrecorded = new Map(entries);
  await eachStoredTimelineRow(engine, pageId, row => {
    canonicalTimelineWork.storedRows++;
    unrecorded.delete(timelineKey({ date: row.date_text, source: row.source, summary: row.summary }));
  });
  return [...unrecorded.values()];
}

/** Classify stored rows against a new body and the writer's prior snapshot's view. */
function classifyTimeline(rows: StoredTimelineRow[], body: CanonicalBody, prior: PriorRows | null, slug: string, writer: ProjectionWriter,
  policy?: TimelineWritePolicy) {
  const incoming = timelineView(body, slug);
  const timeline = incoming.entries;
  const exactIncoming = new Map([...timeline.values()].map(t => [exactTimelineKey(t), sanitizeForJsonb(t.detail ?? '')]));
  const pinned = rows.map((row, i) => {
    const key = timelineKey(row);
    const state: TimelineRowState = exactIncoming.has(exactTimelineKey(row)) ? 'in_body' : timeline.has(key) ? 'drifted'
      : prior?.marked(i, key) ? 'removed_marked' : prior?.entry(i, key) ? 'removed'
      : incoming.superseded.has(key) || prior?.superseded(i, key) ? (hasExtractorDetail(row) ? 'superseded' : 'superseded_annotated') : 'database_only';
    return { ...row, key, state, action: timelineRowAction(writer, state, policy) };
  });
  return { timeline, exactIncoming, pinned };
}

/** True when the body carries at least one timeline entry the coordinator would project. */
export function bodyHasTimelineEntries(body: CanonicalBody, slug: string): boolean {
  return canonicalTimeline(body, slug).size > 0;
}

/** Dates of the timeline rows a write removed (or would remove): the rows it saw in the prior body and dropped. */
export interface TimelineRowsRemoved { count: number; earliest: string; latest: string }

function removedSummary(dates: string[]): TimelineRowsRemoved | null {
  if (!dates.length) return null;
  const sorted = dates.map(d => d.slice(0, 10)).sort();
  return { count: sorted.length, earliest: sorted[0]!, latest: sorted[sorted.length - 1]! };
}

/** The stored rows this writer would delete because the new body dropped their bullets, judged now. */
export async function timelineRowsToRemove(engine: BrainEngine, body: CanonicalBody, prior: PageSnapshot, slug: string,
  writer: ProjectionWriter, policy?: TimelineWritePolicy): Promise<TimelineRowsRemoved | null> {
  const rows = await storedTimeline(engine, prior.page.id);
  const { pinned } = classifyTimeline(rows, body, (await priorView(engine, prior.page, slug, rows)).rows, slug, writer, policy);
  return removedSummary(pinned.filter(row => row.action === 'delete' && (row.state === 'removed' || row.state === 'removed_marked')).map(row => row.date));
}

const collapse = (text: string) => sanitizeForJsonb(text).replace(/\s+/g, ' ').trim();
export const TIMELINE_COMMENT_MARKUP = /<!--|-->/;

/**
 * Render one row as a marked bullet, or null when render-then-extract would
 * not return exactly this tuple and its normalized detail (delimiters in the
 * source, `Referenced in [` backlink receipts, empty sources, ...).
 */
export function renderMaterializedBullet(row: { date: string; source: string; summary: string; detail?: string | null }, slug: string): string | null {
  // Same normalization as timelineKey: sources are trimmed, summaries and details collapsed.
  const tuple = { date: row.date.slice(0, 10), source: sanitizeForJsonb(row.source).trim(), summary: collapse(row.summary) };
  const detail = collapse(row.detail ?? '');
  // Pre-#4277 backlink receipts are graph noise the extractors deliberately skip.
  if (/^Referenced in\s+\[/i.test(tuple.summary)) return null;
  // #6184: a row carrying HTML comment markup (a junk row the citation parser filed from a section END marker)
  // would write a second copy of that marker into the page; it stays database-side until `gbrain repair timeline-comments`.
  if (TIMELINE_COMMENT_MARKUP.test(`${row.source}\n${row.summary}\n${row.detail ?? ''}`)) return null;
  const block = [materializedMarker(tuple), `- **${tuple.date}** | ${tuple.source} — ${tuple.summary}`, ...(detail ? [`  ${detail}`] : [])].join('\n');
  const extracted = [...canonicalTimeline({ compiled_truth: block, timeline: '' }, slug).values()];
  if (extracted.length !== 1) return null;
  const [entry] = extracted;
  const same = entry.date === tuple.date && entry.source === tuple.source && entry.summary === tuple.summary && (entry.detail ?? '') === detail;
  return same && markedTimeline(safeBody({ compiled_truth: block, timeline: '' }), slug).size === 1 ? block : null;
}

export interface TimelineMaterialization { timeline: string; materialized: number; unrenderable: number }

/**
 * Write the writer's `materialize` rows back into the page's timeline section,
 * after existing bullets in date-then-row-id order, one bullet per normalized
 * tuple. Called during preparation, before the body is imported, rendered and
 * digested, by writers that render the canonical file from the database.
 */
export async function materializeTimeline(engine: BrainEngine, body: CanonicalBody, slug: string,
  prior: PageSnapshot | null, writer: ProjectionWriter, policy?: TimelineWritePolicy): Promise<TimelineMaterialization> {
  const timelineText = body.timeline ?? '';
  if (!prior) return { timeline: timelineText, materialized: 0, unrenderable: 0 };
  const rows = await storedTimeline(engine, prior.page.id);
  const { pinned } = classifyTimeline(rows, body, (await priorView(engine, prior.page, slug, rows)).rows, slug, writer, policy);
  const blocks: string[] = [];
  const seen = new Set<string>();
  let unrenderable = 0;
  for (const row of pinned) {
    if (row.action !== 'materialize' || seen.has(row.key)) continue;
    const block = renderMaterializedBullet(row, slug);
    if (!block) { unrenderable++; continue; }
    seen.add(row.key);
    blocks.push(block);
  }
  if (!blocks.length) return { timeline: timelineText, materialized: 0, unrenderable };
  const existing = timelineText.replace(/\s+$/, '');
  return { timeline: [...(existing ? [existing] : []), ...blocks].join('\n'), materialized: blocks.length, unrenderable };
}

/** Database-only timeline rows on one page, split by whether they can be materialized. */
export async function pendingTimelineRows(engine: BrainEngine, page: { id: number; slug: string } & CanonicalBody, exclude: ReadonlySet<number> = new Set()) {
  const { pinned } = classifyTimeline(await storedTimeline(engine, page.id), page, viewRows(timelineView(page, page.slug)), page.slug, 'editing');
  const renderable = new Set<string>();
  let unrenderable = 0;
  for (const row of pinned) {
    if (row.action !== 'materialize' || exclude.has(Number(row.id))) continue;
    if (renderMaterializedBullet(row, page.slug)) renderable.add(row.key);
    else unrenderable++;
  }
  return { materializable: renderable.size, unrenderable };
}

/**
 * Validate a canonical body and compile its provider-free projections. A fence
 * the shared scan refuses throws typed `invalid_fence` (wire `invalid_params`)
 * naming the first defect's fence, section and reason.
 */
/** #6574: a canonical projection at least this slow logs its steps. */
const SLOW_PROJECTION_MS = 30_000;

/** Rows per timeline INSERT (#6574): each statement stays well inside the publication's 5 s statement_timeout. */
const TIMELINE_INSERT_CHUNK = 500;

function timelineChunks(timeline: ReadonlyMap<string, ExtractedTimelineEntry>): ExtractedTimelineEntry[][] {
  const entries = [...timeline.values()];
  const chunks: ExtractedTimelineEntry[][] = [];
  for (let at = 0; at < entries.length; at += TIMELINE_INSERT_CHUNK) chunks.push(entries.slice(at, at + TIMELINE_INSERT_CHUNK));
  return chunks;
}

/**
 * #6574: a page's canonical timeline rows in one statement per chunk, in body order. The publication already holds the page's
 * row lock (lockPageKeys), so the rows go straight to its id; free text is sanitized as addTimelineEntry does (#2011).
 */
function insertCanonicalTimeline(tx: BrainEngine, pageId: number, entries: ExtractedTimelineEntry[]) {
  const rows = JSON.stringify(entries.map((entry, ord) => ({ ord, date: entry.date, source: sanitizeForJsonb(entry.source || ''),
    summary: sanitizeForJsonb(entry.summary), detail: sanitizeForJsonb(entry.detail || '') })));
  return tx.executeRaw(`INSERT INTO timeline_entries (page_id,date,source,summary,detail)
    SELECT $1,r.date::date,r.source,r.summary,r.detail FROM jsonb_to_recordset($2::text::jsonb) AS r(ord integer,date text,source text,summary text,detail text)
    ORDER BY r.ord
    ON CONFLICT (page_id,date,md5(summary),source) DO NOTHING`, [pageId, rows]);
}

/** The page body's own entry for one timeline row (same date, source and summary), inserted when the page carries it. */
export async function reprojectTimelineRow(tx: BrainEngine, pageId: number, body: CanonicalBody, slug: string,
  row: { date: string; source: string | null; summary: string }): Promise<boolean> {
  const entry = canonicalTimeline(body, slug).get(timelineKey({ ...row, date: String(row.date) }));
  if (!entry) return false;
  await insertCanonicalTimeline(tx, pageId, [entry]);
  return true;
}

export function compileCanonicalProjections(page: ParsedPage, slug: string, sourceId: string, factDate: Date | null = null) {
  const scan=scanCanonicalFences({ ...page, slug });
  if(scan.defects.length) throw fenceOperationError(scan.defects[0]!, slug, sourceId);
  const fields:Array<[FenceSection,string]>=[['body',page.compiled_truth],['timeline',page.timeline ?? '']];
  return { factRows: extractFactsFromFenceText(scan.facts,slug,sourceId,{ pageEffectiveDate: factDate }), takes: scan.takes, sections: scan.sections, quoted: scan.quoting ? quotedFenceRows(fields) : null };
}

/**
 * Refuse, rather than silently delete, rows a fence quoted in markdown code
 * still holds: readers project nothing from such a fence, so a real fence
 * wrapped in a code block would otherwise expire its facts and drop its takes.
 */
async function refuseQuotedFenceLoss(tx: BrainEngine, pageId: number, quoted: { facts: Map<string, FenceSection>; takes: Map<string, FenceSection> },
  takeRowsGone: number[], factRows: ReturnType<typeof extractFactsFromFenceText>, slug: string, sourceId: string): Promise<void> {
  const refuse = (fence: 'facts' | 'takes', lost: Array<{ row_num: number; section: FenceSection }>) => fenceOperationError({ reason: 'quoted_fence_rows', fence,
    section: lost[0]!.section, rows: [...new Set(lost.map(r => Number(r.row_num)))].slice(0, FENCE_ROWS_MAX), columns: [], line: null }, slug, sourceId);
  if (quoted.takes.size && takeRowsGone.length) {
    const gone = await tx.executeRaw<{ row_num: number; claim: string }>('SELECT row_num,claim FROM takes WHERE page_id=$1 AND row_num=ANY($2::integer[])', [pageId, takeRowsGone]);
    const lost = gone.flatMap(r => { const section = quoted.takes.get(`${r.row_num}:${r.claim}`); return section ? [{ row_num: r.row_num, section }] : []; });
    if (lost.length) throw refuse('takes', lost);
  }
  if (quoted.facts.size) {
    const kept = new Set(factRows.map(f => `${f.row_num}:${f.fact}:${f.visibility}`));
    const live = await tx.executeRaw<{ row_num: number; fact: string; visibility: string }>(`SELECT row_num,fact,visibility FROM facts
      WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL`, [sourceId, slug]);
    const lost = live.flatMap(r => { const section = kept.has(`${r.row_num}:${r.fact}:${r.visibility}`) ? undefined : quoted.facts.get(`${r.row_num}:${r.fact}`);
      return section ? [{ row_num: r.row_num, section }] : []; });
    if (lost.length) throw refuse('facts', lost);
  }
}

/** A new fence row's number already names a different stored take: typed `invalid_fence`, wire `take_row_collision` (E4). */
function takeCollision(rows: number[], sections: Map<number, FenceSection>, slug: string, sourceId: string): OperationError {
  return fenceOperationError({ reason: 'stored_row_collision', fence: 'takes', section: sections.get(rows[0]!) ?? 'body', rows: rows.slice(0, FENCE_ROWS_MAX), columns: [], line: null },
    slug, sourceId, { legacy_error: 'take_row_collision' });
}

/**
 * Prepare provider-free projections outside publication. `prior` is the
 * caller's snapshot at its observed revision; stored timeline rows and take
 * row numbers are pinned here so the publication transaction only removes
 * what this writer actually edited. A prior body left unread (a segmented page
 * over the single-string ceiling) is read block by block (segmentedPriorView).
 */
export async function prepareCanonicalProjections(engine: BrainEngine, page: ParsedPage, slug: string, sourceId: string,
  prior: PageSnapshot | null, writer: ProjectionWriter, policy?: TimelineWritePolicy): Promise<(tx: BrainEngine, pageId?: number) => Promise<CanonicalProjectionResult>> {
  // #7247: a fence row without a valid_from takes the page's stated date, as extract_facts does, never the import time.
  const factDate = pageFactDate({ slug, frontmatter: page.frontmatter, timeZone: await engine.getConfig('brain.timezone') });
  const { factRows, takes, quoted, sections } = compileCanonicalProjections(page, slug, sourceId, factDate);
  // #6259: a page the content-quality gate hid as junk projects no facts or takes. Rows projected before it
  // was quarantined are left as they are (hiding them would need a new row state); only new projection stops.
  const projectFences = !isQuarantined(page.frontmatter);
  // GBRA-161 PR 2: the incoming body's fences decide the fact and take generations; an absent one retires nothing.
  const fenceReads = { facts: readFenceBlocks(page, 'facts'), takes: readFenceBlocks(page, 'takes') };
  // G5: the links fence projects its own edges; a body with no fence (or one that does not parse) leaves them as they are.
  const fenceLinks = projectFences && !bodyUnread(page) ? await linksFenceProjectionRows(engine, sourceId, page) : null;
  const storedRows = prior ? await storedTimeline(engine, prior.page.id) : [];
  const prior_ = prior ? await priorView(engine, prior.page, slug, storedRows) : null;
  // #8819: the fence rows of the body this write replaces, read at the observed revision, for the fact generation's evidence.
  // An inline body parses without a statement; a segmented one is read from its segments only when the page has positioned facts.
  const priorFenceKeys = !prior || !projectFences ? null
    : !isSegmentedPage(prior.page) ? fenceRowKeys(prior.page, slug, sourceId)
    : (await engine.executeRaw(`SELECT 1 FROM facts WHERE source_id=$1 AND source_markdown_slug=$2
        AND row_num IS NOT NULL AND COALESCE(source,'') NOT LIKE 'cli:extract-conversation-facts%' LIMIT 1`, [sourceId, slug])).length
      ? await pageFenceRowKeys(engine, prior.page, slug, sourceId) : null;
  const { timeline, exactIncoming, pinned } = classifyTimeline(storedRows, page, prior_?.rows ?? null, slug, writer, policy);
  const deletionRows = pinned.filter(row => row.action === 'delete')
    .map(({ id, date, source, summary, detail, state }) => ({ id, date, source, summary, detail, removed: state === 'removed' || state === 'removed_marked' }));
  const refreshRows = pinned.filter(row => row.action === 'refresh_detail')
    .map(row => ({ id: row.id, detail: row.detail, next: exactIncoming.get(exactTimelineKey(row)) }))
    .filter(row => row.next !== row.detail);
  // GBRA-161 PR 2: a write that dropped the takes fence was held and left its rows stored; once a fence parses again it
  // is compared with those rows, not with the fenceless body before it.
  const heldTakeFence = !!prior && projectFences && fenceReads.takes.state === 'parsed' && (await engine.executeRaw(
    `SELECT 1 FROM derived_generations WHERE source_id=$1 AND kind=$2 AND owner_key=$3 AND state='held' AND held_reason='fence_absent'`,
    [sourceId, TAKE_FENCE_KIND, fenceOwnerKey(slug)])).length > 0;
  const priorTakes = heldTakeFence
    ? new Set([...(prior_?.takeRows ?? []), ...(await engine.executeRaw<{ row_num: number }>('SELECT row_num FROM takes WHERE page_id=$1', [prior!.page.id])).map(r => Number(r.row_num))])
    : prior_?.takeRows ?? new Set<number>();
  // TE1 (#6188): when the prior takes fence does not parse, its stored rows count as prior canonical rows, so a
  // row Tier 1 normalized (same number, holder `system` -> `brain`) updates its stored take instead of colliding.
  const priorCanonical = prior && prior_?.takesWarn
    ? new Set([...priorTakes, ...(await engine.executeRaw<{ row_num: number }>('SELECT row_num FROM takes WHERE page_id=$1', [prior.page.id])).map(r => Number(r.row_num))])
    : priorTakes;
  const newTakes = JSON.stringify(takes.filter(t => !priorCanonical.has(t.rowNum)).map(t => ({ row_num: t.rowNum, claim: t.claim, kind: t.kind, holder: t.holder })));
  const takeRowsGone = [...priorTakes].filter(n => !takes.some(t => t.rowNum === n));
  const collisions = async (db: BrainEngine, pageId: number) => (await db.executeRaw<{ row_num: number }>(TAKE_COLLISIONS_SQL, [pageId, newTakes, FENCE_ROWS_MAX])).map(r => Number(r.row_num));
  const refuseCollisions = async (db: BrainEngine, pageId: number) => {
    const rows = await collisions(db, pageId);
    if (rows.length) throw takeCollision(rows, sections.takes, slug, sourceId);
  };
  if (prior && projectFences) await refuseCollisions(engine, prior.page.id);
  // #5984: `pageId` is the caller's own read of the page in this transaction. The
  // statements are issued as pipelines; an engine call that sends more than one
  // statement (insertFacts, addTakesBatch) ends one, so order is kept.
  return async (tx, pageId) => {
    const id = pageId ?? (await tx.readPageSnapshot(slug, { sourceId }))?.page.id;
    if (id == null) return { timelineRowsRemoved: null };
    // #5969: only rows this statement actually deleted count; a row changed since preparation is left alone and uncounted.
    const removedDates: string[] = [];
    // W1 #6783 §12 (a): a timeline row or take more trusted than the writer is never deleted, replaced or deactivated by
    // it. It stays as it is and a `forget` proposal asks the owner (trust/fence-guard.ts), decided before any statement runs.
    const heldTimeline = await heldRows(tx, 'timeline_entries', [...deletionRows, ...refreshRows].map(row => Number(row.id)));
    const deletions = JSON.stringify(deletionRows.filter(row => !heldTimeline.has(Number(row.id))));
    const refreshes = JSON.stringify(refreshRows.filter(row => !heldTimeline.has(Number(row.id))));
    const heldTakes = projectFences ? await guardFenceTakes(tx, id, takes, takeRowsGone) : [];
    const heldRowNums = new Set(heldTakes.map(row => row.row_num));
    const projectedTakes = heldRowNums.size ? takes.filter(take => !heldRowNums.has(take.rowNum)) : takes;
    const contestHeld = async () => [
      ...await contestHeldRows(tx, { sourceId, slug, table: 'timeline_entries', proposer: 'fence_projection',
        rows: [...heldTimeline].map(([rowId, tier]) => ({ id: rowId, tier, change: deletionRows.some(row => Number(row.id) === rowId) ? 'removed' as const : 'detail_changed' as const })) }),
      ...await contestHeldRows(tx, { sourceId, slug, table: 'takes', proposer: 'fence_projection', rows: heldTakes }),
    ];
    if (quoted && projectFences) await refuseQuotedFenceLoss(tx, id, quoted, takeRowsGone, factRows, slug, sourceId);
    // Fact IDs in permanent receipts remain meaningful when a canonical row is
    // removed/replaced. Expire and detach its row position instead of deleting it.
    // Conversation-extractor rows share the page coordinate without a fence
    // (#1928, as in the extract_facts reconcile); their replay owns them. A
    // fence row that takes one of their row numbers wins that position.
    // #8819: only the rows the fact generation found a fence held (`retireIds`) expire. #8194: never a row `gbrain repair
    // open-facts` opens in place (context.ts openedFactRowSql); fence-guard moves it to world with its id.
    const incoming = JSON.stringify(factRows.map(f => ({ row_num: f.row_num, fact: f.fact, visibility: f.visibility })));
    const expireFacts = (ids: number[]) => () => tx.executeRaw(`UPDATE facts f SET expired_at=COALESCE(expired_at,now()),row_num=NULL
      WHERE source_id=$1 AND source_markdown_slug=$2 AND row_num IS NOT NULL AND f.id=ANY($4::bigint[])
      AND (COALESCE(f.source,'') NOT LIKE 'cli:extract-conversation-facts%'
        OR EXISTS (SELECT 1 FROM jsonb_to_recordset($3::text::jsonb) AS c(row_num integer) WHERE c.row_num=f.row_num))
      AND NOT EXISTS (SELECT 1 FROM jsonb_to_recordset($3::text::jsonb) AS n(row_num integer,fact text,visibility text)
        WHERE n.row_num=f.row_num AND n.fact=f.fact AND n.visibility=f.visibility)
      AND NOT ${openedFactRowSql('f', '$3')}`, [sourceId, slug, incoming, ids]);
    // #6574: every fence row's fields refresh in one statement, not one round trip per row.
    const fieldRows = JSON.stringify([...new Map(factRows.map(fact => [fact.row_num, fact])).values()].map(fact => ({ row_num: fact.row_num, kind: fact.kind, notability: fact.notability,
      context: fact.context, valid_from: fact.valid_from?.toISOString() ?? null, valid_until: fact.valid_until?.toISOString() ?? null,
      expired_at: fact.expired_at?.toISOString() ?? null, source: fact.source, confidence: fact.confidence, claim_metric: fact.claim_metric ?? null,
      claim_value: fact.claim_value ?? null, claim_unit: fact.claim_unit ?? null, claim_period: fact.claim_period ?? null, attributed_to: fact.attributed_to ?? null })));
    const factFields = () => factRows.length ? tx.executeRaw(`UPDATE facts f SET kind=r.kind,notability=r.notability,context=r.context,
        valid_from=COALESCE(r.valid_from::timestamptz,f.valid_from),valid_until=r.valid_until::timestamptz,expired_at=r.expired_at::timestamptz,
        source=r.source,confidence=r.confidence,claim_metric=r.claim_metric,claim_value=r.claim_value,claim_unit=r.claim_unit,
        claim_period=r.claim_period,attributed_to=r.attributed_to
      FROM jsonb_to_recordset($3::text::jsonb) AS r(row_num integer,kind text,notability text,context text,valid_from text,valid_until text,
        expired_at text,source text,confidence real,claim_metric text,claim_value double precision,claim_unit text,claim_period text,attributed_to text)
      WHERE f.source_id=$1 AND f.source_markdown_slug=$2 AND f.row_num=r.row_num
        AND (f.kind,f.notability,f.context,f.valid_from,f.valid_until,f.expired_at,f.source,f.confidence,f.claim_metric,f.claim_value,f.claim_unit,
          f.claim_period,f.attributed_to) IS DISTINCT FROM (r.kind,r.notability,r.context,COALESCE(r.valid_from::timestamptz,f.valid_from),
          r.valid_until::timestamptz,r.expired_at::timestamptz,r.source,r.confidence,r.claim_metric,r.claim_value,r.claim_unit,r.claim_period,r.attributed_to)`,
      [sourceId, slug, fieldRows]) : Promise.resolve([]);
    const checkTakes = () => refuseCollisions(tx, id);
    const dropTakes = (held: boolean) => () => held ? Promise.resolve([])
      : tx.executeRaw('DELETE FROM takes WHERE page_id=$1 AND row_num=ANY($2::integer[])', [id, takeRowsGone.filter(n => !heldRowNums.has(n))]);
    // A fence row's resolution cells are written as Markdown says, so a revert to a version that carries them restores
    // them. A row without resolution cells keeps the stored resolution (a database-only grade, e.g. unmanaged
    // grade_takes) unless its claim changed in place, which drops the old claim's grade. A held take keeps its grade:
    // the guard decided before any of these statements run (W1 #6783 §6.2).
    const fenceResolved = projectedTakes.filter(take => take.resolvedAt !== undefined || take.resolvedQuality !== undefined);
    const claims = JSON.stringify(projectedTakes.map(take => ({ row_num: take.rowNum, claim: take.claim })));
    const clearChangedResolutions = () => projectedTakes.length ? tx.executeRaw(`UPDATE takes k SET resolved_at=NULL,resolved_quality=NULL,
        resolved_outcome=NULL,resolved_source=NULL,resolved_value=NULL,resolved_unit=NULL,resolved_by=NULL
      FROM jsonb_to_recordset($2::text::jsonb) AS n(row_num integer,claim text)
      WHERE k.page_id=$1 AND k.row_num=n.row_num AND k.resolved_at IS NOT NULL AND k.claim IS DISTINCT FROM n.claim`, [id, claims]) : Promise.resolve([]);
    const resolutions = JSON.stringify(fenceResolved.map(take => ({ row_num: take.rowNum, resolved_at: take.resolvedAt ?? null,
      resolved_quality: take.resolvedQuality ?? null,
      resolved_outcome: take.resolvedQuality === 'correct' ? true : take.resolvedQuality === 'incorrect' ? false : null,
      resolved_source: take.resolvedEvidence ?? null, resolved_value: take.resolvedValue ?? null, resolved_unit: take.resolvedUnit ?? null,
      resolved_by: take.resolvedBy ?? null })));
    const resolveTakes = () => fenceResolved.length ? tx.executeRaw(`UPDATE takes k SET resolved_at=r.resolved_at::timestamptz,resolved_quality=r.resolved_quality,
        resolved_outcome=r.resolved_outcome,resolved_source=r.resolved_source,resolved_value=r.resolved_value,resolved_unit=r.resolved_unit,
        resolved_by=r.resolved_by
      FROM jsonb_to_recordset($2::text::jsonb) AS r(row_num integer,resolved_at text,resolved_quality text,resolved_outcome boolean,
        resolved_source text,resolved_value real,resolved_unit text,resolved_by text)
      WHERE k.page_id=$1 AND k.row_num=r.row_num AND (k.resolved_at,k.resolved_quality,k.resolved_outcome,k.resolved_source,k.resolved_value,k.resolved_unit,
        k.resolved_by) IS DISTINCT FROM (r.resolved_at::timestamptz,r.resolved_quality,r.resolved_outcome,r.resolved_source,r.resolved_value,r.resolved_unit,
        r.resolved_by)`, [id, resolutions]) : Promise.resolve([]);
    // Event-page references have a different canonical origin and remain intact;
    // new rows carry their Markdown detail on insert, pinned rows refresh only from their preimage.
    const timelineRows = [
      () => tx.executeRaw<{ date: string; removed: boolean }>(`DELETE FROM timeline_entries t USING jsonb_to_recordset($2::text::jsonb) AS d(id integer,date date,source text,summary text,detail text,removed boolean)
        WHERE t.page_id=$1 AND t.event_page_id IS NULL AND t.id=d.id AND t.date=d.date AND t.source=d.source
          AND t.summary=d.summary AND t.detail=d.detail RETURNING t.date::text AS date, d.removed`, [id, deletions])
        .then(rows => { for (const row of rows) if (row.removed) removedDates.push(row.date); return rows; }),
      ...timelineChunks(timeline).map(rows => () => insertCanonicalTimeline(tx, id, rows)),
      () => tx.executeRaw(`UPDATE timeline_entries t SET detail=r.next FROM jsonb_to_recordset($2::text::jsonb) AS r(id integer,detail text,next text)
        WHERE t.page_id=$1 AND t.event_page_id IS NULL AND t.id=r.id AND t.detail=r.detail`, [id, refreshes]),
    ];
    const started = Date.now();
    const steps: Array<[string, number]> = [];
    const step = async <T>(name: string, run: () => Promise<T>): Promise<T> => {
      const at = Date.now();
      try { return await run(); } finally { steps.push([name, Date.now() - at]); }
    };
    try {
      if (!projectFences) {
        await step('timeline', () => pipelined(tx, timelineRows));
        const held = await step('held', contestHeld);
        return { timelineRowsRemoved: removedSummary(removedDates), ...(held.length ? { contested: held } : {}) };
      }
      // GBRA-161 PR 2: the fact and take generations decide first. A held one (its fence absent from this body) retires and
      // overwrites nothing, so its rows stay as they are; every row a published one retires is recorded with its before-state.
      const factGeneration = await step('fact_generation', () => projectFactFenceGeneration(tx, { sourceId, slug, read: fenceReads.facts, incoming, rows: factRows,
        writer: 'projection', prior: priorFenceKeys }));
      const factsHeld = factGeneration.state === 'held';
      // #8819: a row the fence never held stays active; a fence row claiming its number takes it, and the row moves past the page's rows.
      const kept = factsHeld ? [] : factGeneration.kept;
      if (kept.length) await step('fact_kept', () => moveKeptRows(tx, sourceId, slug, kept, factRows.map(f => f.row_num)));
      const takesHeld = (await step('take_generation', () => projectTakeFenceGeneration(tx, { sourceId, slug, pageId: id, read: fenceReads.takes,
        takes: projectedTakes, gone: takeRowsGone.filter(n => !heldRowNums.has(n)), writer: 'projection' }))).state === 'held';
      // #5575: rows more trusted than the writer are never expired by it, and new rows pass the write gate (trust/fence-guard.ts).
      // The timeline rows are independent of the fact and take rows, so they ride in the first pipeline. The expire uses the
      // generation's retire predicate under the page lock, so it runs only when that generation found rows to retire.
      const guard = factsHeld ? { rows: [] as typeof factRows, finish: async () => [] as string[] } : await step('timeline', () => guardFenceRows(tx, { sourceId, slug, incoming, rows: factRows, cfg: loadWriteGateConfig, kept },
        () => pipelined(tx, factGeneration.retiring ? [expireFacts(factGeneration.retireIds), ...timelineRows] : timelineRows)));
      if (factsHeld) await step('timeline', () => pipelined(tx, timelineRows));
      if (guard.rows.length) await step('facts', () => tx.insertFacts(guard.rows, { source_id: sourceId })); // gbrain-allow-direct-insert: canonical fence projection shares the journal publication transaction
      const contested = await step('fact_trust', () => guard.finish(tx));
      // W1 #6783 §12 (b): a fence row's field rewrite can lower a more trusted fact (the tier trigger); what was derived from it follows.
      const above = factRows.length ? await factsAboveWriter(tx, sourceId, slug) : [];
      await step('fact_fields', () => pipelined(tx, [factFields, checkTakes, dropTakes(takesHeld), clearChangedResolutions]));
      if (above.length) await propagateFactLowering(tx, above);
      if (projectedTakes.length) {
        await step('takes', async () => {
          await tx.addTakesBatch(projectedTakes.map(t => takesPreparation.toCanonicalBatchInput(id, t)));
          await resolveTakes();
        });
      }
      if (fenceLinks) await step('links_fence', () => projectLinksFence(tx, { id, source_id: sourceId }, fenceLinks));
      const all = [...contested, ...await step('held', contestHeld)];
      return { timelineRowsRemoved: removedSummary(removedDates), ...(all.length ? { contested: all } : {}) };
    } finally {
      // #6574: a slow projection names its slow step and its row counts, so a round-trip-bound publish is visible in one line.
      const elapsed = Date.now() - started;
      if (elapsed >= SLOW_PROJECTION_MS) {
        process.stderr.write(`[persistence] phase=publication reason=slow_projection source=${sourceId} slug=${slug} elapsed_ms=${elapsed}`
          + ` timeline_rows=${timeline.size} fact_rows=${factRows.length} take_rows=${takes.length}`
          + ` steps=${steps.map(([name, ms]) => `${name}:${ms}`).join(',')}\n`);
      }
    }
  };
}

export interface CanonicalProjectionResult { timelineRowsRemoved: TimelineRowsRemoved | null; /** #5575 A5: trust proposals the guarded fence re-projection filed. */ contested?: string[] }