Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
31.0 KB
Raw Blame
/**
 * Timeline extraction from rendered markdown — CORE module.
 *
 * Lives in core (not commands/extract.ts, its historical home) because core
 * consumers (timeline-write-through.ts re-derives the canonical DB tuple from
 * the spliced bullet; the cycle synthesize path) must not import the command
 * module: commands/extract.ts transitively loads the write-through/ops layer,
 * and a core->commands import closes a module cycle that leaves the command
 * module partially evaluated under dynamic import ("Export named ... not
 * found" at runtime, invisible to tsc). commands/extract.ts re-exports these
 * names, so its existing importers are unaffected.
 */

import { parseInlineCitationTimelineEntries, findTimelineSourceDelimiter, parseTimelineEntries } from './link-extraction.ts';
import { isDatedTimelineLine, isValidDate, supersededInlineCitationEntries } from './timeline-citations.ts';
import { createHash } from 'node:crypto';
import type { BrainEngine } from './engine.ts';
import { firstMaterializedMarkerIndex } from './timeline-marker.ts';
import { BodyReadStopped, bodyBlocks, detachedValue, retainedGenerationBlocks } from './page-body/segment-blocks.ts';

export interface ExtractedTimelineEntry {
  slug: string;
  date: string;
  source: string;
  summary: string;
  detail?: string;
}

// #3957: the link-aware `Source — Summary` delimiter finder moved to
// link-extraction.ts (findTimelineSourceDelimiter) so the DB-side parser
// (parseTimelineEntries) applies the IDENTICAL split — FS- and DB-extracted
// rows must share one (source, summary) shape or the timeline dedup index
// duplicates every bullet extracted through both paths.
const findDelimiterOutsideLinks = findTimelineSourceDelimiter;

// A Format 1 bullet line; this extractor's inline-citation pass (Format 3) skips it.
const bulletLinePattern = /^-\s+\*\*\d{4}-\d{2}-\d{2}\*\*\s*\|/;

/** Extract timeline entries from markdown content */
export function extractTimelineFromContent(content: string, slug: string): ExtractedTimelineEntry[] {
  const entries: ExtractedTimelineEntry[] = [];

  // Format 1: Bullet — - **YYYY-MM-DD** | Source — Summary
  // The delimiter search is link-aware (see findDelimiterOutsideLinks): a
  // no-delimiter bullet is kept whole as the summary rather than fragmented.
  const bulletPattern = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|\s*(.+)$/gm;
  let match;
  while ((match = bulletPattern.exec(content)) !== null) {
    if (!isValidDate(match[1])) continue;
    const rest = match[2].trim();
    // #4277: dated auto-generated backlink receipts
    // (`- **date** | Referenced in [X](y.md)`) are graph-maintenance noise —
    // the date is the backlink write's, not an entity event's. Skip them.
    // Pre-split guard (rest must START with the marker) mirrors
    // parseTimelineEntries in link-extraction.ts so FS- and DB-side
    // extraction stay in lockstep, and leaves write-through rendered
    // `source — summary` bullets that merely mention the phrase intact.
    if (/^Referenced in\s+\[/i.test(rest)) continue;
    const at = findDelimiterOutsideLinks(rest);
    if (at >= 0) {
      entries.push({ slug, date: match[1], source: rest.slice(0, at).trim(), summary: rest.slice(at + 1).trim() });
    } else {
      entries.push({ slug, date: match[1], source: 'markdown', summary: rest });
    }
  }

  // Format 2: Header — ### YYYY-MM-DD — Title
  const headerPattern = /^###\s+(\d{4}-\d{2}-\d{2})\s*[—–-]\s*(.+)$/gm;
  while ((match = headerPattern.exec(content)) !== null) {
    if (!isValidDate(match[1])) continue;
    const afterIdx = match.index + match[0].length;
    const nextHeader = content.indexOf('\n### ', afterIdx);
    const nextSection = content.indexOf('\n## ', afterIdx);
    const endIdx = Math.min(
      nextHeader >= 0 ? nextHeader : content.length,
      nextSection >= 0 ? nextSection : content.length,
    );
    // #5567: a materialized bullet appended after this header is its own entry.
    const section = content.slice(afterIdx, endIdx);
    const marker = firstMaterializedMarkerIndex(section);
    const detail = (marker >= 0 ? section.slice(0, marker) : section).trim();
    entries.push({ slug, date: match[1], source: 'markdown', summary: match[2].trim(), detail: detail || undefined });
  }

  // Format 3: Inline citation — [Source: <source>, YYYY-MM-DD]
  //
  // This is the citation convention gbrain's own quality rules require on
  // every brain write (skills/conventions/quality.md), so dated evidence is
  // pervasive in curated pages — but until now the extractor could not see
  // it, and a page whose dates all live in citations scored zero timeline
  // coverage. The entry's summary is the sentence the citation annotates
  // (the surrounding line with citation markers stripped).
  //
  // Lines already captured by Format 1 are skipped: a timeline bullet often
  // carries its own [Source: ...] citation, and re-extracting it would file
  // a duplicate entry under a different (source, summary) shape that the
  // DB-level uniqueness cannot collapse.
  for (const entry of parseInlineCitationTimelineEntries(content, {
    skipLine: (line) => bulletLinePattern.test(line),
  })) {
    entries.push({ slug, date: entry.date, source: entry.source, summary: entry.summary });
  }

  return entries;
}

type TimelineTuple = Pick<ExtractedTimelineEntry, 'date' | 'source' | 'summary'>;
const tupleKey = (e: TimelineTuple) => JSON.stringify([e.date, e.source, e.summary]);

const LEGACY_DASHES = '—–-';

/**
 * #7949: how the file walk read a `- **YYYY-MM-DD** | text` bullet before #3280: cut at its first dash of any kind,
 * spaced or not (`foo-bar.de` -> source `foo`, summary `bar.de`), as its lazy `(.+?)\s*[—–-]\s*(.+)$` did. Scanned
 * by hand so a long line costs linear time.
 */
function legacySplitBullets(content: string): TimelineTuple[] {
  const out: TimelineTuple[] = [];
  for (const line of content.split(/\r?\n/)) {
    const m = /^-\s+\*\*(\d{4}-\d{2}-\d{2})\*\*\s*\|(.*)$/.exec(line);
    if (!m) continue;
    const rest = m[2]!;
    const start = rest.search(/\S/);
    if (start < 0) continue;
    let at = -1;
    for (let i = start + 1; i < rest.length - 1; i++) if (LEGACY_DASHES.includes(rest[i]!)) { at = i; break; }
    if (at < 0 && start > 0 && start < rest.length - 1 && LEGACY_DASHES.includes(rest[start]!)) at = start;
    if (at >= 0) out.push({ date: m[1]!, source: rest.slice(0, at).trim(), summary: rest.slice(at + 1).trim() });
  }
  return out;
}

/**
 * The tuples older readings of this text filed and the current one may no longer produce: the inline-citation pass of
 * either parser before one citation could name several dated sources (#6226, supersededInlineCitationEntries) and the
 * file walk's bullet split before #3280 (#7949, legacySplitBullets). Callers compare only rows the text does not keep.
 */
export function supersededTimelineReadings(content: string): TimelineTuple[] {
  return [
    ...supersededInlineCitationEntries(content, { skipLine: (line) => bulletLinePattern.test(line) }),
    ...supersededInlineCitationEntries(content, { skipLine: isDatedTimelineLine }),
    ...legacySplitBullets(content),
  ];
}

/**
 * The detail an extractor itself writes for a citation row: none (file
 * extraction) or `Source: <source>` (database extraction). Any other detail
 * was added by someone, so a row carrying it is never retired as an older
 * reading (#6226, taste T3): timelineKey ignores detail.
 */
export function hasExtractorDetail(row: { source: string; detail?: string | null }): boolean {
  const detail = (row.detail ?? '').replace(/\s+/g, ' ').trim();
  return detail === '' || detail === `Source: ${row.source}`.replace(/\s+/g, ' ').trim();
}

/**
 * Every (date, source, summary) tuple a page text yields under either timeline
 * parser: the file-walk parser above and the DB-side parseTimelineEntries.
 * Reconciliation keeps and retracts against this union, so the file and DB
 * extraction paths never undo each other's rows.
 */
function markdownTimelineKeys(text: string, slug: string): Set<string> {
  const keys = new Set(extractTimelineFromContent(text, slug).map(tupleKey));
  for (const entry of parseTimelineEntries(text)) keys.add(tupleKey({ ...entry, source: entry.source ?? '' }));
  return keys;
}

/** Membership of timeline keys: a Set of them, or a TimelineKeySet. */
export interface TimelineKeyLookup { has(key: string): boolean }

const LOW = 2 ** 32;
const AMBIGUOUS = 1;
const MARKED = 2;

/**
 * A set of keys held as two hashes of each (53 and 32 bits of its SHA-256), not the key strings: a few dozen bytes a
 * key instead of a few hundred, for sets with one key per stored timeline row of a large page. A key never added
 * answers true only when both of its hashes collide with an added key's (about 2^-85 a pair); two added keys whose
 * first hashes collide share one entry, which then answers for either. `mark` flags an added key (a stored row the
 * body produces), so one entry per row answers both questions.
 */
export class TimelineKeySet implements TimelineKeyLookup {
  /** Open addressing over two plain number arrays (a Map costs about 80 bytes an entry, these about 25): slot i holds the first hash + 1 (0: empty) and the second hash plus its flags times LOW. */
  private firsts: number[] = new Array<number>(1024).fill(0);
  private values: number[] = new Array<number>(1024).fill(0);
  private count = 0;
  add(key: string): void {
    const [a, b] = keyDigest(key);
    const i = this.slot(a);
    if (this.firsts[i] === 0) {
      this.firsts[i] = a + 1;
      this.values[i] = b;
      if (++this.count * 4 > this.firsts.length * 3) this.grow();
      return;
    }
    const at = this.values[i]!;
    if (at % LOW !== b) this.values[i] = at - (at % LOW) + b + (Math.floor(at / LOW) & AMBIGUOUS ? 0 : AMBIGUOUS * LOW);
  }
  has(key: string): boolean {
    return this.entry(key) !== -1;
  }
  /** Flag an added key; false when it was not added. */
  mark(key: string): boolean {
    const i = this.entry(key);
    if (i === -1) return false;
    if (!(Math.floor(this.values[i]! / LOW) & MARKED)) this.values[i] = this.values[i]! + MARKED * LOW;
    return true;
  }
  marked(key: string): boolean {
    const i = this.entry(key);
    return i !== -1 && (Math.floor(this.values[i]! / LOW) & MARKED) !== 0;
  }
  get size(): number { return this.count; }
  /** The slot holding first hash `a`, or the empty slot where it goes. */
  private slot(a: number): number {
    const mask = this.firsts.length - 1;
    let i = a % this.firsts.length;
    while (this.firsts[i] !== 0 && this.firsts[i] !== a + 1) i = (i + 1) & mask;
    return i;
  }
  private entry(key: string): number {
    const [a, b] = keyDigest(key);
    const i = this.slot(a);
    if (this.firsts[i] === 0) return -1;
    const at = this.values[i]!;
    return (Math.floor(at / LOW) & AMBIGUOUS) !== 0 || at % LOW === b ? i : -1;
  }
  private grow(): void {
    const { firsts, values } = this;
    this.firsts = new Array<number>(firsts.length * 2).fill(0);
    this.values = new Array<number>(firsts.length * 2).fill(0);
    for (let j = 0; j < firsts.length; j++) {
      if (firsts[j] === 0) continue;
      const i = this.slot(firsts[j]! - 1);
      this.firsts[i] = firsts[j]!;
      this.values[i] = values[j]!;
    }
  }
}

function keyDigest(key: string): [number, number] {
  const hash = createHash('sha256').update(key).digest();
  return [hash.readUIntBE(0, 6) * 32 + (hash[6]! >> 3), hash.readUInt32BE(8)];
}

/**
 * What a page text says about its timeline rows: the tuples either parser yields, and the older readings it
 * supersedes. Read from a segmented body (segmentedTimelineText) with `within`, both hold only those keys, and a
 * stored row whose key is not in `within` (one written after the read) is never retracted against them. `runs` holds
 * the keys of the body's runs whose blocks were their stored text, which a retained generation need not read again.
 */
export interface TimelineTextKeys { kept: TimelineKeyLookup; superseded: TimelineKeyLookup; within?: TimelineKeyLookup; runs?: ReadonlySet<string> }

/** The key retraction compares a stored row by (`within` of segmentedTimelineText). */
export const storedTimelineKey = (row: TimelineTuple): string => tupleKey(row);

export interface SegmentedTimelineOptions {
  /**
   * The keys (storedTimelineKey) of the page's stored rows: kept is then the keys of `within` the body produces (marked
   * there, so the result holds no set of its own) and superseded only keys of `within`.
   */
  within?: TimelineKeySet;
  /** Collect the entries parseTimelineEntries reads (default true); `parsed` counts them either way. */
  entries?: boolean;
  /** Asked before each segment read (bodyBlocks): true ends the read with BodyReadStopped. */
  stop?: () => boolean;
  /** Each block's text, for a caller deriving more from the same walk. */
  onBlock?: (text: string) => void;
}

/**
 * TimelineTextKeys of a segmented page's `${compiled_truth}\n${timeline}`, block by block (page-body/segment-blocks.ts),
 * with the entries parseTimelineEntries reads there, in body order, and their count. Blocks start outside timeline
 * detail runs, so each entry is read whole. With `within` and `entries: false` the result holds at most one key per
 * stored row, not one per entry.
 */
export async function segmentedTimelineText(engine: Pick<BrainEngine, 'executeRaw' | 'readSegmentTexts'>, page: { id: number; slug: string; source_id?: string },
  opts: SegmentedTimelineOptions = {}): Promise<TimelineTextKeys & { entries: ReturnType<typeof parseTimelineEntries>; parsed: number; runs: Set<string> }> {
  const kept = new Set<string>();
  const superseded = new Set<string>();
  const runs = new Set<string>();
  const entries: ReturnType<typeof parseTimelineEntries> = [];
  const { within } = opts;
  let parsed = 0;
  for await (const block of bodyBlocks(engine as BrainEngine, page, { runKeys: runs, ...(opts.stop ? { stop: opts.stop } : {}) })) {
    for (const key of markdownTimelineKeys(block.text, page.slug)) if (!within) kept.add(key); else within.mark(key);
    for (const entry of supersededTimelineReadings(block.text)) {
      const key = tupleKey(entry);
      if (!within || within.has(key)) superseded.add(key);
    }
    const read = parseTimelineEntries(block.text);
    parsed += read.length;
    if (opts.entries !== false) entries.push(...detachedValue(read));
    opts.onBlock?.(block.text);
  }
  return { kept: within ? { has: key => within.marked(key) } : kept, superseded, ...(within ? { within } : {}), runs, entries, parsed };
}

/** A page's stored non-event timeline row as reconciliation reads it (`date` is `YYYY-MM-DD`). */
export type StoredTimelineTuple = TimelineTuple & { id: number; detail: string | null };

function storedTimelineRows(engine: Pick<BrainEngine, 'executeRaw'>, sourceId: string, slug: string): Promise<StoredTimelineTuple[]> {
  return engine.executeRaw<StoredTimelineTuple>(
    `SELECT t.id, to_char(t.date, 'YYYY-MM-DD') AS date, t.source, t.summary, t.detail FROM timeline_entries t
      JOIN pages p ON p.id = t.page_id
      WHERE p.source_id = $1 AND p.slug = $2 AND t.event_page_id IS NULL`, [sourceId, slug]);
}

const STORED_ROWS_PAGE = 5_000;

/**
 * Each stored non-event timeline row of a page (`date_text` is `date::text`, unrecordedCanonicalTimeline's form),
 * STORED_ROWS_PAGE rows at a time by id, so a page with many rows is never held as one array.
 */
export async function eachStoredTimelineRow(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number,
  fn: (row: StoredTimelineTuple & { date_text: string }) => void): Promise<void> {
  for (let after = 0; ;) {
    const rows = await engine.executeRaw<StoredTimelineTuple & { date_text: string }>(
      `SELECT id, to_char(date, 'YYYY-MM-DD') AS date, date::text AS date_text, source, summary, detail FROM timeline_entries
        WHERE page_id = $1 AND event_page_id IS NULL AND id > $2 ORDER BY id LIMIT $3`, [pageId, after, STORED_ROWS_PAGE]);
    for (const row of rows) fn(row);
    if (rows.length < STORED_ROWS_PAGE) return;
    after = Number(rows[rows.length - 1]!.id);
  }
}

/**
 * The stored rows of a page a segmented read of it can retract: those whose key `keys` (segmentedTimelineText with
 * `within`) neither keeps nor leaves out of `within`, streamed (eachStoredTimelineRow).
 */
export async function unkeptTimelineRows(engine: Pick<BrainEngine, 'executeRaw'>, pageId: number, keys: TimelineTextKeys): Promise<StoredTimelineTuple[]> {
  const out: StoredTimelineTuple[] = [];
  await eachStoredTimelineRow(engine, pageId, ({ date_text: _text, ...row }) => {
    const key = tupleKey(row);
    if (!keys.kept.has(key) && (!keys.within || keys.within.has(key))) out.push(row);
  });
  return out;
}

/** A stored version a timeline key was read from, and what must still hold for it to count in a later transaction. */
type VersionWitness = { id: number; storage: 'inline'; digests: [string, string] }
  | { id: number; storage: 'segmented'; generation: number; hash: string | null };

/**
 * What a page's stored versions say about the timeline keys retraction asked about (#9362): for each key, the versions
 * whose text produces it (`produced`) or files it under an older reading (`superseded`), newest first. `done` once
 * filled: a caller fills it with a dry run before its publication transaction, and the retraction inside it then reads no
 * version body, only checks that a version it names is still the page's with the same body.
 */
export interface VersionTimelineScan {
  done: boolean;
  produced: Map<string, VersionWitness[]>;
  superseded: Map<string, VersionWitness[]>;
}

export const versionTimelineScan = (): VersionTimelineScan => ({ done: false, produced: new Map(), superseded: new Map() });

/**
 * Reconcile a page's timeline rows with its current text: retract every row
 * an earlier version of the page produced that the current text no longer
 * does (a corrected or deleted dated bullet, however many edits ago). Rows no
 * version of the page ever produced (enrichment, meeting fan-out, inferred
 * anchors) and event-page projections are never touched. The page_versions
 * scan only runs when the page holds a row the current text does not produce,
 * reads one version body per statement and keeps only the keys of those rows.
 * `versions` carries that scan from a dry run before a
 * publication transaction into it (VersionTimelineScan). A retained segmented generation is read
 * run by run (retainedGenerationBlocks), skipping each run whose key the live body
 * or an earlier generation already read; `stop` is asked before each of its segment reads.
 * Returns the orphaned rows; they are deleted unless `dryRun`. `storedRows`
 * is the caller's own read of those rows (a batched walk), used instead of reading them here.
 * `current` is the page text, or its keys when the caller read a segmented body (segmentedTimelineText).
 * `expectedRevision` is the page revision `current` was read at (GBRA-161 PR 5, T-2): the delete then runs with the page
 * row locked and only while the page is still at that revision; a page that moved deletes nothing and returns no rows.
 */
export async function retractRemovedTimelineEntries(
  engine: Pick<BrainEngine, 'executeRaw' | 'readManifest' | 'readSegmentTexts'>,
  slug: string,
  sourceId: string,
  current: string | TimelineTextKeys,
  opts: { dryRun?: boolean; storedRows?: ReadonlyArray<StoredTimelineTuple>; stop?: () => boolean; expectedRevision?: string; versions?: VersionTimelineScan } = {},
): Promise<Array<TimelineTuple & { id: number }>> {
  const kept = typeof current === 'string' ? markdownTimelineKeys(current, slug) : current.kept;
  const within = typeof current === 'string' ? undefined : current.within;
  const rows = opts.storedRows ?? await storedTimelineRows(engine, sourceId, slug);
  const extra = rows.filter(row => !kept.has(tupleKey(row)) && (!within || within.has(tupleKey(row))));
  // #9362: a filled `versions` (read before this transaction) is checked, not read again; a row it was not asked about stays.
  const replay = opts.versions?.done === true;
  if (!extra.length) {
    if (opts.versions) opts.versions.done = true;
    return [];
  }
  // The page's stored versions, one body per statement (never every body in one result), newest first; identical
  // bodies are parsed once and `stop` is asked before each body read.
  const readVersions = async (wanted: ReadonlySet<string>, seen: Set<string>): Promise<VersionTimelineScan> => {
    const read = versionTimelineScan();
    const parsed = new Map<string, { produced: string[]; superseded: string[] }>();
    const keysIn = async (texts: Iterable<string> | AsyncIterable<string>) => {
      const found = { produced: new Set<string>(), superseded: new Set<string>() };
      for await (const text of texts) {
        for (const key of markdownTimelineKeys(text, slug)) if (wanted.has(key)) found.produced.add(key);
        for (const entry of supersededTimelineReadings(text)) if (wanted.has(tupleKey(entry))) found.superseded.add(tupleKey(entry));
      }
      return { produced: [...found.produced], superseded: [...found.superseded] };
    };
    const versions = await engine.executeRaw<{ id: number; body_storage: string; page_id: number; generation: number | null; content_hash: string | null }>(
      `SELECT v.id, v.body_storage, v.page_id, v.body_generation AS generation, v.content_hash FROM page_versions v JOIN pages p ON p.id = v.page_id
        WHERE p.source_id = $1 AND p.slug = $2 ORDER BY v.id DESC`, [sourceId, slug]);
    for (const version of versions) {
      let witness: VersionWitness;
      let found: { produced: string[]; superseded: string[] };
      if (version.body_storage !== 'segmented') {
        if (opts.stop?.()) throw new BodyReadStopped(slug);
        const [body] = await engine.executeRaw<{ compiled_truth: string; timeline: string | null; head: string; tail: string }>(
          `SELECT compiled_truth, timeline, md5(compiled_truth) AS head, md5(coalesce(timeline, '')) AS tail FROM page_versions WHERE id = $1`, [version.id]);
        if (!body) continue;
        witness = { id: Number(version.id), storage: 'inline', digests: [body.head, body.tail] };
        const cached = `inline:${body.head}:${body.tail}`;
        found = parsed.get(cached) ?? await keysIn([`${body.compiled_truth}\n${body.timeline ?? ''}`]);
        parsed.set(cached, found);
      } else {
        witness = { id: Number(version.id), storage: 'segmented', generation: Number(version.generation), hash: version.content_hash };
        const cached = `segmented:${version.page_id}:${version.generation}`;
        found = parsed.get(cached) ?? await keysIn(retainedGenerationBlocks(engine, { id: Number(version.page_id), slug, sourceId }, Number(version.generation),
          { seen, ...(opts.stop ? { stop: opts.stop } : {}) }));
        parsed.set(cached, found);
      }
      for (const [into, keys] of [[read.produced, found.produced], [read.superseded, found.superseded]] as const) {
        for (const key of keys) {
          const witnesses = into.get(key);
          if (witnesses) witnesses.push(witness);
          else into.set(key, [witness]);
        }
      }
    }
    read.done = true;
    return read;
  };
  // Inside a later transaction: the witness version is still this page's, with the body it was read with (no body is returned).
  const stillHolds = async (witness: VersionWitness) => (witness.storage === 'inline'
    ? await engine.executeRaw(`SELECT 1 FROM page_versions v JOIN pages p ON p.id = v.page_id WHERE v.id = $1 AND p.source_id = $2 AND p.slug = $3
        AND v.body_storage = 'inline' AND md5(v.compiled_truth) = $4 AND md5(coalesce(v.timeline, '')) = $5`, [witness.id, sourceId, slug, ...witness.digests])
    : await engine.executeRaw(`SELECT 1 FROM page_versions v JOIN pages p ON p.id = v.page_id WHERE v.id = $1 AND p.source_id = $2 AND p.slug = $3
        AND v.body_storage = 'segmented' AND v.body_generation = $4 AND v.content_hash IS NOT DISTINCT FROM $5`, [witness.id, sourceId, slug, witness.generation, witness.hash])).length > 0;
  const scan = replay ? opts.versions! : await readVersions(new Set(extra.map(tupleKey)), new Set(typeof current === 'string' ? [] : current.runs ?? []));
  if (opts.versions && !replay) Object.assign(opts.versions, scan);
  const holds = new Map<number, boolean>();
  const confirmed = async (witnesses: VersionWitness[] | undefined) => {
    if (!witnesses?.length) return false;
    if (!replay) return true;
    for (const witness of witnesses) {
      if (!holds.has(witness.id)) holds.set(witness.id, await stillHolds(witness));
      if (holds.get(witness.id)) return true;
    }
    return false;
  };
  // #6226, #7949: rows an older reading filed for this text or a stored version.
  const supersededNow = typeof current === 'string' ? new Set(supersededTimelineReadings(current).map(tupleKey)) : current.superseded;
  const found: Array<TimelineTuple & { id: number }> = [];
  for (const row of extra) {
    const key = tupleKey(row);
    if (!await confirmed(scan.produced.get(key))
      && !(hasExtractorDetail(row) && (supersededNow.has(key) || await confirmed(scan.superseded.get(key))))) continue;
    const { detail: _detail, ...tuple } = row;
    found.push(tuple);
  }
  if (!found.length) return [];
  // W1 #6783 §12 (a): a row more trusted than the page's restatement, or held by an open contest, stays; the owner decides.
  const [{ contestHeldRows, heldRows }, { derivedWriteTrust }, { storedTrustTier }] = await Promise.all([
    import('./trust/fence-guard.ts'), import('./trust/taint.ts'), import('./trust/tier.ts')]);
  const [page] = await engine.executeRaw<{ id: number; trust_tier: string | null }>('SELECT id, trust_tier FROM pages WHERE source_id = $1 AND slug = $2', [sourceId, slug]);
  const writer = derivedWriteTrust({ channel: 'derive:timeline', inputs: page ? [{ table: 'pages', id: Number(page.id), tier: storedTrustTier(page.trust_tier) }] : [], projection: true }).tier;
  const held = await heldRows(engine as BrainEngine, 'timeline_entries', found.map(row => Number(row.id)), { writer, pending: true });
  const orphans = found.filter(row => !held.has(Number(row.id)));
  if (opts.dryRun) return orphans;
  await contestHeldRows(engine as BrainEngine, { sourceId, slug, table: 'timeline_entries', proposer: 'timeline_reconcile', writer,
    rows: [...held].map(([id, tier]) => ({ id, tier, change: 'removed' as const })) });
  if (!orphans.length) return orphans;
  const remove = (db: Pick<BrainEngine, 'executeRaw'>) => db.executeRaw(
    `DELETE FROM timeline_entries WHERE id IN (SELECT jsonb_array_elements_text($1::text::jsonb)::int)`,
    [JSON.stringify(orphans.map(row => row.id))]);
  if (opts.expectedRevision === undefined) {
    await remove(engine);
    return orphans;
  }
  const expected = opts.expectedRevision;
  return (engine as BrainEngine).transaction(async tx => {
    const [current] = await tx.executeRaw<{ revision: string }>(
      'SELECT knowledge_revision::text AS revision FROM pages WHERE source_id = $1 AND slug = $2 AND deleted_at IS NULL FOR SHARE', [sourceId, slug]);
    if (current?.revision !== expected) return [];
    await remove(tx);
    return orphans;
  });
}

/**
 * Retract a page's removed rows against its stored text: a segmented body is read block by block, keeping only the
 * keys of its stored rows (segmentedTimelineText `within`, a TimelineKeySet), and the rows are streamed, not held.
 */
async function retractStoredPage(engine: BrainEngine, page: { id: number; slug: string; source_id: string; compiled_truth: string; timeline: string | null; body_storage: string },
  opts: { dryRun?: boolean; expectedRevision?: string }) {
  if (page.body_storage !== 'segmented') return retractRemovedTimelineEntries(engine, page.slug, page.source_id, `${page.compiled_truth}\n${page.timeline ?? ''}`, opts);
  const within = new TimelineKeySet();
  await eachStoredTimelineRow(engine, page.id, row => within.add(tupleKey(row)));
  if (!within.size) return [];
  const keys = await segmentedTimelineText(engine, page, { within, entries: false });
  return retractRemovedTimelineEntries(engine, page.slug, page.source_id, keys, { ...opts, storedRows: await unkeptTimelineRows(engine, page.id, keys) });
}

export interface TimelineOrphanPruneResult {
  pagesScanned: number;
  orphans: number;
  removed: number;
  examples: Array<{ source_id: string; slug: string; date: string; summary: string }>;
}

/**
 * Managed brains guard `timeline_entries`: the retraction runs as a coordinated
 * write under the page lock, against the text read inside that transaction.
 */
async function retractCoordinated(engine: BrainEngine, pageId: number, slug: string, sourceId: string) {
  const { withCoordinatedWrite } = await import('./persistence/context.ts');
  const { maintenanceAttribution } = await import('./persistence/attribution.ts');
  const attribution = await maintenanceAttribution(engine);
  return engine.transaction(tx => withCoordinatedWrite(tx, [sourceId], async () => {
    await tx.lockPageKeys([{ sourceId, slug }]);
    const [page] = await tx.executeRaw<{ compiled_truth: string; timeline: string | null; body_storage: string }>(
      'SELECT compiled_truth, timeline, body_storage FROM pages WHERE id = $1 AND deleted_at IS NULL', [pageId]);
    if (!page) return [];
    return retractStoredPage(tx, { ...page, id: pageId, slug, source_id: sourceId }, {});
  }, attribution));
}

/**
 * One-time prune of timeline rows orphaned before extraction reconciled them
 * (#4649): runs the reconciliation above against each page's stored text for
 * every live page that holds timeline rows, optionally scoped to one source
 * and capped at `limit` pages (the doctor check samples; the CLI prunes all).
 */
export async function pruneTimelineOrphans(
  engine: BrainEngine,
  opts: { sourceId?: string; dryRun: boolean; limit?: number; coordinated?: boolean },
): Promise<TimelineOrphanPruneResult> {
  const result: TimelineOrphanPruneResult = { pagesScanned: 0, orphans: 0, removed: 0, examples: [] };
  let afterId = 0;
  for (;;) {
    const pageSize = Math.min(200, (opts.limit ?? Infinity) - result.pagesScanned);
    if (pageSize <= 0) break;
    const params: unknown[] = [afterId, pageSize];
    if (opts.sourceId) params.push(opts.sourceId);
    const pages = await engine.executeRaw<{ id: number; slug: string; source_id: string; compiled_truth: string; timeline: string | null; body_storage: string; revision: string }>(
      `SELECT p.id, p.slug, p.source_id, p.compiled_truth, p.timeline, p.body_storage, p.knowledge_revision::text AS revision FROM pages p
        WHERE p.id > $1 AND p.deleted_at IS NULL${opts.sourceId ? ' AND p.source_id = $3' : ''}
          AND EXISTS (SELECT 1 FROM timeline_entries t WHERE t.page_id = p.id AND t.event_page_id IS NULL)
        ORDER BY p.id LIMIT $2`, params);
    if (!pages.length) break;
    for (const page of pages) {
      const orphans = opts.coordinated && !opts.dryRun
        ? await retractCoordinated(engine, page.id, page.slug, page.source_id)
        : await retractStoredPage(engine, page, { dryRun: opts.dryRun, expectedRevision: page.revision });
      result.orphans += orphans.length;
      if (!opts.dryRun) result.removed += orphans.length;
      for (const row of orphans.slice(0, Math.max(0, 5 - result.examples.length))) {
        result.examples.push({ source_id: page.source_id, slug: page.slug, date: row.date, summary: row.summary });
      }
    }
    result.pagesScanned += pages.length;
    afterId = pages[pages.length - 1]!.id;
  }
  return result;
}