Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
23.7 KB
Raw Blame
/**
 * `gbrain extract timeline --source db` (#5904 probe, E28).
 *
 * `timeline_entries` is a guarded table: on a managed brain (every fresh
 * `gbrain init` brain is one) a raw batch insert is refused by
 * `managed_writer_guard`. Managed brains therefore publish each page that
 * needs a change as one database-only maintenance request
 * (`managed_maintenance_timeline_extract`, owner-checked by
 * `maintenancePreflight`): under the page lock, bound to the revision that
 * was read, the rows an earlier version produced and the current text no
 * longer does are retracted and the timeline tuples the canonical body
 * projects with no stored row are inserted. A page with nothing to change
 * admits no request. Unmanaged brains keep the batched raw insert.
 *
 * A refused write is reported as what it is: nothing was written and the
 * timeline rows import already stored are untouched. Refusals are counted
 * apart from written rows and make the command exit non-zero.
 */
import type { BrainEngine, TimelineBatchInput } from '../core/engine.ts';
import type { Page, PageType } from '../core/types.ts';
import { parseTimelineEntries, deriveTimelineAnchor } from '../core/link-extraction.ts';
import { eachStoredTimelineRow, retractRemovedTimelineEntries, segmentedTimelineText, storedTimelineKey, TimelineKeySet, unkeptTimelineRows, versionTimelineScan, type VersionTimelineScan } from '../core/timeline-extract.ts';
import { timelineKey } from '../core/timeline-marker.ts';
import { managedPersistenceEnabled } from '../core/persistence/ownership.ts';
import { maintenanceTransaction } from '../core/persistence/attribution.ts';
import { segmentedCanonicalCollector, unrecordedCanonicalTimeline } from '../core/persistence/canonical-projections.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
import { maintenancePreflight, submitDatabaseMaintenanceIntent, type MaintenanceAuthority } from '../core/persistence/prepared-maintenance.ts';
import { authorizeWrite } from '../core/persistence/authority.ts';
import { getWriteRequest } from '../core/persistence/journal.ts';
import { isTerminal, type WriteRequest } from '../core/persistence/model.ts';
import { digest } from '../core/persistence/digest.ts';
import type { PreparedMutation } from '../core/persistence/coordinator.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import { readFix } from '../core/ops/op-fix.ts';
import { createProgress } from '../core/progress.ts';
import { importAnalyzeEveryPages, maybeRefreshPlannerStats } from '../core/planner-stats.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { extractCompletionKey, filterRefsSince } from './extract.ts';
import { isQuarantined } from '../core/quarantine.ts';
import { pageSnapshotKey } from '../core/page-snapshot-batch.ts';
import type { PageSnapshot } from '../core/page-state/types.ts';

const BATCH_SIZE = 100;
export const TIMELINE_EXTRACT_INTENT = 'managed_maintenance_timeline_extract';
export const TIMELINE_REFUSAL_DOCS = 'docs/guides/write-refusals.md#extract-timeline-refused';

export interface TimelineDbResult {
  created: number;
  pages: number;
  /** Pages (managed) or rows (unmanaged batches) whose write was refused. */
  refused: number;
  /** Managed pages edited during the run; they keep their stored rows and are picked up by the next run. */
  skipped: number;
  /** Managed pages whose request was accepted but has not committed yet; rerunning resumes the same request. */
  pending: number;
  refusal_codes: string[];
}

export interface TimelineDbOptions {
  dryRun: boolean;
  jsonMode: boolean;
  quiet?: boolean;
  typeFilter?: PageType;
  since?: string;
  sourceIdFilter?: string;
  inferDates?: boolean;
  /** Restrict the walk to these slugs (requires sourceIdFilter). */
  slugs?: readonly string[];
  /** S4 (P2.18): receives `extractCompletionKey` for each page whose timeline write landed (or had nothing to write). */
  completed?: Set<string>;
}

/**
 * The refusal reason: the error's own code, unless it is the coordinator's
 * generic `storage_error`, whose message carries the database refusal
 * (`writer_coordinator_required: ...`) that names the real reason.
 */
export function refusalCode(error: unknown): string {
  const message = error instanceof Error ? error.message : String(error);
  const code = (error as { code?: unknown })?.code;
  const own = typeof code === 'string' && /^[a-z][a-z0-9_]+$/.test(code) ? code : null;
  if (own && own !== 'storage_error') return own;
  return /(?:^|[\s(])([a-z][a-z0-9]*_[a-z0-9_]+):/.exec(message)?.[1] ?? own ?? 'storage_error';
}

function reportRefusal(opts: TimelineDbOptions, code: string, where: string, message: string): void {
  if (opts.jsonMode) {
    process.stderr.write(JSON.stringify({ event: 'timeline_refused', code, where, error: message }) + '\n');
  } else {
    console.error(`  refused: ${code}; nothing written; existing timeline rows are untouched (${where})`);
  }
}

function anchorFor(page: Page, slug: string) {
  return deriveTimelineAnchor({ slug, title: page.title, effectiveDate: page.effective_date, effectiveDateSource: page.effective_date_source });
}

async function walkRefs(engine: BrainEngine, opts: TimelineDbOptions) {
  const all = await (await import('../core/guests/derived.ts')).withoutGuestSources(engine, 'extract', opts.sourceIdFilter
    ? (await engine.listAllPageRefs()).filter(r => r.source_id === opts.sourceIdFilter)
    : await engine.listAllPageRefs(), ref => ref.source_id);
  const wanted = opts.slugs ? new Set(opts.slugs) : null;
  return filterRefsSince(wanted ? all.filter(r => wanted.has(r.slug)) : all, opts.since);
}

/**
 * Walk the selected pages from the database and add their timeline rows.
 * Managed brains publish per page through the coordinator; unmanaged brains
 * batch raw inserts (`ON CONFLICT DO NOTHING`).
 */
export async function extractTimelineFromDB(engine: BrainEngine, opts: TimelineDbOptions): Promise<TimelineDbResult> {
  const refs = await walkRefs(engine, opts);
  const result: TimelineDbResult = { created: 0, pages: 0, refused: 0, skipped: 0, pending: 0, refusal_codes: [] };
  const authorities = new Map<string, Promise<MaintenanceAuthority>>();
  const authorityFor = (sourceId: string) => {
    if (!authorities.has(sourceId)) authorities.set(sourceId, maintenancePreflight(engine, sourceId).then(a => a!));
    return authorities.get(sourceId)!;
  };
  const codes = new Set<string>();
  const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
  progress.start('extract.timeline_db', refs.length);
  const managed = !opts.dryRun && await managedPersistenceEnabled(engine);
  const dryRunSeen = opts.dryRun ? new Set<string>() : null;
  const batch: TimelineBatchInput[] = [];
  // PGLite has no autovacuum: a walk that grows timeline_entries from empty
  // re-plans its per-page reads against stale statistics and slows with
  // every row, so it refreshes them on the bulk-import cadence.
  const analyzeEvery = opts.dryRun ? 0 : await importAnalyzeEveryPages(engine);
  let walked = 0;

  // Unmanaged pages whose rows sit in the unflushed batch, and pages a refused batch held: completion follows the flush.
  const unflushed = new Set<string>(), refusedPages = new Set<string>();
  async function flush() {
    if (batch.length === 0) return;
    const snapshot = batch.slice(), held = [...unflushed];
    batch.length = 0;
    unflushed.clear();
    try {
      result.created += await maintenanceTransaction(engine, tx => tx.addTimelineEntriesBatch(snapshot, { auditSite: 'extract.timeline_db' }));
      for (const key of held) if (!refusedPages.has(key)) opts.completed?.add(key);
    } catch (e) {
      for (const key of held) { refusedPages.add(key); opts.completed?.delete(key); }
      const code = refusalCode(e);
      codes.add(code);
      result.refused += snapshot.length;
      reportRefusal(opts, code, `batch of ${snapshot.length} rows`, e instanceof Error ? e.message : String(e));
    }
  }

  // Pages are read BATCH_SIZE at a time (one snapshot statement, and on a
  // managed brain one read of their stored rows); a managed page with nothing
  // to change then admits no request without another read.
  let readBatch: TimelineReadBatch = { start: 0, covered: 0, snapshots: new Map(), stored: new Map() };
  for (const [index, { slug, source_id }] of refs.entries()) {
    if (analyzeEvery > 0 && walked > 0 && walked % analyzeEvery === 0) {
      await flush();
      await maybeRefreshPlannerStats(engine, 'extract', { throttle: false }).catch(() => undefined);
    }
    walked++;
    if (index >= readBatch.start + readBatch.covered) readBatch = await readTimelineBatch(engine, refs, index, managed);
    const snapshot = readBatch.snapshots.get(pageSnapshotKey(source_id, slug)) ?? null;
    if (managed) {
      try {
        const outcome = await publishPageTimeline(engine, await authorityFor(source_id), slug, source_id, opts,
          { snapshot, stored: !snapshot ? [] : isSegmentedPage(snapshot.page) ? undefined : readBatch.stored.get(Number(snapshot.page.id)) ?? [] });
        if (outcome === 'skipped') result.skipped++;
        else if (outcome === 'pending') result.pending++;
        else if (outcome !== null) { result.created += outcome; result.pages++; opts.completed?.add(extractCompletionKey(source_id, slug, snapshot!.revision)); }
      } catch (e) {
        const code = refusalCode(e);
        codes.add(code);
        result.refused++;
        reportRefusal(opts, code, `${source_id}:${slug}`, e instanceof Error ? e.message : String(e));
      }
      progress.tick(1);
      continue;
    }

    const page = snapshot?.page ?? null;
    if (!page || isQuarantined(page.frontmatter)) continue;
    if (opts.typeFilter && page.type !== opts.typeFilter) continue;
    // A segmented body is read block by block (segmentedTimelineText), never as one string.
    const segmented = isSegmentedPage(page) ? await segmentedTimelineText(engine, page) : null;
    const fullContent = segmented ?? page.compiled_truth + '\n' + page.timeline;
    if (!opts.dryRun) await retractRemovedTimelineEntries(engine, slug, source_id, fullContent, snapshot ? { expectedRevision: snapshot.revision } : {});
    let entries = segmented ? segmented.entries : parseTimelineEntries(page.compiled_truth + '\n' + page.timeline);
    // --infer-dates: pages with no in-body timeline line but a trustworthy
    // content date (frontmatter / filename) get one anchor entry at that date.
    if (entries.length === 0 && opts.inferDates) {
      const anchor = anchorFor(page, slug);
      if (anchor) entries = [anchor];
    }
    const completion = extractCompletionKey(source_id, slug, snapshot!.revision);
    if (!opts.dryRun) { if (entries.length) unflushed.add(completion); else opts.completed?.add(completion); }
    for (const [n, entry] of entries.entries()) {
      if (dryRunSeen) {
        const key = `${source_id}::${slug}::${entry.date}::${entry.summary}`;
        if (dryRunSeen.has(key)) continue;
        dryRunSeen.add(key);
        if (opts.jsonMode) {
          process.stdout.write(JSON.stringify({
            action: 'add_timeline', slug, source_id, date: entry.date,
            summary: entry.summary, ...(entry.detail ? { detail: entry.detail } : {}),
          }) + '\n');
        } else if (!opts.quiet) {
          console.log(`  ${slug}: ${entry.date} — ${entry.summary}`);
        }
        result.created++;
      } else {
        batch.push({ slug, date: entry.date, source: entry.source, summary: entry.summary, detail: entry.detail || '', source_id });
        if (batch.length >= BATCH_SIZE) { await flush(); if (n < entries.length - 1) unflushed.add(completion); }
      }
    }
    result.pages++;
    progress.tick(1);
  }
  await flush();
  progress.finish();
  result.refusal_codes = [...codes];

  if (!opts.jsonMode && !opts.quiet) {
    const label = opts.dryRun ? '(dry run) would create' : 'created';
    console.log(`Timeline: ${label} ${result.created} entries from ${result.pages} pages (db source)` +
      (result.skipped ? `; ${result.skipped} page(s) changed during the run were left for the next run` : '') +
      (result.pending ? `; ${result.pending} page(s) accepted but still pending (writer busy); rerun the same command to confirm` : ''));
  }
  if (result.refused > 0) {
    const unit = managed ? 'page(s)' : 'row(s)';
    console.error(
      `[extract timeline] refused: ${result.refusal_codes.join(', ')}; ${result.refused} ${unit} not written, ${result.created} written. ` +
      `Existing timeline rows are untouched (import already stored the rows of each page it wrote). ` +
      `Recovery: gbrain extract --stale${opts.sourceIdFilter ? ` --source-id ${opts.sourceIdFilter}` : ''}. Docs: ${TIMELINE_REFUSAL_DOCS}`,
    );
  }
  return result;
}

/** A page's stored non-event timeline rows, read once for both planning checks. */
interface StoredTimelineRows { id: number; date: string; date_text: string; source: string; summary: string; detail: string | null }

interface TimelineReadBatch {
  start: number;
  covered: number;
  snapshots: Map<string, PageSnapshot>;
  /** Managed walks only: stored rows by page id (inline pages). */
  stored: Map<number, StoredTimelineRows[]>;
}

/** The stored non-event timeline rows of each page, by page id. */
async function storedTimelineRows(engine: BrainEngine, pageIds: readonly number[]): Promise<Map<number, StoredTimelineRows[]>> {
  const stored = new Map<number, StoredTimelineRows[]>();
  const rows = await engine.executeRaw<StoredTimelineRows & { page_id: number }>(
    `SELECT page_id, id, to_char(date, 'YYYY-MM-DD') AS date, date::text AS date_text, source, summary, detail FROM timeline_entries
      WHERE page_id = ANY($1::int[]) AND event_page_id IS NULL ORDER BY date, id`, [pageIds]);
  for (const { page_id, ...row } of rows) {
    const list = stored.get(Number(page_id)) ?? [];
    list.push(row);
    stored.set(Number(page_id), list);
  }
  return stored;
}

async function readTimelineBatch(engine: BrainEngine, refs: ReadonlyArray<{ slug: string; source_id: string }>, start: number,
  withStoredRows: boolean): Promise<TimelineReadBatch> {
  const { snapshots, covered } = await engine.readPageSnapshotsBatch(refs.slice(start, start + BATCH_SIZE)
    .map(ref => ({ slug: ref.slug, sourceId: ref.source_id })));
  // A segmented page's rows are streamed by its plan (plannedTimeline), not held for the batch.
  const pageIds = [...snapshots.values()].filter(snapshot => !isSegmentedPage(snapshot.page)).map(snapshot => Number(snapshot.page.id));
  const stored = withStoredRows && pageIds.length ? await storedTimelineRows(engine, pageIds) : new Map<number, StoredTimelineRows[]>();
  return { start, covered, snapshots, stored };
}

/**
 * The timeline change one page needs: removed rows to retract and canonical
 * tuples (or an inferred anchor) to insert. `stored` is the caller's read of
 * the page's rows; without it they are read once here. A segmented body is
 * read once, block by block, keeping only what differs from the stored rows
 * (the keys of the rows it produces, the entries no row records). `retract`
 * deletes the removed rows; `stop` is asked before each segment read.
 */
async function plannedTimeline(engine: BrainEngine, page: Page, slug: string, sourceId: string, inferDates: boolean, stored?: StoredTimelineRows[],
  opts: { retract?: boolean; stop?: () => boolean; versions?: VersionTimelineScan } = {}) {
  const retraction = { dryRun: opts.retract !== true, ...(opts.stop ? { stop: opts.stop } : {}), ...(opts.versions ? { versions: opts.versions } : {}) };
  const strip = (rows: StoredTimelineRows[]) => rows.map(({ date_text: _text, ...row }) => row);
  let entries: Array<{ date: string; source?: string; summary: string; detail?: string }>;
  let removed: unknown[];
  let parsed: number;
  let rowCount: number;
  if (!isSegmentedPage(page)) {
    const rows = stored ?? (await storedTimelineRows(engine, [Number(page.id)])).get(Number(page.id)) ?? [];
    const text = page.compiled_truth + '\n' + page.timeline;
    removed = await retractRemovedTimelineEntries(engine, slug, sourceId, text, { ...retraction, storedRows: strip(rows) });
    entries = await unrecordedCanonicalTimeline(engine, page.id, page, slug, rows.map(row => ({ date: row.date_text, source: row.source, summary: row.summary })));
    parsed = parseTimelineEntries(text).length;
    rowCount = rows.length;
  } else {
    // A segmented body is read once, through the engine given (inside the coordinator's transaction too), and its
    // rows are held as key digests (TimelineKeySet), never as one array unless the caller read them.
    const within = new TimelineKeySet();
    const recorded = new TimelineKeySet();
    rowCount = 0;
    const take = (row: StoredTimelineRows) => {
      within.add(storedTimelineKey(row));
      recorded.add(timelineKey({ date: row.date_text, source: row.source, summary: row.summary }));
      rowCount++;
    };
    if (stored) stored.forEach(take);
    else await eachStoredTimelineRow(engine, Number(page.id), take);
    const collector = segmentedCanonicalCollector(slug, recorded);
    const keys = await segmentedTimelineText(engine, page, { within, entries: false, ...(opts.stop ? { stop: opts.stop } : {}), onBlock: text => collector.add(text) });
    removed = await retractRemovedTimelineEntries(engine, slug, sourceId, keys, { ...retraction,
      storedRows: stored ? strip(stored) : await unkeptTimelineRows(engine, Number(page.id), keys) });
    entries = [...collector.result().values()];
    parsed = keys.parsed;
  }
  if (entries.length === 0 && inferDates && parsed === 0 && rowCount === 0) {
    const anchor = anchorFor(page, slug);
    if (anchor) entries = [anchor];
  }
  return { removed: removed.length, entries };
}

/** Under the coordinator's page lock: retract removed rows (checked against `versions`, read before the lock), insert the planned tuples. */
async function writePageTimeline(tx: BrainEngine, page: Page, slug: string, sourceId: string, inferDates: boolean, versions: VersionTimelineScan): Promise<number> {
  const plan = await plannedTimeline(tx, page, slug, sourceId, inferDates, undefined, { retract: true, versions });
  if (!plan.entries.length) return 0;
  return tx.addTimelineEntriesBatch(plan.entries.map(entry => ({ slug, date: entry.date, source: entry.source,
    summary: entry.summary, detail: entry.detail || '', source_id: sourceId })), { auditSite: 'extract.timeline_db' });
}

/**
 * One page on a managed brain: null when the page is missing or filtered out,
 * 0 when nothing changes (no request admitted), else the rows the committed
 * request added. The request id is bound to the page revision, so a rerun
 * replays an accepted request instead of admitting a second one. `read` is
 * the caller's batched read of the page and its stored rows.
 */
async function publishPageTimeline(engine: BrainEngine, authority: MaintenanceAuthority, slug: string, sourceId: string,
  opts: TimelineDbOptions, read?: { snapshot: PageSnapshot | null; stored: StoredTimelineRows[] | undefined }, stop?: () => boolean): Promise<number | null | 'skipped' | 'pending'> {
  const snapshot = read ? read.snapshot : await engine.readPageSnapshot(slug, { sourceId });
  if (!snapshot || isQuarantined(snapshot.page.frontmatter) || (opts.typeFilter && snapshot.page.type !== opts.typeFilter)) return null;
  const plan = await plannedTimeline(engine, snapshot.page, slug, sourceId, opts.inferDates === true, read?.stored, stop ? { stop } : {});
  if (!plan.removed && !plan.entries.length) return 0;
  for (let attempt = 0; ; attempt++) {
    const h = digest(['extract-timeline-db-v1', sourceId, slug, snapshot.revision, attempt]);
    const requestId = `${h.slice(0, 8)}-${h.slice(8, 12)}-4${h.slice(13, 16)}-a${h.slice(17, 20)}-${h.slice(20, 32)}`;
    const prior = await getWriteRequest(engine, authority.writer.principal, requestId);
    if (prior && isTerminal(prior) && prior.state !== 'committed') continue;
    try {
      const receipt = await submitDatabaseMaintenanceIntent(engine, authority, slug,
        { kind: TIMELINE_EXTRACT_INTENT, expected_revision: snapshot.revision, infer_dates: opts.inferDates === true }, requestId);
      return Number(receipt.added ?? 0);
    } catch (error) {
      if (error instanceof OperationError && ['revision_conflict', 'page_not_found', 'page_identity_changed'].includes(error.code)) return 'skipped';
      if (error instanceof OperationError && error.code === 'write_pending') return 'pending';
      throw error;
    }
  }
}

/**
 * One page's timeline on a managed brain, through the coordinator (the serve
 * sweep's path): rows added, 0 when nothing changes, or 'unsettled' when the
 * page changed mid-run or its request is still pending (retry next time).
 * `stop` is the caller's budget, asked before each segment the planning read
 * takes (BodyReadStopped ends it before any request is admitted).
 */
export async function publishManagedPageTimeline(engine: BrainEngine, slug: string, sourceId: string, stop?: () => boolean): Promise<number | 'unsettled'> {
  const authority = await maintenancePreflight(engine, sourceId);
  if (!authority) throw opError('writer_coordinator_required', 'This brain is not managed; write the timeline batch directly.',
    `Source ${sourceId} has no managed writer, so there is no coordinator to publish through.`);
  const outcome = await publishPageTimeline(engine, authority, slug, sourceId, { dryRun: false, jsonMode: false }, undefined, stop);
  return outcome === 'skipped' || outcome === 'pending' ? 'unsettled' : outcome ?? 0;
}

/** Preparer for `managed_maintenance_timeline_extract`: a database-only publication on the page key. */
function timelinePageChanged(row: WriteRequest, message: string): OperationError {
  return opError('page_identity_changed', message,
    `Page ${row.slug} in source ${row.source_id} changed before its queued timeline extraction ran, so this page was skipped. Read the page that holds the slug now; gbrain extract timeline --source db --source-id ${row.source_id} extracts it again under a new request.`,
    { fix: readFix(`Shows which page holds ${row.slug} now and its revision.`, { argv: ['gbrain', 'get', '--source', row.source_id, '--', row.slug], mcp: { tool: 'get_page', arguments: { slug: row.slug, source_id: row.source_id } } }) });
}

export async function prepareTimelineExtract(engine: BrainEngine, row: WriteRequest): Promise<PreparedMutation> {
  const snapshot = await engine.readPageSnapshot(row.slug, { sourceId: row.source_id });
  if (!snapshot || snapshot.page.id !== Number(row.page_id)) throw timelinePageChanged(row, 'The page was deleted or replaced before its timeline was extracted.');
  await authorizeWrite(engine, row.authority, 'submit_job', row.slug);
  const inferDates = (row.intent as { infer_dates?: unknown } | null)?.infer_dates === true;
  // #9362: the page's stored versions are read here, one body per statement, not inside the publication transaction.
  // A segmented page's own snapshots are capped (pruneSegmentedVersions) and read segment by segment there, so its
  // body is planned here only when it also keeps inline snapshots, whose number nothing caps.
  const versions = versionTimelineScan();
  if (!isSegmentedPage(snapshot.page) || (await engine.executeRaw(`SELECT 1 FROM page_versions WHERE page_id = $1 AND body_storage = 'inline' LIMIT 1`,
    [snapshot.page.id])).length) await plannedTimeline(engine, snapshot.page, row.slug, row.source_id, inferDates, undefined, { versions });
  return { observedRevision: snapshot.revision, noop: true,
    validate: async tx => { await authorizeWrite(tx, row.authority, 'submit_job', row.slug); },
    apply: async tx => {
      const current = await tx.readPageSnapshot(row.slug, { sourceId: row.source_id });
      if (!current) throw timelinePageChanged(row, 'The page disappeared before its timeline was extracted.');
      return { status: 'completed', added: await writePageTimeline(tx, current.page, row.slug, row.source_id, inferDates, versions) };
    } };
}