Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* `gbrain extract timeline --source db` (#5904 probe, E28).
*
* `timeline_entries` is a guarded table: on a managed brain (every fresh
* `gbrain init` brain is one) a raw batch insert is refused by
* `managed_writer_guard`. Managed brains therefore publish each page that
* needs a change as one database-only maintenance request
* (`managed_maintenance_timeline_extract`, owner-checked by
* `maintenancePreflight`): under the page lock, bound to the revision that
* was read, the rows an earlier version produced and the current text no
* longer does are retracted and the timeline tuples the canonical body
* projects with no stored row are inserted. A page with nothing to change
* admits no request. Unmanaged brains keep the batched raw insert.
*
* A refused write is reported as what it is: nothing was written and the
* timeline rows import already stored are untouched. Refusals are counted
* apart from written rows and make the command exit non-zero.
*/
import type { BrainEngine, TimelineBatchInput } from '../core/engine.ts';
import type { Page, PageType } from '../core/types.ts';
import { parseTimelineEntries, deriveTimelineAnchor } from '../core/link-extraction.ts';
import { eachStoredTimelineRow, retractRemovedTimelineEntries, segmentedTimelineText, storedTimelineKey, TimelineKeySet, unkeptTimelineRows, versionTimelineScan, type VersionTimelineScan } from '../core/timeline-extract.ts';
import { timelineKey } from '../core/timeline-marker.ts';
import { managedPersistenceEnabled } from '../core/persistence/ownership.ts';
import { maintenanceTransaction } from '../core/persistence/attribution.ts';
import { segmentedCanonicalCollector, unrecordedCanonicalTimeline } from '../core/persistence/canonical-projections.ts';
import { isSegmentedPage } from '../core/page-body/page-shape.ts';
import { maintenancePreflight, submitDatabaseMaintenanceIntent, type MaintenanceAuthority } from '../core/persistence/prepared-maintenance.ts';
import { authorizeWrite } from '../core/persistence/authority.ts';
import { getWriteRequest } from '../core/persistence/journal.ts';
import { isTerminal, type WriteRequest } from '../core/persistence/model.ts';
import { digest } from '../core/persistence/digest.ts';
import type { PreparedMutation } from '../core/persistence/coordinator.ts';
import { OperationError, opError } from '../core/ops/contract.ts';
import { readFix } from '../core/ops/op-fix.ts';
import { createProgress } from '../core/progress.ts';
import { importAnalyzeEveryPages, maybeRefreshPlannerStats } from '../core/planner-stats.ts';
import { getCliOptions, cliOptsToProgressOptions } from '../core/cli-options.ts';
import { extractCompletionKey, filterRefsSince } from './extract.ts';
import { isQuarantined } from '../core/quarantine.ts';
import { pageSnapshotKey } from '../core/page-snapshot-batch.ts';
import type { PageSnapshot } from '../core/page-state/types.ts';
const BATCH_SIZE = 100;
export const TIMELINE_EXTRACT_INTENT = 'managed_maintenance_timeline_extract';
export const TIMELINE_REFUSAL_DOCS = 'docs/guides/write-refusals.md#extract-timeline-refused';
export interface TimelineDbResult {
created: number;
pages: number;
/** Pages (managed) or rows (unmanaged batches) whose write was refused. */
refused: number;
/** Managed pages edited during the run; they keep their stored rows and are picked up by the next run. */
skipped: number;
/** Managed pages whose request was accepted but has not committed yet; rerunning resumes the same request. */
pending: number;
refusal_codes: string[];
}
export interface TimelineDbOptions {
dryRun: boolean;
jsonMode: boolean;
quiet?: boolean;
typeFilter?: PageType;
since?: string;
sourceIdFilter?: string;
inferDates?: boolean;
/** Restrict the walk to these slugs (requires sourceIdFilter). */
slugs?: readonly string[];
/** S4 (P2.18): receives `extractCompletionKey` for each page whose timeline write landed (or had nothing to write). */
completed?: Set<string>;
}
/**
* The refusal reason: the error's own code, unless it is the coordinator's
* generic `storage_error`, whose message carries the database refusal
* (`writer_coordinator_required: ...`) that names the real reason.
*/
export function refusalCode(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
const code = (error as { code?: unknown })?.code;
const own = typeof code === 'string' && /^[a-z][a-z0-9_]+$/.test(code) ? code : null;
if (own && own !== 'storage_error') return own;
return /(?:^|[\s(])([a-z][a-z0-9]*_[a-z0-9_]+):/.exec(message)?.[1] ?? own ?? 'storage_error';
}
function reportRefusal(opts: TimelineDbOptions, code: string, where: string, message: string): void {
if (opts.jsonMode) {
process.stderr.write(JSON.stringify({ event: 'timeline_refused', code, where, error: message }) + '\n');
} else {
console.error(` refused: ${code}; nothing written; existing timeline rows are untouched (${where})`);
}
}
function anchorFor(page: Page, slug: string) {
return deriveTimelineAnchor({ slug, title: page.title, effectiveDate: page.effective_date, effectiveDateSource: page.effective_date_source });
}
async function walkRefs(engine: BrainEngine, opts: TimelineDbOptions) {
const all = await (await import('../core/guests/derived.ts')).withoutGuestSources(engine, 'extract', opts.sourceIdFilter
? (await engine.listAllPageRefs()).filter(r => r.source_id === opts.sourceIdFilter)
: await engine.listAllPageRefs(), ref => ref.source_id);
const wanted = opts.slugs ? new Set(opts.slugs) : null;
return filterRefsSince(wanted ? all.filter(r => wanted.has(r.slug)) : all, opts.since);
}
/**
* Walk the selected pages from the database and add their timeline rows.
* Managed brains publish per page through the coordinator; unmanaged brains
* batch raw inserts (`ON CONFLICT DO NOTHING`).
*/
export async function extractTimelineFromDB(engine: BrainEngine, opts: TimelineDbOptions): Promise<TimelineDbResult> {
const refs = await walkRefs(engine, opts);
const result: TimelineDbResult = { created: 0, pages: 0, refused: 0, skipped: 0, pending: 0, refusal_codes: [] };
const authorities = new Map<string, Promise<MaintenanceAuthority>>();
const authorityFor = (sourceId: string) => {
if (!authorities.has(sourceId)) authorities.set(sourceId, maintenancePreflight(engine, sourceId).then(a => a!));
return authorities.get(sourceId)!;
};
const codes = new Set<string>();
const progress = createProgress(cliOptsToProgressOptions(getCliOptions()));
progress.start('extract.timeline_db', refs.length);
const managed = !opts.dryRun && await managedPersistenceEnabled(engine);
const dryRunSeen = opts.dryRun ? new Set<string>() : null;
const batch: TimelineBatchInput[] = [];
// PGLite has no autovacuum: a walk that grows timeline_entries from empty
// re-plans its per-page reads against stale statistics and slows with
// every row, so it refreshes them on the bulk-import cadence.
const analyzeEvery = opts.dryRun ? 0 : await importAnalyzeEveryPages(engine);
let walked = 0;
// Unmanaged pages whose rows sit in the unflushed batch, and pages a refused batch held: completion follows the flush.
const unflushed = new Set<string>(), refusedPages = new Set<string>();
async function flush() {
if (batch.length === 0) return;
const snapshot = batch.slice(), held = [...unflushed];
batch.length = 0;
unflushed.clear();
try {
result.created += await maintenanceTransaction(engine, tx => tx.addTimelineEntriesBatch(snapshot, { auditSite: 'extract.timeline_db' }));
for (const key of held) if (!refusedPages.has(key)) opts.completed?.add(key);
} catch (e) {
for (const key of held) { refusedPages.add(key); opts.completed?.delete(key); }
const code = refusalCode(e);
codes.add(code);
result.refused += snapshot.length;
reportRefusal(opts, code, `batch of ${snapshot.length} rows`, e instanceof Error ? e.message : String(e));
}
}
// Pages are read BATCH_SIZE at a time (one snapshot statement, and on a
// managed brain one read of their stored rows); a managed page with nothing
// to change then admits no request without another read.
let readBatch: TimelineReadBatch = { start: 0, covered: 0, snapshots: new Map(), stored: new Map() };
for (const [index, { slug, source_id }] of refs.entries()) {
if (analyzeEvery > 0 && walked > 0 && walked % analyzeEvery === 0) {
await flush();
await maybeRefreshPlannerStats(engine, 'extract', { throttle: false }).catch(() => undefined);
}
walked++;
if (index >= readBatch.start + readBatch.covered) readBatch = await readTimelineBatch(engine, refs, index, managed);
const snapshot = readBatch.snapshots.get(pageSnapshotKey(source_id, slug)) ?? null;
if (managed) {
try {
const outcome = await publishPageTimeline(engine, await authorityFor(source_id), slug, source_id, opts,
{ snapshot, stored: !snapshot ? [] : isSegmentedPage(snapshot.page) ? undefined : readBatch.stored.get(Number(snapshot.page.id)) ?? [] });
if (outcome === 'skipped') result.skipped++;
else if (outcome === 'pending') result.pending++;
else if (outcome !== null) { result.created += outcome; result.pages++; opts.completed?.add(extractCompletionKey(source_id, slug, snapshot!.revision)); }
} catch (e) {
const code = refusalCode(e);
codes.add(code);
result.refused++;
reportRefusal(opts, code, `${source_id}:${slug}`, e instanceof Error ? e.message : String(e));
}
progress.tick(1);
continue;
}
const page = snapshot?.page ?? null;
if (!page || isQuarantined(page.frontmatter)) continue;
if (opts.typeFilter && page.type !== opts.typeFilter) continue;
// A segmented body is read block by block (segmentedTimelineText), never as one string.
const segmented = isSegmentedPage(page) ? await segmentedTimelineText(engine, page) : null;
const fullContent = segmented ?? page.compiled_truth + '\n' + page.timeline;
if (!opts.dryRun) await retractRemovedTimelineEntries(engine, slug, source_id, fullContent, snapshot ? { expectedRevision: snapshot.revision } : {});
let entries = segmented ? segmented.entries : parseTimelineEntries(page.compiled_truth + '\n' + page.timeline);
// --infer-dates: pages with no in-body timeline line but a trustworthy
// content date (frontmatter / filename) get one anchor entry at that date.
if (entries.length === 0 && opts.inferDates) {
const anchor = anchorFor(page, slug);
if (anchor) entries = [anchor];
}
const completion = extractCompletionKey(source_id, slug, snapshot!.revision);
if (!opts.dryRun) { if (entries.length) unflushed.add(completion); else opts.completed?.add(completion); }
for (const [n, entry] of entries.entries()) {
if (dryRunSeen) {
const key = `${source_id}::${slug}::${entry.date}::${entry.summary}`;
if (dryRunSeen.has(key)) continue;
dryRunSeen.add(key);
if (opts.jsonMode) {
process.stdout.write(JSON.stringify({
action: 'add_timeline', slug, source_id, date: entry.date,
summary: entry.summary, ...(entry.detail ? { detail: entry.detail } : {}),
}) + '\n');
} else if (!opts.quiet) {
console.log(` ${slug}: ${entry.date} — ${entry.summary}`);
}
result.created++;
} else {
batch.push({ slug, date: entry.date, source: entry.source, summary: entry.summary, detail: entry.detail || '', source_id });
if (batch.length >= BATCH_SIZE) { await flush(); if (n < entries.length - 1) unflushed.add(completion); }
}
}
result.pages++;
progress.tick(1);
}
await flush();
progress.finish();
result.refusal_codes = [...codes];
if (!opts.jsonMode && !opts.quiet) {
const label = opts.dryRun ? '(dry run) would create' : 'created';
console.log(`Timeline: ${label} ${result.created} entries from ${result.pages} pages (db source)` +
(result.skipped ? `; ${result.skipped} page(s) changed during the run were left for the next run` : '') +
(result.pending ? `; ${result.pending} page(s) accepted but still pending (writer busy); rerun the same command to confirm` : ''));
}
if (result.refused > 0) {
const unit = managed ? 'page(s)' : 'row(s)';
console.error(
`[extract timeline] refused: ${result.refusal_codes.join(', ')}; ${result.refused} ${unit} not written, ${result.created} written. ` +
`Existing timeline rows are untouched (import already stored the rows of each page it wrote). ` +
`Recovery: gbrain extract --stale${opts.sourceIdFilter ? ` --source-id ${opts.sourceIdFilter}` : ''}. Docs: ${TIMELINE_REFUSAL_DOCS}`,
);
}
return result;
}
/** A page's stored non-event timeline rows, read once for both planning checks. */
interface StoredTimelineRows { id: number; date: string; date_text: string; source: string; summary: string; detail: string | null }
interface TimelineReadBatch {
start: number;
covered: number;
snapshots: Map<string, PageSnapshot>;
/** Managed walks only: stored rows by page id (inline pages). */
stored: Map<number, StoredTimelineRows[]>;
}
/** The stored non-event timeline rows of each page, by page id. */
async function storedTimelineRows(engine: BrainEngine, pageIds: readonly number[]): Promise<Map<number, StoredTimelineRows[]>> {
const stored = new Map<number, StoredTimelineRows[]>();
const rows = await engine.executeRaw<StoredTimelineRows & { page_id: number }>(
`SELECT page_id, id, to_char(date, 'YYYY-MM-DD') AS date, date::text AS date_text, source, summary, detail FROM timeline_entries
WHERE page_id = ANY($1::int[]) AND event_page_id IS NULL ORDER BY date, id`, [pageIds]);
for (const { page_id, ...row } of rows) {
const list = stored.get(Number(page_id)) ?? [];
list.push(row);
stored.set(Number(page_id), list);
}
return stored;
}
async function readTimelineBatch(engine: BrainEngine, refs: ReadonlyArray<{ slug: string; source_id: string }>, start: number,
withStoredRows: boolean): Promise<TimelineReadBatch> {
const { snapshots, covered } = await engine.readPageSnapshotsBatch(refs.slice(start, start + BATCH_SIZE)
.map(ref => ({ slug: ref.slug, sourceId: ref.source_id })));
// A segmented page's rows are streamed by its plan (plannedTimeline), not held for the batch.
const pageIds = [...snapshots.values()].filter(snapshot => !isSegmentedPage(snapshot.page)).map(snapshot => Number(snapshot.page.id));
const stored = withStoredRows && pageIds.length ? await storedTimelineRows(engine, pageIds) : new Map<number, StoredTimelineRows[]>();
return { start, covered, snapshots, stored };
}
/**
* The timeline change one page needs: removed rows to retract and canonical
* tuples (or an inferred anchor) to insert. `stored` is the caller's read of
* the page's rows; without it they are read once here. A segmented body is
* read once, block by block, keeping only what differs from the stored rows
* (the keys of the rows it produces, the entries no row records). `retract`
* deletes the removed rows; `stop` is asked before each segment read.
*/
async function plannedTimeline(engine: BrainEngine, page: Page, slug: string, sourceId: string, inferDates: boolean, stored?: StoredTimelineRows[],
opts: { retract?: boolean; stop?: () => boolean; versions?: VersionTimelineScan } = {}) {
const retraction = { dryRun: opts.retract !== true, ...(opts.stop ? { stop: opts.stop } : {}), ...(opts.versions ? { versions: opts.versions } : {}) };
const strip = (rows: StoredTimelineRows[]) => rows.map(({ date_text: _text, ...row }) => row);
let entries: Array<{ date: string; source?: string; summary: string; detail?: string }>;
let removed: unknown[];
let parsed: number;
let rowCount: number;
if (!isSegmentedPage(page)) {
const rows = stored ?? (await storedTimelineRows(engine, [Number(page.id)])).get(Number(page.id)) ?? [];
const text = page.compiled_truth + '\n' + page.timeline;
removed = await retractRemovedTimelineEntries(engine, slug, sourceId, text, { ...retraction, storedRows: strip(rows) });
entries = await unrecordedCanonicalTimeline(engine, page.id, page, slug, rows.map(row => ({ date: row.date_text, source: row.source, summary: row.summary })));
parsed = parseTimelineEntries(text).length;
rowCount = rows.length;
} else {
// A segmented body is read once, through the engine given (inside the coordinator's transaction too), and its
// rows are held as key digests (TimelineKeySet), never as one array unless the caller read them.
const within = new TimelineKeySet();
const recorded = new TimelineKeySet();
rowCount = 0;
const take = (row: StoredTimelineRows) => {
within.add(storedTimelineKey(row));
recorded.add(timelineKey({ date: row.date_text, source: row.source, summary: row.summary }));
rowCount++;
};
if (stored) stored.forEach(take);
else await eachStoredTimelineRow(engine, Number(page.id), take);
const collector = segmentedCanonicalCollector(slug, recorded);
const keys = await segmentedTimelineText(engine, page, { within, entries: false, ...(opts.stop ? { stop: opts.stop } : {}), onBlock: text => collector.add(text) });
removed = await retractRemovedTimelineEntries(engine, slug, sourceId, keys, { ...retraction,
storedRows: stored ? strip(stored) : await unkeptTimelineRows(engine, Number(page.id), keys) });
entries = [...collector.result().values()];
parsed = keys.parsed;
}
if (entries.length === 0 && inferDates && parsed === 0 && rowCount === 0) {
const anchor = anchorFor(page, slug);
if (anchor) entries = [anchor];
}
return { removed: removed.length, entries };
}
/** Under the coordinator's page lock: retract removed rows (checked against `versions`, read before the lock), insert the planned tuples. */
async function writePageTimeline(tx: BrainEngine, page: Page, slug: string, sourceId: string, inferDates: boolean, versions: VersionTimelineScan): Promise<number> {
const plan = await plannedTimeline(tx, page, slug, sourceId, inferDates, undefined, { retract: true, versions });
if (!plan.entries.length) return 0;
return tx.addTimelineEntriesBatch(plan.entries.map(entry => ({ slug, date: entry.date, source: entry.source,
summary: entry.summary, detail: entry.detail || '', source_id: sourceId })), { auditSite: 'extract.timeline_db' });
}
/**
* One page on a managed brain: null when the page is missing or filtered out,
* 0 when nothing changes (no request admitted), else the rows the committed
* request added. The request id is bound to the page revision, so a rerun
* replays an accepted request instead of admitting a second one. `read` is
* the caller's batched read of the page and its stored rows.
*/
async function publishPageTimeline(engine: BrainEngine, authority: MaintenanceAuthority, slug: string, sourceId: string,
opts: TimelineDbOptions, read?: { snapshot: PageSnapshot | null; stored: StoredTimelineRows[] | undefined }, stop?: () => boolean): Promise<number | null | 'skipped' | 'pending'> {
const snapshot = read ? read.snapshot : await engine.readPageSnapshot(slug, { sourceId });
if (!snapshot || isQuarantined(snapshot.page.frontmatter) || (opts.typeFilter && snapshot.page.type !== opts.typeFilter)) return null;
const plan = await plannedTimeline(engine, snapshot.page, slug, sourceId, opts.inferDates === true, read?.stored, stop ? { stop } : {});
if (!plan.removed && !plan.entries.length) return 0;
for (let attempt = 0; ; attempt++) {
const h = digest(['extract-timeline-db-v1', sourceId, slug, snapshot.revision, attempt]);
const requestId = `${h.slice(0, 8)}-${h.slice(8, 12)}-4${h.slice(13, 16)}-a${h.slice(17, 20)}-${h.slice(20, 32)}`;
const prior = await getWriteRequest(engine, authority.writer.principal, requestId);
if (prior && isTerminal(prior) && prior.state !== 'committed') continue;
try {
const receipt = await submitDatabaseMaintenanceIntent(engine, authority, slug,
{ kind: TIMELINE_EXTRACT_INTENT, expected_revision: snapshot.revision, infer_dates: opts.inferDates === true }, requestId);
return Number(receipt.added ?? 0);
} catch (error) {
if (error instanceof OperationError && ['revision_conflict', 'page_not_found', 'page_identity_changed'].includes(error.code)) return 'skipped';
if (error instanceof OperationError && error.code === 'write_pending') return 'pending';
throw error;
}
}
}
/**
* One page's timeline on a managed brain, through the coordinator (the serve
* sweep's path): rows added, 0 when nothing changes, or 'unsettled' when the
* page changed mid-run or its request is still pending (retry next time).
* `stop` is the caller's budget, asked before each segment the planning read
* takes (BodyReadStopped ends it before any request is admitted).
*/
export async function publishManagedPageTimeline(engine: BrainEngine, slug: string, sourceId: string, stop?: () => boolean): Promise<number | 'unsettled'> {
const authority = await maintenancePreflight(engine, sourceId);
if (!authority) throw opError('writer_coordinator_required', 'This brain is not managed; write the timeline batch directly.',
`Source ${sourceId} has no managed writer, so there is no coordinator to publish through.`);
const outcome = await publishPageTimeline(engine, authority, slug, sourceId, { dryRun: false, jsonMode: false }, undefined, stop);
return outcome === 'skipped' || outcome === 'pending' ? 'unsettled' : outcome ?? 0;
}
/** Preparer for `managed_maintenance_timeline_extract`: a database-only publication on the page key. */
function timelinePageChanged(row: WriteRequest, message: string): OperationError {
return opError('page_identity_changed', message,
`Page ${row.slug} in source ${row.source_id} changed before its queued timeline extraction ran, so this page was skipped. Read the page that holds the slug now; gbrain extract timeline --source db --source-id ${row.source_id} extracts it again under a new request.`,
{ fix: readFix(`Shows which page holds ${row.slug} now and its revision.`, { argv: ['gbrain', 'get', '--source', row.source_id, '--', row.slug], mcp: { tool: 'get_page', arguments: { slug: row.slug, source_id: row.source_id } } }) });
}
export async function prepareTimelineExtract(engine: BrainEngine, row: WriteRequest): Promise<PreparedMutation> {
const snapshot = await engine.readPageSnapshot(row.slug, { sourceId: row.source_id });
if (!snapshot || snapshot.page.id !== Number(row.page_id)) throw timelinePageChanged(row, 'The page was deleted or replaced before its timeline was extracted.');
await authorizeWrite(engine, row.authority, 'submit_job', row.slug);
const inferDates = (row.intent as { infer_dates?: unknown } | null)?.infer_dates === true;
// #9362: the page's stored versions are read here, one body per statement, not inside the publication transaction.
// A segmented page's own snapshots are capped (pruneSegmentedVersions) and read segment by segment there, so its
// body is planned here only when it also keeps inline snapshots, whose number nothing caps.
const versions = versionTimelineScan();
if (!isSegmentedPage(snapshot.page) || (await engine.executeRaw(`SELECT 1 FROM page_versions WHERE page_id = $1 AND body_storage = 'inline' LIMIT 1`,
[snapshot.page.id])).length) await plannedTimeline(engine, snapshot.page, row.slug, row.source_id, inferDates, undefined, { versions });
return { observedRevision: snapshot.revision, noop: true,
validate: async tx => { await authorizeWrite(tx, row.authority, 'submit_job', row.slug); },
apply: async tx => {
const current = await tx.readPageSnapshot(row.slug, { sourceId: row.source_id });
if (!current) throw timelinePageChanged(row, 'The page disappeared before its timeline was extracted.');
return { status: 'completed', added: await writePageTimeline(tx, current.page, row.slug, row.source_id, inferDates, versions) };
} };
}