Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6712 PR 2 (L5): the tail of a managed sync whose checkpoint committed: projection statistics and link extraction for the
* pages the run imported. It runs after the drain lease is released and outside the drain's in-pass governor, so a slow
* tail never holds the source or reads as a stalled pass. The checkpoint's own transaction records the obligation
* (`recordSyncTail`), so a process that dies between the checkpoint commit and the tail leaves it for the next tail on the
* source; the tail is bounded, and work it leaves undone stays durable (the obligation, and every unextracted page stays
* stale for `gbrain extract --stale`). A failed or truncated tail attaches one degraded notice with its fix; the run's
* outcome stays synced.
*/
import type { BrainEngine } from '../engine.ts';
import type { Notice } from '../agent-output.ts';
import { refreshProjectionStatistics } from '../search/projection-statistics.ts';
import { extractManagedStaleLinks, type ManagedLinkExtraction } from './links-maintenance.ts';
import { faultPoint } from './fault-points.ts';
import { currentDrainLease } from './sync-lease.ts';
import { readSyncTail, SYNC_TAIL_OP, type TailObligation } from './sync-tail-record.ts';
import type { SyncResult } from '../../commands/sync.ts';
/** The tail's link-extraction budget (it was the governed pass's `FINISH_BUDGET_MS`); a caller deadline shortens it. */
export const SYNC_TAIL_BUDGET_MS = 5 * 60_000;
/** Link extraction per tail, as before the move. */
const SYNC_TAIL_MAX_PAGES = 1000;
export interface SyncTailReport {
/**
* `done`: every step ran to the end; `truncated`: links stopped at the budget (the rest stay stale); `degraded`: the planner
* statistics could not be refreshed (doctor's check owns it); `deferred`: the caller's deadline left no budget, and `failed`:
* a step threw (both keep the obligation for the next tail).
*/
status: 'done' | 'truncated' | 'degraded' | 'deferred' | 'failed';
runs: number;
statistics: 'refreshed' | 'not_needed' | 'failed';
links?: ManagedLinkExtraction;
/** Pages this run imported that the tail's link extraction left underived (present when links ran); they stay stale. */
underived?: number;
notice?: Notice;
}
/**
* Runs the source's owed tail, bounded by `budgetMs` and the caller's deadline. Never throws: a failure is the report's
* `failed` status and its notice. The obligation is cleared (compare-and-swap) only when every step ran; a checkpoint
* that committed meanwhile keeps its own.
*/
export async function runSyncTail(engine: BrainEngine, sourceId: string, opts: { signal?: AbortSignal; deadlineAtMs?: number; budgetMs?: number } = {}): Promise<SyncTailReport | null> {
let owed: TailObligation[] | null;
try { owed = await readSyncTail(engine, sourceId); } catch (error) {
const failure = `reading the scheduled step failed: ${error instanceof Error ? error.message : String(error)}`;
return { status: 'failed', runs: 0, statistics: 'not_needed', notice: tailNotice(sourceId, 'failed', { statistics: 'not_needed', failure }) };
}
if (!owed) return null;
const deadlineAt = Date.now() + Math.max(0, Math.min(opts.budgetMs ?? SYNC_TAIL_BUDGET_MS, opts.deadlineAtMs === undefined ? Infinity : opts.deadlineAtMs - Date.now()));
const changed = owed.reduce((sum, item) => sum + (Number(item.changed) || 0), 0);
const extract = owed.some(item => item.extract);
let statistics: SyncTailReport['statistics'] = 'not_needed', links: ManagedLinkExtraction | undefined, failure: string | null = null, slugs: string[] = [];
try {
await faultPoint('sync:tail_started', { sourceId });
if (changed > 0) statistics = await refreshProjectionStatistics(engine, changed) ? 'refreshed' : 'failed';
if (extract && deadlineAt > Date.now()) {
// A cursor run's imports are its manifest's; a pass writes none, so its imports are its committed requests since its pin.
const passes = owed.filter(item => item.since);
slugs = (await engine.executeRaw<{ slug: string }>(`SELECT DISTINCT slug FROM (SELECT e->>'slug' AS slug FROM op_checkpoints m, jsonb_array_elements(m.completed_keys) e
WHERE m.op='managed-sync-manifest' AND m.fingerprint=ANY($1::text[]) AND e->>'action'='import' AND e->>'slug' IS NOT NULL
UNION SELECT r.slug FROM persistence_requests r, jsonb_to_recordset($2::text::jsonb) AS p(run_id text, since timestamptz, at timestamptz)
WHERE r.recovery IS NULL AND NOT r.compacted AND r.state='committed' AND r.completed_at BETWEEN p.since AND p.at AND r.source_id=$3
AND r.intent->>'passId'=p.run_id AND r.intent->>'kind'='managed_sync_import') imported LIMIT ${SYNC_TAIL_MAX_PAGES + 1}`,
[owed.map(item => item.run_id), JSON.stringify(passes.map(item => ({ run_id: item.run_id, since: item.since, at: item.at }))), sourceId],
{ timeoutMs: Math.max(1, deadlineAt - Date.now()) })).map(row => row.slug);
if (deadlineAt > Date.now()) links = await extractManagedStaleLinks(engine, { sourceId, maxPages: SYNC_TAIL_MAX_PAGES, timeBudgetMs: deadlineAt - Date.now(), signal: opts.signal, mentions: false, slugs });
}
} catch (error) {
failure = error instanceof Error ? error.message : String(error);
}
const ran = failure === null && (!extract || links !== undefined);
if (ran) await engine.executeRaw('DELETE FROM op_checkpoints WHERE op=$1 AND fingerprint=$2 AND completed_keys=$3::text::jsonb', [SYNC_TAIL_OP, sourceId, JSON.stringify(owed)]).catch(() => undefined);
// Imported pages are derived first, so the run left some underived only when it visited fewer pages than it imported;
// `remaining` is the source's whole stale backlog, older pages included, and is not this run's to report.
const underived = links ? Math.max(0, slugs.length - links.pages - links.skipped) : 0;
const truncated = underived > 0 && (Date.now() >= deadlineAt || links!.pages + links!.skipped >= SYNC_TAIL_MAX_PAGES);
const status: SyncTailReport['status'] = failure !== null ? 'failed' : !ran ? 'deferred' : truncated ? 'truncated' : statistics === 'failed' ? 'degraded' : 'done';
return { status, runs: owed.length, statistics, ...(links ? { links, underived } : {}), ...(status === 'done' ? {} : { notice: tailNotice(sourceId, status, { statistics, underived, failure }) }) };
}
function tailNotice(sourceId: string, status: Exclude<SyncTailReport['status'], 'done'>, d: { statistics: SyncTailReport['statistics']; underived?: number; failure: string | null }): Notice {
const extract = { argv: ['gbrain', 'extract', '--stale', '--source-id', sourceId], consent: [], actor: 'agent' as const, requires_exclusive: false,
why: 'Derives links for the pages this sync imported that are still stale; it changes no page content.',
verify: { argv: ['gbrain', 'sync', 'status', '--source', sourceId, '--json'] } };
if (status === 'truncated') {
return { code: 'sync_tail_degraded', kind: 'degraded', fix: extract,
why: `The sync of source ${sourceId} is synced, but link extraction stopped at its budget with ${d.underived} imported page(s) not yet derived; their links and backlinks are missing until they are extracted.` };
}
if (status === 'degraded') {
return { code: 'sync_tail_degraded', kind: 'degraded',
fix: { argv: ['gbrain', 'doctor', '--json'], consent: [], actor: 'agent', requires_exclusive: false, why: 'Names the planner-statistics check and the database-owner step that fixes it; changes nothing.' },
why: `The sync of source ${sourceId} is synced, but the search planner statistics could not be refreshed (database-owner maintenance is required), so searches may plan slowly.` };
}
const what = status === 'deferred' ? 'link extraction did not start (the run\'s deadline had passed)' : `the post-sync step failed (${d.failure ?? 'unknown error'})`;
return { code: 'sync_tail_degraded', kind: 'degraded', fix: extract,
why: `The sync of source ${sourceId} is synced, but ${what}; the step stays scheduled, the next sync of this source runs it again, and the fix runs the links now.` };
}
/**
* After the drain lease is released: runs the source's owed tail and folds it into the result (`tail`, and `links` when
* links ran). A dry run, a run that never held the lease (a follower) and a call still inside a lease (its holder runs
* the tail) run none.
*/
export async function withSyncTail(engine: BrainEngine, sourceId: string, result: SyncResult, opts: { signal?: AbortSignal; deadlineAtMs?: number; budgetMs?: number } = {}): Promise<SyncResult> {
if (result.status === 'dry_run' || result.leaseHeld || currentDrainLease(sourceId)) return result;
const tail = await runSyncTail(engine, sourceId, opts);
if (!tail) return result;
const { links, ...report } = tail;
return { ...result, tail: report, ...(links ? { links } : {}) };
}