Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #5762: the managed sync checkpoint's incomplete-receipt validation and the
* `checkpoint_validation_timeout` refusal.
*
* The validation is three index probes whose union is exactly the former
* single predicate (a receipt with recovery; an open page receipt of the run;
* a non-committed page receipt of the run with no committed sibling at the
* same index). Each probe is served by an index: `persistence_requests_recovery`,
* `persistence_requests_sync_run_open` and, for the sibling lookup,
* `persistence_requests_sync_run_committed`. #6735: the sibling lookup is a
* `LATERAL ... LIMIT 1` probe keyed on (source, run, index), which the planner
* cannot flatten into an anti-join over a materialized scan of every committed
* receipt of the run, so its cost is linear in the run's non-committed receipts. A statement timeout of this one
* statement is a deterministic failure on a large request table, not
* contention, so it becomes the terminal typed code instead of being released
* as `database_contention` and retried ahead of every other write.
*
* The coordinator persists only the code and message, so the hint is rebuilt
* by one formatter from a fresh read of the index state plus the request's
* recorded source and processing options, for the sync summary and for
* `gbrain write-request` / MCP `get_write_request`.
*/
import type { BrainEngine } from '../engine.ts';
import { OperationError, opError } from '../ops/contract.ts';
import { PERSISTENCE_REQUEST_ONLINE_INDEXES, PERSISTENCE_SYNC_RUN_INDEXES } from './schema.ts';
import { docsAnchor } from './connector-errors.ts';
import type { SyncProcessingOptions } from './sync-authority.ts';
import type { SyncCursorOptions } from './sync-prepare.ts';
export const CHECKPOINT_VALIDATION_TIMEOUT = 'checkpoint_validation_timeout';
export const CHECKPOINT_VALIDATION_TIMEOUT_MESSAGE = 'The sync checkpoint validation timed out on a large request table; the checkpoint did not commit.';
export const REQUEST_INDEXES_REPAIR_COMMAND = 'gbrain repair request-indexes --apply';
const SYNC_PAGE_KINDS = "r.intent->>'kind' IN ('managed_sync_import','managed_sync_delete')";
/**
* #6643: clause 3 skips a terminal, recovery-free receipt the cursor settled another way: one the run converted in place
* ($3, #5988) and one at an index the cursor passed without a commit (waived or held: the settled ranges $4..$5).
* #6712 retired the knock-on exemption ($6): no page is cancelled for another page's failure any more, and the cursor
* converts a legacy knock-on it meets in place (sync-run.ts `refreezeInPlace`).
*/
const incompleteSyncReceiptSql = (limit: number) => `(SELECT r.request_id::text AS request_id FROM persistence_requests r
WHERE r.worktree_id=$1::uuid AND r.recovery IS NOT NULL LIMIT ${limit})
UNION ALL (SELECT r.request_id::text FROM persistence_requests r
WHERE r.worktree_id=$1::uuid AND r.intent->>'runId'=$2 AND r.state<>'committed' AND ${SYNC_PAGE_KINDS}
AND r.state IN ('queued','running','recovering') LIMIT ${limit})
UNION ALL (SELECT r.request_id::text FROM persistence_requests r
LEFT JOIN LATERAL (SELECT true AS found FROM persistence_requests committed WHERE committed.source_id=r.source_id
AND committed.intent->>'runId'=$2 AND committed.intent->>'index'=r.intent->>'index'
AND committed.state='committed' AND committed.intent ? 'runId' LIMIT 1) sibling ON true
WHERE r.worktree_id=$1::uuid AND r.intent->>'runId'=$2 AND r.state<>'committed' AND ${SYNC_PAGE_KINDS}
AND sibling.found IS NULL
AND NOT (r.request_id::text=ANY($3::text[]))
AND NOT (r.state IN ('failed','cancelled','conflict') AND r.recovery IS NULL
AND EXISTS (SELECT 1 FROM unnest($4::int[], $5::int[]) AS settled(lo, hi) WHERE (r.intent->>'index')::int BETWEEN settled.lo AND settled.hi)) LIMIT ${limit})
LIMIT ${limit}`;
export const INCOMPLETE_SYNC_RECEIPT_SQL = incompleteSyncReceiptSql(1);
/** #6643: the index ranges a cursor passed without a commit, and the index it began recording them at. */
export interface SyncSettledIndexes { from: number; ranges: Array<[number, number]> }
/**
* The request id of a receipt that blocks the run's checkpoint, or null.
* `superseded` names failed content-refusal requests the run converted in
* place (#5988): their entry was held, so they no longer block the checkpoint.
* `settled` (#6643) is the cursor's record of the indexes it passed without a
* commit; null for a checkpoint frozen before #6643, whose whole run predates it.
*/
export async function findIncompleteSyncReceipt(tx: Pick<BrainEngine, 'executeRaw'>, worktreeId: string, runId: string, superseded: string[] = [],
settled: SyncSettledIndexes | null = null): Promise<string | null> {
try {
const ranges = settled?.ranges ?? [];
const [row] = await tx.executeRaw<{ request_id: string }>(INCOMPLETE_SYNC_RECEIPT_SQL,
[worktreeId, runId, superseded, ranges.map(range => range[0]), ranges.map(range => range[1])]);
return row?.request_id ?? null;
} catch (error) {
const failure = error as { code?: string; message?: string };
if (failure?.code !== '57014' || !/statement timeout/.test(String(failure.message))) throw error;
throw opError(CHECKPOINT_VALIDATION_TIMEOUT, CHECKPOINT_VALIDATION_TIMEOUT_MESSAGE,
`Page writes that sync run ${runId} already committed stay committed; only its checkpoint did not. This usually means the request table lacks its lookup indexes: preview that repair and apply it after the user approves, then sync again.`,
{ fix: { argv: ['gbrain', 'repair', 'request-indexes', '--json'], consent: [], actor: 'agent', why: 'Previews the missing request-table indexes without changing anything.', requires_exclusive: false } });
}
}
/** The pass stamp's validation statement (`findOpenPassWork`): `$1` the worktree, `$2` the stamp's own request id. */
export const OPEN_PASS_WORK_SQL = `(SELECT r.request_id::text AS request_id FROM persistence_requests r
WHERE r.worktree_id=$1::uuid AND r.recovery IS NOT NULL LIMIT 1)
UNION ALL (SELECT r.request_id::text FROM persistence_requests r
WHERE r.worktree_id=$1::uuid AND r.state IN ('queued','running','recovering') AND r.request_id<>$2::uuid
AND r.intent->>'kind' IN ('managed_sync_import','managed_sync_delete','managed_sync_checkpoint') LIMIT 1)
LIMIT 1`;
/**
* #6712 PR 4b (design v3.2 §3.3, clause 2): a live pass's stamp commits only with no recovery row on the worktree and no
* other open managed-sync request of it (page kinds and the checkpoint, filtered on the row). The open read is served by
* `persistence_requests_pending` (`worktree_id, sequence`, partial on `queued|running|recovering`); the recovery read by
* `persistence_requests_recovery`. Returns the blocking request id, or null. A statement timeout is the typed
* `checkpoint_validation_timeout`, as clause 1 of the cursor validation.
*/
export async function findOpenPassWork(tx: Pick<BrainEngine, 'executeRaw'>, worktreeId: string, selfRequestId: string,
/** A writeback's folded stamp: its own recovery record is the file publication's journal, which its commit resolves. */
ownRecovery = false): Promise<string | null> {
try {
const [row] = await tx.executeRaw<{ request_id: string }>(ownRecovery ? OPEN_PASS_WORK_SQL.replace('r.recovery IS NOT NULL LIMIT 1', 'r.recovery IS NOT NULL AND r.request_id<>$2::uuid LIMIT 1') : OPEN_PASS_WORK_SQL,
[worktreeId, selfRequestId]);
return row?.request_id ?? null;
} catch (error) {
const failure = error as { code?: string; message?: string };
if (failure?.code !== '57014' || !/statement timeout/.test(String(failure.message))) throw error;
throw opError(CHECKPOINT_VALIDATION_TIMEOUT, CHECKPOINT_VALIDATION_TIMEOUT_MESSAGE,
'Page writes this sync pass already committed stay committed; only its stamp did not. This usually means the request table lacks its lookup indexes: preview that repair and apply it after the user approves, then sync again.',
{ fix: { argv: ['gbrain', 'repair', 'request-indexes', '--json'], consent: [], actor: 'agent', why: 'Previews the missing request-table indexes without changing anything.', requires_exclusive: false } });
}
}
/**
* #6735: `findIncompleteSyncReceipt` as a resume asks it, outside the coordinator: a timeout reads as no known blocker,
* so the revived checkpoint is validated again inside its own publication instead of the manifest being rediscovered.
*/
export async function probeCheckpointBlocker(...args: Parameters<typeof findIncompleteSyncReceipt>): Promise<string | null> {
return findIncompleteSyncReceipt(...args).catch(error => {
if (error instanceof OperationError && error.code === CHECKPOINT_VALIDATION_TIMEOUT) return null;
throw error;
});
}
/**
* #6643: every receipt (up to 20) that would block the run's checkpoint now, with what `sync status` needs to tell a
* knock-on from a root; read-only. The checkpoint itself names only the first (`findIncompleteSyncReceipt`).
*/
export async function listIncompleteSyncReceipts(engine: Pick<BrainEngine, 'executeRaw'>, worktreeId: string, runId: string, superseded: string[] = [],
settled: SyncSettledIndexes | null = null): Promise<Array<{ request_id: string; slug: string; state: string; code: string | null; index: number | null; recovery: boolean }>> {
const ranges = settled?.ranges ?? [];
return engine.executeRaw(`SELECT r.request_id::text AS request_id, r.slug, r.state, r.error_code AS code, (r.intent->>'index')::int AS index, r.recovery IS NOT NULL AS recovery
FROM persistence_requests r WHERE r.request_id::text IN (SELECT request_id FROM (${incompleteSyncReceiptSql(20)}) blockers) ORDER BY r.sequence LIMIT 20`,
[worktreeId, runId, superseded, ranges.map(range => range[0]), ranges.map(range => range[1])]);
}
/** #6643: `settled` with `[from, to]` added, merged into the last range when contiguous. */
export function addSettledRange(settled: SyncSettledIndexes, from: number, to: number): SyncSettledIndexes {
if (to < from) return settled;
const last = settled.ranges.at(-1);
if (last && from <= last[1] + 1) return { ...settled, ranges: [...settled.ranges.slice(0, -1), [last[0], Math.max(last[1], to)]] };
return { ...settled, ranges: [...settled.ranges, [from, to]] };
}
export interface RequestIndexState {
name: string;
state: 'valid' | 'missing' | 'invalid' | 'building';
progress?: { phase: string; blocks_done: number; blocks_total: number };
}
/**
* A fresh read of the online-built request indexes (#5762's checkpoint pair and #6511's slug index by default): valid,
* missing, INVALID, or still being built by another session. The checkpoint timeout hint reads the pair it depends on.
*/
export async function readRequestIndexStates(engine: Pick<BrainEngine, 'executeRaw' | 'kind'>,
indexes: ReadonlyArray<{ name: string }> = PERSISTENCE_REQUEST_ONLINE_INDEXES): Promise<RequestIndexState[]> {
const names = indexes.map(index => index.name);
if (engine.kind !== 'postgres') {
const rows = await engine.executeRaw<{ name: string; present: boolean }>(
'SELECT n.name, to_regclass(n.name) IS NOT NULL AS present FROM unnest($1::text[]) AS n(name)', [names]);
return rows.map(row => ({ name: row.name, state: row.present ? 'valid' : 'missing' }));
}
const rows = await engine.executeRaw<{ name: string; valid: boolean | null; phase: string | null; blocks_done: string | null; blocks_total: string | null }>(
`SELECT n.name, i.indisvalid AS valid, p.phase, p.blocks_done::text, p.blocks_total::text
FROM unnest($1::text[]) WITH ORDINALITY AS n(name, position)
LEFT JOIN pg_index i ON i.indexrelid = to_regclass(n.name)
LEFT JOIN LATERAL (SELECT phase, blocks_done, blocks_total FROM pg_stat_progress_create_index p
WHERE p.relid = i.indrelid AND p.index_relid IN (i.indexrelid, 0) LIMIT 1) p ON true
ORDER BY n.position`, [names]);
return rows.map(row => row.valid === null ? { name: row.name, state: 'missing' }
: row.valid ? { name: row.name, state: 'valid' }
: row.phase !== null ? { name: row.name, state: 'building', progress: { phase: row.phase, blocks_done: Number(row.blocks_done ?? 0), blocks_total: Number(row.blocks_total ?? 0) } }
: { name: row.name, state: 'invalid' });
}
const shellWord = (value: string) => /^[\w./@:=+-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
/**
* The exact retry for the failed run: the same source and discovery options, plus the recorded processing flags. #6712
* PR 3 (L7): a plain sync retries a recorded failure by itself, so the command carries no retry flag.
*/
export function checkpointRetryCommand(input: { sourceId: string; processingOptions?: Partial<SyncProcessingOptions> | null; syncOptions?: SyncCursorOptions | null; repoPath?: string | null }): string {
const base = `gbrain sync --source ${shellWord(input.sourceId)} --no-pull`;
// A compacted receipt (intent cleared) or a checkpoint admitted before the options were recorded
// cannot name its cursor; never print assumed defaults for it.
if (!input.syncOptions) return `${base} with the same options as the failed run (this receipt no longer records them)`;
const options = input.processingOptions ?? {};
const cursor = input.syncOptions;
return [base, input.repoPath ? `--repo ${shellWord(input.repoPath)}` : '', options.noEmbed ? '--no-embed' : '', options.noExtract ? '--no-extract' : '',
options.noSchemaPack ? '--no-schema-pack' : '', cursor?.full ? '--full' : '', cursor?.workingTree ? '--working-tree' : '',
cursor?.srcSubpath ? `--src-subpath ${shellWord(cursor.srcSubpath)}` : '', ...(cursor?.exclude ?? []).map(pattern => `--exclude ${shellWord(pattern)}`),
...(cursor?.includeHidden ?? []).map(pattern => `--include-hidden ${shellWord(pattern)}`), cursor?.strategy ? `--strategy ${shellWord(cursor.strategy)}` : '']
.filter(Boolean).join(' ');
}
export interface CheckpointTimeoutHint { reason: string; message: string; suggestion: string; detail: 'index_building' | 'index_missing' | 'indexes_valid'; docs: string }
/** The one formatter for the refusal: a three-state hint from the index state, plus the filled retry command. */
export function formatCheckpointTimeoutHint(indexes: RequestIndexState[], input: { requestId: string | null; sourceId: string;
processingOptions?: Partial<SyncProcessingOptions> | null; syncOptions?: SyncCursorOptions | null; repoPath?: string | null }): CheckpointTimeoutHint {
const retry = checkpointRetryCommand(input);
const base = { reason: CHECKPOINT_VALIDATION_TIMEOUT, message: CHECKPOINT_VALIDATION_TIMEOUT_MESSAGE, docs: docsAnchor(CHECKPOINT_VALIDATION_TIMEOUT) };
const building = indexes.filter(index => index.state === 'building');
if (building.length) return { ...base, detail: 'index_building',
suggestion: `This release's request-index build is still running (${building.map(index => `${index.name}: ${index.progress?.phase ?? 'building'}`
+ `${index.progress?.blocks_total ? `, ${index.progress.blocks_done}/${index.progress.blocks_total} blocks` : ''}`).join('; ')}). `
+ `Wait for it to finish (gbrain doctor shows persistence_request_indexes), then run: ${retry}` };
const broken = indexes.filter(index => index.state === 'missing' || index.state === 'invalid');
if (broken.length) return { ...base, detail: 'index_missing',
suggestion: `The request index ${broken.map(index => `${index.name} is ${index.state === 'invalid' ? 'INVALID' : 'missing'}`).join(' and ')}. `
+ `Rebuild it: ${REQUEST_INDEXES_REPAIR_COMMAND} — then run: ${retry}` };
return { ...base, detail: 'indexes_valid',
suggestion: `The request indexes are valid, so there is nothing to wait for. Run: ${retry}. If it times out again, report request `
+ `${input.requestId ?? '<request-id>'} and attach the persistence_request_indexes and persistence_request_growth entries of gbrain doctor --json.` };
}
export async function checkpointTimeoutHint(engine: Pick<BrainEngine, 'executeRaw' | 'kind'>, input: Parameters<typeof formatCheckpointTimeoutHint>[1]): Promise<CheckpointTimeoutHint> {
return formatCheckpointTimeoutHint(await readRequestIndexStates(engine, PERSISTENCE_SYNC_RUN_INDEXES), input);
}