Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
import type { BrainEngine } from '../engine.ts';
import { mirrorManagedSyncFailure, clearManagedSyncFailure } from '../sync-failure-ledger.ts';
import type { WriteRequest } from './model.ts';
import { checkpointRetryCommand } from './checkpoint-validation.ts';
import type { SyncProcessingOptions } from './sync-authority.ts';
import type { SyncCursorOptions } from './sync-prepare.ts';
import { resolveManagedSyncContext, syncGit, syncGitPath } from './sync-discovery.ts';
import { readGitHold } from './sync-holds.ts';
import { pruneMirrorOnlyManagedFailures } from './managed-sync-orphans.ts';
import { publicFailureDetail } from './publication-failure.ts';
import { VERSION } from '../../version.ts';
export interface ManagedSyncFailure {
source_id: string;
source_incarnation: string;
path: string;
code: string;
message: string;
request_id: string | null;
run_id: string;
target: string | null;
cursor_key: string;
phase: 'discovery' | 'freeze' | 'admission' | 'receipt' | 'checkpoint' | 'resume';
state: string;
observation_id: string;
first_seen: string;
attempts: number;
/**
* GBRA-133 S-60: how many runs in a row stopped on this path with this code and message (1 on the first); a different
* path, code or message starts over. Absent on failures recorded before it was kept.
*/
repeats?: number;
/** GBRA-133 S-60: the invocation that recorded it, so a resumed cursor's same observation still counts a new run. */
invocation?: string;
/** The cursor-selecting options of the failed run; absent on failures recorded before they were kept. */
syncOptions?: SyncCursorOptions;
processingOptions?: Partial<SyncProcessingOptions>;
/** #6513: the receipt's public failure detail (`publicFailureDetail`: fixed identifiers and the closed `reason`), when the receipt stored one. */
write_error_detail?: Record<string, unknown>;
/**
* #6643: the request whose failure this one is a knock-on of (or, for a checkpoint, the receipt that blocks it),
* followed through recorded knock-on roots. `request_id` null with `unknown` true: the canceller recorded no root
* (a receipt from before #6643); the victim is then the only request named.
*/
root_cause?: ManagedSyncRootCause;
/** #6712 PR 3 (L7): the gbrain version that recorded the failure; absent on failures recorded before it was kept. */
gbrain_version?: string;
}
export interface ManagedSyncRootCause { request_id: string | null; slug: string | null; state: string | null; code: string | null; message: string | null;
/** The cursor fence cause when the root is a cursor refusal (`outside_fence`, `run_replaced`, `window_dropped`). */
cause?: string; unknown?: true }
export async function recordManagedSyncFailure(engine: BrainEngine, value: Omit<ManagedSyncFailure, 'first_seen' | 'attempts' | 'repeats'> & { first_seen?: string }): Promise<{ failure: ManagedSyncFailure; ledgerRecorded: boolean }> {
const failure = { ...value, gbrain_version: VERSION, first_seen: value.first_seen ?? new Date().toISOString(), attempts: 1, repeats: 1 };
const recorded = await engine.transaction(async tx => {
await tx.executeRaw("SELECT set_config('synchronous_commit','on',true)");
const [row] = await tx.executeRaw<{ completed_keys: [ManagedSyncFailure] }>(`
WITH pair AS (SELECT completed_keys->0 AS was, $2::text::jsonb->0 AS incoming FROM op_checkpoints WHERE op='managed-sync-failure' AND fingerprint=$1),
same AS (SELECT was, incoming, was->>'path'=incoming->>'path' AND was->>'code'=incoming->>'code' AND was->>'message'=incoming->>'message' AS same,
COALESCE((was->>'repeats')::int,1)+1 AS repeats FROM pair)
INSERT INTO op_checkpoints(op,fingerprint,completed_keys) VALUES('managed-sync-failure',$1,$2::text::jsonb)
ON CONFLICT(op,fingerprint) DO UPDATE SET completed_keys=CASE
WHEN op_checkpoints.completed_keys->0->>'observation_id'=EXCLUDED.completed_keys->0->>'observation_id'
THEN CASE WHEN (SELECT same AND incoming->>'invocation' IS NOT NULL AND was->>'invocation' IS DISTINCT FROM incoming->>'invocation' FROM same)
THEN jsonb_build_array(op_checkpoints.completed_keys->0 || jsonb_build_object('repeats',(SELECT repeats FROM same),'invocation',EXCLUDED.completed_keys->0->>'invocation'))
ELSE op_checkpoints.completed_keys END
WHEN op_checkpoints.completed_keys->0->>'target' IS DISTINCT FROM EXCLUDED.completed_keys->0->>'target'
THEN EXCLUDED.completed_keys
ELSE jsonb_build_array(EXCLUDED.completed_keys->0 || jsonb_build_object(
'first_seen',op_checkpoints.completed_keys->0->>'first_seen',
'attempts',COALESCE((op_checkpoints.completed_keys->0->>'attempts')::int,0)+1,
'repeats',(SELECT CASE WHEN same THEN repeats ELSE 1 END FROM same))) END,
updated_at=now()
RETURNING completed_keys`, [value.cursor_key, JSON.stringify([failure])]);
return row.completed_keys[0];
});
let ledgerRecorded = false;
try {
mirrorManagedSyncFailure({ ...recorded, message: formatManagedSyncFailure(recorded) });
ledgerRecorded = true;
} catch { }
return { failure: recorded, ledgerRecorded };
}
export async function clearManagedSyncFailureAfterSuccess(engine: BrainEngine, key: string): Promise<void> {
const removed = await engine.executeRaw(`DELETE FROM op_checkpoints WHERE op='managed-sync-failure' AND fingerprint=$1
AND NOT EXISTS(SELECT 1 FROM op_checkpoints c WHERE c.op='managed-sync' AND c.fingerprint=$1 AND COALESCE(c.completed_keys->0->>'done','false')<>'true') RETURNING fingerprint`, [key]);
if (removed.length) try { clearManagedSyncFailure(key); } catch { }
}
/** Where a successful managed sync of a source stands: its registered root, Git root and the commit it imported. */
export interface ManagedSyncImported { sourceId: string; incarnation: string; root: string; gitRoot: string; target: string }
/** The imported state of a managed source on its owner host; throws when this host does not own an active binding. */
export async function readManagedSyncImported(engine: BrainEngine, sourceId: string): Promise<ManagedSyncImported | null> {
const context = await resolveManagedSyncContext(engine, { sourceId, noPull: true });
const [source] = await engine.executeRaw<{ last_commit: string | null }>('SELECT last_commit FROM sources WHERE id=$1 AND incarnation=$2::uuid', [sourceId, context.incarnation]);
if (!source?.last_commit) return null;
return { sourceId, incarnation: context.incarnation, root: context.root, gitRoot: context.gitRoot, target: source.last_commit };
}
function blobAt(gitRoot: string, commit: string, path: string): string | null {
try { return syncGit(gitRoot, ['rev-parse', '--verify', '--quiet', `${commit}:${path}`]).trim() || null; } catch { return null; }
}
/**
* #6288: why a recorded managed failure is not settled by the imported state, or null when it is. The imported commit is
* the source's last successful managed sync, which leaves every path it does not hold imported as of that commit; so a
* failure is settled when its file exists there and the database holds a live page for it with no hold on the path, or
* when the file is gone there and no live page claims it.
*/
export async function unsettledReason(engine: BrainEngine, run: ManagedSyncImported, failure: Pick<ManagedSyncFailure, 'source_id' | 'source_incarnation' | 'path'>): Promise<string | null> {
if (failure.source_id !== run.sourceId || failure.source_incarnation !== run.incarnation) return 'source_changed';
if (failure.path.startsWith('<')) return 'no_recorded_file';
const gitPath = syncGitPath(run, failure.path);
const present = blobAt(run.gitRoot, run.target, gitPath) !== null;
const live = await engine.executeRaw('SELECT 1 FROM pages WHERE source_id=$1 AND deleted_at IS NULL AND source_path = ANY($2::text[]) LIMIT 1', [run.sourceId, [failure.path, gitPath]]);
if (!present) return live.length ? 'not_deleted' : null;
if (!live.length) return 'not_imported';
return await readGitHold(engine, run.sourceId, run.incarnation, failure.path) ? 'held' : null;
}
async function deleteFailure(engine: BrainEngine, key: string, failure: ManagedSyncFailure): Promise<boolean> {
const removed = await engine.executeRaw(`DELETE FROM op_checkpoints WHERE op='managed-sync-failure' AND fingerprint=$1
AND completed_keys->0->>'observation_id'=$2 RETURNING fingerprint`, [key, failure.observation_id]);
if (removed.length) try { clearManagedSyncFailure(key); } catch { }
return removed.length > 0;
}
async function sourceFailureRows(engine: BrainEngine, sourceId: string): Promise<Array<{ key: string; failure: ManagedSyncFailure; unfinished: boolean }>> {
const rows = await engine.executeRaw<{ key: string; failure: ManagedSyncFailure; unfinished: boolean }>(`SELECT f.fingerprint AS key,f.completed_keys->0 AS failure,
EXISTS(SELECT 1 FROM op_checkpoints c WHERE c.op='managed-sync' AND c.fingerprint=f.fingerprint AND COALESCE(c.completed_keys->0->>'done','false')<>'true') AS unfinished
FROM op_checkpoints f WHERE f.op='managed-sync-failure' AND f.completed_keys->0->>'source_id'=$1 ORDER BY f.fingerprint`, [sourceId]);
return rows;
}
/**
* #6288: after a successful managed sync, clears this source's recorded failures under other cursor keys (a `--full` run's
* key is never resumed by autopilot) whose cursor is finished or gone and whose file the imported commit already settles.
* #6518: then prunes the source's local mirror rows whose durable failure row is already gone (`mirror_only`); a failed
* durable read prunes nothing.
*/
export async function clearSettledManagedSyncFailures(engine: BrainEngine, run: ManagedSyncImported): Promise<number> {
let cleared = 0;
for (const row of await sourceFailureRows(engine, run.sourceId)) {
if (row.unfinished || await unsettledReason(engine, run, row.failure)) continue;
if (await deleteFailure(engine, row.key, row.failure)) cleared++;
}
await pruneMirrorOnlyManagedFailures(engine, run.sourceId);
return cleared;
}
export interface ManagedFailureAcknowledgement { cleared: ManagedSyncFailure[]; refused: Array<{ failure: ManagedSyncFailure; reason: string }> }
/**
* #6288: `gbrain sync --source <id> --acknowledge-managed <request_id|path>` (host CLI only). Removes the matching recorded
* failures only when the imported state already settles them; anything else is refused by reason, so no content is skipped.
*/
export async function acknowledgeManagedSyncFailures(engine: BrainEngine, run: ManagedSyncImported, selector: string): Promise<ManagedFailureAcknowledgement> {
const result: ManagedFailureAcknowledgement = { cleared: [], refused: [] };
for (const row of await sourceFailureRows(engine, run.sourceId)) {
if (row.failure.request_id !== selector && row.failure.path !== selector) continue;
const reason = row.unfinished ? 'cursor_unfinished' : await unsettledReason(engine, run, row.failure);
if (reason) result.refused.push({ failure: row.failure, reason });
else if (await deleteFailure(engine, row.key, row.failure)) result.cleared.push(row.failure);
}
return result;
}
export async function readManagedSyncFailures(engine: BrainEngine, sourceIds?: string[]): Promise<ManagedSyncFailure[]> {
const rows = await engine.executeRaw<{ cursor_key: string; value: { sourceId: string; incarnation: string; runId: string; index: number; target: string; pending?: { requestId: string };
syncOptions: SyncCursorOptions | null; processingOptions: Partial<SyncProcessingOptions> | null };
receipt: Pick<WriteRequest, 'state' | 'request_id' | 'error_code' | 'error_message' | 'error_detail'> | null; failure: ManagedSyncFailure | null; path: string | null; updated_at: string }>(`
SELECT c.fingerprint AS cursor_key,jsonb_build_object('sourceId',s.id,'incarnation',s.incarnation,'runId',c.completed_keys->0->>'runId',
'index',c.completed_keys->0->'index','target',c.completed_keys->0->>'target',
'syncOptions',c.completed_keys->0->'syncOptions','processingOptions',c.completed_keys->0->'processingOptions',
'pending',jsonb_build_object('requestId',c.completed_keys->0->'pending'->>'requestId')) AS value,
CASE WHEN r.id IS NULL THEN NULL ELSE jsonb_build_object('state',r.state,'request_id',r.request_id,'error_code',r.error_code,'error_message',r.error_message,'error_detail',r.error_detail) END AS receipt,
f.completed_keys->0 AS failure,m.completed_keys->((c.completed_keys->0->>'index')::int)->>'path' AS path,c.updated_at
FROM op_checkpoints c
JOIN sources s ON s.id=c.completed_keys->0->>'sourceId' AND s.incarnation::text=c.completed_keys->0->>'incarnation'
LEFT JOIN persistence_requests r ON r.source_id=s.id AND r.source_incarnation=s.incarnation AND r.request_id=(c.completed_keys->0->'pending'->>'requestId')::uuid
AND r.principal_id=c.completed_keys->0->'authority'->'writer'->'principal'->>'id'
AND r.principal_kind=c.completed_keys->0->'authority'->'writer'->'principal'->>'kind'
LEFT JOIN op_checkpoints f ON f.op='managed-sync-failure' AND f.fingerprint=c.fingerprint
LEFT JOIN op_checkpoints m ON m.op='managed-sync-manifest' AND m.fingerprint=c.completed_keys->0->>'runId'
WHERE c.op='managed-sync' AND COALESCE(c.completed_keys->0->>'done','false')<>'true'
AND ($1::text[] IS NULL OR s.id=ANY($1::text[]))`, [sourceIds ?? null]);
const failures: ManagedSyncFailure[] = rows.map(row => {
const c = row.value, r = row.receipt;
const options = { ...(c.syncOptions ? { syncOptions: c.syncOptions } : {}), ...(c.processingOptions ? { processingOptions: c.processingOptions } : {}) };
// #6513: the pending receipt's bounded public detail rides the failure it belongs to (the row's own copy, when recorded, is the same projection).
const detail = r?.error_detail ? publicFailureDetail(r.error_detail) : undefined;
if (row.failure && (!r || !['failed', 'conflict', 'cancelled'].includes(r.state) || row.failure.request_id === r.request_id)) {
return { ...options, ...(detail && row.failure.request_id === r?.request_id ? { write_error_detail: detail } : {}), ...row.failure };
}
return { ...options, ...(detail ? { write_error_detail: detail } : {}), source_id: c.sourceId, source_incarnation: c.incarnation, path: row.path ?? '<checkpoint>', code: r?.error_code ?? 'sync_incomplete',
message: r?.error_message ?? 'The durable sync cursor is unfinished; resume the accepted run.', request_id: c.pending?.requestId ?? null,
run_id: c.runId, target: c.target, cursor_key: row.cursor_key, phase: r ? 'receipt' : 'resume', state: r?.state ?? 'unfinished',
observation_id: c.pending?.requestId ?? `${c.runId}:${c.index}`, first_seen: new Date(row.updated_at).toISOString(), attempts: 1 };
});
const orphaned = await engine.executeRaw<{ failure: ManagedSyncFailure }>(`
SELECT f.completed_keys->0 AS failure FROM op_checkpoints f
JOIN sources s ON s.id=f.completed_keys->0->>'source_id' AND s.incarnation::text=f.completed_keys->0->>'source_incarnation'
WHERE f.op='managed-sync-failure' AND ($1::text[] IS NULL OR s.id=ANY($1::text[]))
AND NOT EXISTS(SELECT 1 FROM op_checkpoints c WHERE c.op='managed-sync' AND c.fingerprint=f.fingerprint)`, [sourceIds ?? null]);
return [...failures, ...orphaned.map(row => row.failure)];
}
export function formatManagedSyncFailure(failure: ManagedSyncFailure): string {
const clean = (value: string) => value.replace(/[\x00-\x1f\x7f-\x9f]/g, ' ').slice(0, 1000);
return `source=${clean(failure.source_id)} path=${clean(failure.path)} code=${clean(failure.code)}: ${clean(failure.message)} request=${failure.request_id ?? '<not-admitted>'} run=${failure.run_id} target=${failure.target ?? '<undiscovered>'}${publicationDetailText(failure.write_error_detail)}${rootCauseText(failure.root_cause, clean)}`;
}
/** #6643: the root-cause clause of a failure line: the request, page and error that caused it, or that none was recorded. */
export function rootCauseText(root: ManagedSyncRootCause | undefined, clean: (value: string) => string = value => value): string {
if (!root) return '';
if (!root.request_id) return ' root_cause=<not recorded: this receipt predates root tracking>';
return ` root_cause=request ${root.request_id}${root.slug ? ` (${clean(root.slug)})` : ''}: ${root.state ?? 'missing'}${root.code ? ` ${clean(root.code)}` : ''}${root.cause ? ` cause=${root.cause}` : ''}${root.message ? ` ${clean(root.message)}` : ''}`;
}
const IDENT = /^[A-Za-z_][A-Za-z0-9_.:-]{0,79}$/;
/** #6513: the fixed identifiers of a public failure detail (`raiser`, `table`) and its closed `reason`, as ` key=value` text; nothing else of it. */
export function publicationDetailText(detail: Record<string, unknown> | undefined): string {
if (!detail) return '';
return (['raiser', 'table', 'reason'] as const).map(key => typeof detail[key] === 'string' && IDENT.test(detail[key] as string) ? ` ${key}=${detail[key]}` : '').join('');
}
/** The retry for one managed failure: same source and recorded cursor-selecting options, or an honest note when they were not recorded. */
export function managedSyncRetryCommand(failure: ManagedSyncFailure, repoPath?: string | null): string {
return checkpointRetryCommand({ sourceId: failure.source_id, processingOptions: failure.processingOptions, syncOptions: failure.syncOptions ?? null, repoPath });
}
export function syncFailureJsonFields(result: { failedFiles?: number; failureCodes?: Array<{ code: string; count: number }>; failures?: ManagedSyncFailure[]; runId?: string; fromCommit?: string | null; toCommit?: string; bankedFiles?: number }): Record<string, unknown> {
return { ...(result.failedFiles !== undefined ? { failed_files: result.failedFiles } : {}),
...(result.failureCodes ? { failure_codes: result.failureCodes } : {}), ...(result.failures ? { failures: result.failures } : {}),
...(result.runId ? { run_id: result.runId, from_commit: result.fromCommit, target_commit: result.toCommit, banked_files: result.bankedFiles, counts_scope: 'run_cumulative' } : {}) };
}