Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6176: which phase a running write request's claim is in, and how long it
* has been there, so a stall names its phase instead of `cause_unknown`.
*
* The consumer keeps a `ClaimPhaseClock` per claim and every claim renewal
* (claim-lease.ts cadence, 10 s) stores `claimPhaseStamp` in the request's
* `claim_phase` column (v217): the phase, the claim start, the phase start and
* the execution token, so a later claim of the same request never reads an
* earlier claim's ages. Recording rides the renewal: no extra statement per write.
*
* #6278 adds the step: preparers call `enterClaimStep` at each await
* boundary (authority, binding, git, raw hash, origin checks, import screen,
* link resolution, adoption reads), which names the step, what the await
* waits on (`git`, `fs`, `db`, `pool`; `unknown` when nothing observed it)
* and checks the preparation's signal, so cancellation reaches a preparer
* between its queries even when the query itself takes no signal. The stamp
* also names the owner process (kind, pid, build), so writer status can tell
* a sync CLI from a `gbrain serve` without raw SQL. The claim statements
* stamp a single claim at claim time and a group stamps each wave's members
* at dispatch, so a kill before the first renewal still shows a `preparing`
* stamp of the current token (the expired-claim reclaim charges exactly
* those, journal.ts).
*
* `claimStateOf` reads a request row back. A claim whose renewal cannot reach
* its row (it lapsed while still `running`) is inside a transaction that holds
* the row, or its owner is gone; `publication_started` means files were being
* published and the recovery path finishes the request. `claimStall` is the
* running-overdue verdict (`preparation_overdue`) writer status and doctor
* attach as `claim.stall`; the terminal give-up is the `preparation_stalled`
* receipt, so the two never share a name.
*
* #6317 adds the process identity and the last statement: the owner is
* `{kind, pid, nonce, pid_ns, version}` (a per-process nonce, because
* containers sharing one home reuse pids; `isOwnerThisProcess` compares the
* nonce whenever the stamp has one), and `recordClaimSql` keeps the label of
* the last raw statement a preparer issued (first keyword plus table, never
* text or parameters; `boundedReads` records it), stamped as `last_sql` so
* writer status, the drain's stall line and the consumer's log lines
* (`claimTripleText`) name the same step / waiting_on / last_sql triple.
*/
import type { BrainEngine } from '../engine.ts';
import { VERSION } from '../../version.ts';
import { consumerIdentity, consumerProvablyDead, sameProcess } from './consumer-heartbeat.ts';
export type ClaimPhaseName = 'preparing' | 'publishing';
/** #6405: `publication` is the publish phase's own wait (a stamp in `publishing` no longer reads `unknown`); #6288 `publication_deadline` is a publish past `persistence.publication_ceiling_ms`. */
export const WAITING_ON = ['git', 'fs', 'db', 'pool', 'publication', 'publication_deadline', 'unknown'] as const;
export type WaitingOn = typeof WAITING_ON[number];
export interface ClaimPhaseClock {
phase: ClaimPhaseName;
claimedAt: number;
since: number;
/** The preparation step in flight (`enterClaimStep`), null before the first boundary. */
step: string | null;
stepSince: number;
waitingOn: WaitingOn;
/**
* #6278: the preparation's cancellation. `enterClaimStep` checks it at every
* boundary; a preparer passes it to a raw statement only when that statement
* can block on a lock or a pool reservation and is not a shared memo read
* (a signalled statement reserves its own connection and skips the memo).
*/
signal?: AbortSignal;
/**
* #6278: when the preparation's budget runs out (`claimedAt` plus the
* budget), so a read that can wait on a relation lock runs under a
* transaction-local `statement_timeout` of the time left (`boundedReads`).
* Absent with the deadlines switch off.
*/
deadlineAt?: number;
/** #6317: the last raw statement the preparation issued (label only) and when. */
lastSql?: { label: string; at: number };
}
/**
* The process that holds a claim, as the stamp records it. #6317: `nonce` (random per process) and `pid_ns` (the pid
* namespace where readable) tell a reused pid or another container's pid from this process; an older owner stamps neither.
*/
export interface ClaimOwner { kind: string; pid: number; version: string; nonce?: string; pid_ns?: string | null; boot_id?: string | null; start?: string | null }
/** #6317: the step / waiting_on / last_sql triple of a claim, as stamped and as read back. */
export interface ClaimLastSql { label: string; age_ms: number | null }
export function startClaimPhase(now = Date.now(), signal?: AbortSignal, budgetMs?: number): ClaimPhaseClock {
return { phase: 'preparing', claimedAt: now, since: now, step: null, stepSince: now, waitingOn: 'unknown', ...(signal ? { signal } : {}),
...(budgetMs !== undefined ? { deadlineAt: now + budgetMs } : {}) };
}
export function enterClaimPhase(clock: ClaimPhaseClock, phase: ClaimPhaseName, now = Date.now()): void {
if (clock.phase === phase) return;
clock.phase = phase;
clock.since = now;
clock.step = null;
clock.stepSince = now;
clock.waitingOn = phase === 'publishing' ? 'publication' : 'unknown';
}
/**
* A preparer's step boundary: names the step and what its await waits on,
* and throws the preparation's abort reason once the budget has cut it off
* (`signal` defaults to the clock's). `clock` is absent for a preparation
* outside the consumer (the sync's waiver and origin checks), which still
* gets the cancellation check from its own signal.
*/
export function enterClaimStep(clock: ClaimPhaseClock | undefined, step: string, signal: AbortSignal | undefined = clock?.signal, waitingOn: WaitingOn = 'unknown', now = Date.now()): void {
signal?.throwIfAborted();
if (!clock) return;
if (clock.step !== step) { clock.step = step; clock.stepSince = now; }
clock.waitingOn = waitingOn;
}
/**
* #7436: runs a wait on something other than the database (a Git child process) inside the current step, with the
* stamp's `waiting_on` naming it until it ends; the step keeps its name, so a stall reads `step=link_resolution
* waiting_on=git` instead of blaming the last statement. Checks the preparation's cancellation first, like a step boundary.
*/
export async function waitingOnWhile<T>(clock: ClaimPhaseClock | undefined, waitingOn: WaitingOn, run: () => Promise<T>): Promise<T> {
clock?.signal?.throwIfAborted();
if (!clock) return run();
const prior = clock.waitingOn;
clock.waitingOn = waitingOn;
try { return await run(); } finally { if (clock.waitingOn === waitingOn) clock.waitingOn = prior; }
}
/** #6317: the label of a raw statement for the stamp: its first keyword and the first table it names; never text or parameters. */
export function sqlLabel(sql: string): string {
const text = sql.replace(/\s+/g, ' ').trim();
const keyword = /^[A-Za-z]+/.exec(text)?.[0]?.toUpperCase() ?? 'SQL';
const table = /\b(?:FROM|INTO|UPDATE|JOIN)\s+(?:ONLY\s+)?("?[A-Za-z_][\w.]*"?)/i.exec(text)?.[1]?.replaceAll('"', '');
return table ? `${keyword} ${table}` : keyword;
}
/** #6317: records the statement a preparer is about to issue on its clock (`boundedReads` calls it at every raw read). */
export function recordClaimSql(clock: ClaimPhaseClock | undefined, sql: string, now = Date.now()): void {
if (clock) clock.lastSql = { label: sqlLabel(sql), at: now };
}
/** #6317: the ` step=… waiting_on=… last_sql=…` text every `[persistence]` cut-off and hold line carries (C3). */
export function claimTripleText(clock: Pick<ClaimPhaseClock, 'step' | 'waitingOn' | 'lastSql'> | undefined, now = Date.now()): string {
if (!clock) return ' step=none waiting_on=unknown last_sql=none';
const last = clock.lastSql ? `${clock.lastSql.label.replaceAll(' ', '_')}@${Math.round((now - clock.lastSql.at) / 1000)}s` : 'none';
return ` step=${clock.step ?? 'none'} waiting_on=${clock.waitingOn} last_sql=${last}`;
}
/** #6317: whether a stamped owner is this process: the pid, and the nonce when the stamp carries one (a reused pid is another process). */
export function isOwnerThisProcess(owner: Pick<ClaimOwner, 'pid' | 'nonce'> | null | undefined): boolean {
return claimOwnerIsThisProcess(owner);
}
export const OWNER_COMMANDS: ReadonlySet<string> = new Set(['sync', 'serve', 'jobs', 'autopilot', 'mcp', 'dream', 'cycle', 'sources', 'migrate-graduation', 'put', 'import']);
let ownerOverride: ClaimOwner | undefined;
/** The gbrain command this process runs (`cli` when none is recognisable). */
export function claimOwnerKind(): string {
if (ownerOverride) return ownerOverride.kind;
const command = process.argv.slice(2).find(arg => !arg.startsWith('-'));
return command && OWNER_COMMANDS.has(command) ? command : 'cli';
}
let oneShotCommand = false;
/**
* GBRA-156: set by the CLI's shared-operation dispatch, which runs one operation and exits. The consumer such a process
* starts (service.ts) leaves its housekeeping off the write's path and rations it per host (housekeeping-turn.ts), and its
* pool opens a second connection beside the first (db.ts).
*/
export function markOneShotCommand(on = true): void { oneShotCommand = on; }
export function isOneShotCommand(): boolean { return oneShotCommand; }
/** The owning process for the stamp: the gbrain command this process runs (`cli` when none is recognisable), its pid, nonce, pid namespace and build. */
export function claimOwner(): ClaimOwner {
if (ownerOverride) return ownerOverride;
const { pid, nonce, pid_ns, boot_id, start } = consumerIdentity();
return { kind: claimOwnerKind(), pid, version: VERSION, nonce, pid_ns, boot_id, start };
}
/** Whether a stamped owner is this process: the pid and, when the stamp carries one, the nonce (#6317: `owner_pid === process.pid` alone is not enough). */
export function claimOwnerIsThisProcess(owner: Pick<ClaimOwner, 'pid' | 'nonce'> | null | undefined): boolean {
return sameProcess(owner);
}
/**
* #6650: whether the claim stamped `raw` (a `claim_phase` value) was held by a process this one can prove dead
* (`consumerProvablyDead`: same boot and pid namespace, its pid gone or reused). A stamp without the markers, from
* an older owner, another host or container, is never provably dead.
*/
export function claimOwnerProvablyDead(raw: unknown): boolean {
const stamp = (typeof raw === 'string' ? safeJson(raw) : raw) as { owner?: Record<string, unknown> } | null;
const owner = stamp?.owner;
if (!owner || typeof owner.pid !== 'number') return false;
const text = (key: string) => typeof owner[key] === 'string' ? owner[key] as string : null;
return consumerProvablyDead({ pid: owner.pid, nonce: text('nonce'), pid_ns: text('pid_ns'), boot_id: text('boot_id'), start: text('start') });
}
/** Test seam. */
export function setClaimOwnerForTest(owner: ClaimOwner | undefined): void { ownerOverride = owner; }
/** The `claim_phase` jsonb a renewal (or a claim, or a group's dispatch mark) stores for the claim holding `token`. */
export function claimPhaseStamp(clock: ClaimPhaseClock, token: string | null, owner: ClaimOwner = claimOwner()): string {
return JSON.stringify({ phase: clock.phase, claimed_at: new Date(clock.claimedAt).toISOString(), since: new Date(clock.since).toISOString(), token,
step: clock.step, step_since: new Date(clock.stepSince).toISOString(), waiting_on: clock.waitingOn, owner,
last_sql: clock.lastSql ? { label: clock.lastSql.label, at: new Date(clock.lastSql.at).toISOString() } : null });
}
/**
* #6278: the SQL that charges a reclaimed expired claim to its request's
* `preparation_attempts`: only a claim whose owner stamped it `preparing`
* under the token being reclaimed (a kill mid-preparation), never one that
* was publishing, undispatched (no stamp of its token) or stamped by an
* earlier claim.
*/
export const EXPIRED_PREPARING_CHARGE_SQL = (r: string) =>
`CASE WHEN ${r}.claim_phase->>'phase'='preparing' AND ${r}.claim_phase->>'token'=${r}.execution_token::text THEN 1 ELSE 0 END`;
/** `persistence.max_claim_ms`: how long a write may hold its claim before doctor reports it as stalled. */
export const MAX_CLAIM_CONFIG_KEY = 'persistence.max_claim_ms';
export const MAX_CLAIM_DEFAULT_MS = 600_000;
const MAX_CLAIM_MIN_MS = 60_000;
const MAX_CLAIM_MAX_MS = 86_400_000;
export function parseMaxClaimMs(raw: string | null | undefined): number | null {
const text = raw?.trim() ?? '';
if (!/^\d+$/.test(text)) return null;
const n = Number(text);
return n >= MAX_CLAIM_MIN_MS && n <= MAX_CLAIM_MAX_MS ? n : null;
}
/** `config set` validation; the refusal text, or null when the value is valid (or the key is another one). */
export function validateMaxClaimConfigValue(key: string, value: string): string | null {
if (key !== MAX_CLAIM_CONFIG_KEY || parseMaxClaimMs(value) !== null) return null;
return `invalid_params: ${MAX_CLAIM_CONFIG_KEY} must be a whole number of milliseconds from ${MAX_CLAIM_MIN_MS} to ${MAX_CLAIM_MAX_MS} `
+ `(default ${MAX_CLAIM_DEFAULT_MS}, 10 minutes; for example gbrain config set ${MAX_CLAIM_CONFIG_KEY} 900000) (got '${value}'). Nothing was written.`;
}
export async function readMaxClaimMs(engine: Pick<BrainEngine, 'getConfig'>): Promise<number> {
return parseMaxClaimMs(await engine.getConfig(MAX_CLAIM_CONFIG_KEY).catch(() => null)) ?? MAX_CLAIM_DEFAULT_MS;
}
/**
* Where a running claim is: `preparing` or `publishing` (recorded by a live
* renewal), `publication_transaction` (the claim lapsed while running and
* nothing was published yet), `file_publication` (files were being published)
* or `unrecorded` (no stamp of this claim yet, or an owner that predates v217).
*/
export type ClaimStatePhase = ClaimPhaseName | 'publication_transaction' | 'file_publication' | 'unrecorded';
export interface ClaimState {
phase: ClaimStatePhase;
claim_age_ms: number | null;
phase_age_ms: number | null;
/** The preparation step the owner last recorded; null before the first boundary or on a stamp from an older owner. */
step: string | null;
step_age_ms: number | null;
/** What the recorded step waits on; `unknown` when nothing observed it (older stamps included), never inferred. */
waiting_on: WaitingOn;
/** The process holding the claim, when its stamp recorded one. */
owner: ClaimOwner | null;
/** #6317: the last raw statement the owner's preparation issued (label and age), null when none was recorded. */
last_sql: ClaimLastSql | null;
/** Whether the same request continues without anyone acting (after its lease lapses or its transaction ends). */
resumes_on_its_own: boolean;
/** #9227: present when the owner provably exited: no process renews the claim and the next write to the brain releases it. */
owner_exited?: true;
why: string;
}
export interface ClaimRow {
state: string;
claim_phase?: unknown;
execution_token?: string | null;
/** `claim_expires_at < now()`, read on the database clock. */
claim_lapsed?: boolean | null;
publication_started?: boolean | null;
/** #9227: last written before `abandonedClaimCutoff` (PGLite, read by a process that holds the brain); the owner has exited. */
abandoned?: boolean | null;
/** #9227: also prove the stamped owner dead the way the consumer's dead-owner scan does (`claimOwnerProvablyDead`). */
probe_owner?: boolean;
}
const age = (at: unknown, now: number): number | null => {
const ms = typeof at === 'string' || at instanceof Date ? new Date(at).getTime() : NaN;
return Number.isFinite(ms) ? Math.max(0, Math.floor(now - ms)) : null;
};
function stampOwner(raw: unknown): ClaimOwner | null {
const owner = raw && typeof raw === 'object' ? raw as Record<string, unknown> : null;
if (!owner || typeof owner.kind !== 'string' || typeof owner.pid !== 'number' || typeof owner.version !== 'string') return null;
return { kind: owner.kind, pid: owner.pid, version: owner.version, ...(typeof owner.nonce === 'string' ? { nonce: owner.nonce } : {}),
...(typeof owner.pid_ns === 'string' || owner.pid_ns === null ? { pid_ns: owner.pid_ns } : {}) };
}
/** #6317: the stamp's `last_sql` read back with its age; null when the stamp has none (an owner older than #6317 included). */
export function stampLastSql(raw: unknown, now: number): ClaimLastSql | null {
const last = raw && typeof raw === 'object' ? raw as Record<string, unknown> : null;
return last && typeof last.label === 'string' ? { label: last.label, age_ms: age(last.at, now) } : null;
}
export function claimStateOf(row: ClaimRow, now = Date.now()): ClaimState | null {
if (row.state !== 'running') return null;
const raw = typeof row.claim_phase === 'string' ? safeJson(row.claim_phase) : row.claim_phase;
const stamp = raw && typeof raw === 'object' ? raw as Record<string, unknown> : null;
const own = !!stamp && !!row.execution_token && stamp.token === row.execution_token;
const claimAge = own ? age(stamp!.claimed_at, now) : null;
const step = own && typeof stamp!.step === 'string' ? stamp!.step : null;
const detail = { step, step_age_ms: step ? age(stamp!.step_since, now) : null,
waiting_on: own && (WAITING_ON as readonly unknown[]).includes(stamp!.waiting_on) ? stamp!.waiting_on as WaitingOn : 'unknown' as const,
owner: own ? stampOwner(stamp!.owner) : null, last_sql: own ? stampLastSql(stamp!.last_sql, now) : null };
if (row.publication_started) return { phase: 'file_publication', claim_age_ms: claimAge, phase_age_ms: null, ...detail, resumes_on_its_own: false,
why: 'Its files were being published; the publication recovery path restores or finishes them when the owning process scans its roots again (after a restart if it hung).' };
if (row.abandoned === true || row.probe_owner && own && claimOwnerProvablyDead(stamp)) {
const recorded = own && (stamp!.phase === 'preparing' || stamp!.phase === 'publishing') ? stamp!.phase as 'preparing' | 'publishing' : 'unrecorded';
return { phase: recorded, claim_age_ms: claimAge, phase_age_ms: recorded === 'unrecorded' ? null : age(stamp!.since, now), ...detail, resumes_on_its_own: true, owner_exited: true,
why: `Its owner process has exited (${row.abandoned === true ? 'it last wrote the claim before this process opened the brain, which admits one process at a time' : 'same machine boot, its process is gone'}) and nothing renews the claim. The next write from this machine (or a gbrain process already running here, within seconds) releases it and runs the request again; nothing else is needed.` };
}
if (row.claim_lapsed) return { phase: 'publication_transaction', claim_age_ms: claimAge, phase_age_ms: null, ...detail, resumes_on_its_own: true,
why: 'Its claim lapsed while it was still running and nothing was published: a transaction holds the request row (renewals cannot reach it) or its owner is gone. It is requeued once that transaction ends or the owner process exits.' };
// #6650: an undispatched group follower's stamp names its token and owner but no phase yet.
if (!own || (stamp!.phase !== 'preparing' && stamp!.phase !== 'publishing')) return { phase: 'unrecorded', claim_age_ms: claimAge, phase_age_ms: null, ...detail, resumes_on_its_own: false,
why: 'The owner has not recorded this claim\'s phase (a claim younger than one renewal, a group member its owner has not started yet, or an owner that predates phase recording).' };
const phase = stamp!.phase === 'publishing' ? 'publishing' : 'preparing';
return { phase, claim_age_ms: claimAge, phase_age_ms: age(stamp!.since, now), ...detail, resumes_on_its_own: false,
why: `Its owner is still renewing the claim while it is ${phase === 'preparing' ? 'preparing the write' : 'publishing the write'}${step ? ` (step ${step}, waiting on ${detail.waiting_on})` : ''}, so the lease never lapses and nothing else on its root runs until that work settles or the owner restarts.` };
}
function safeJson(text: string): unknown {
try { return JSON.parse(text); } catch { return null; }
}
/** The running-overdue verdict: a `preparing` claim older than its budget. */
export interface ClaimStall { reason: 'preparation_overdue'; step: string | null; step_age_ms: number | null; waiting_on: WaitingOn; budget_ms: number }
export function claimStall(claim: ClaimState | null, budgetMs: number): ClaimStall | null {
if (!claim || claim.owner_exited || claim.phase !== 'preparing' || claim.phase_age_ms === null || claim.phase_age_ms < budgetMs) return null;
return { reason: 'preparation_overdue', step: claim.step, step_age_ms: claim.step_age_ms, waiting_on: claim.waiting_on, budget_ms: budgetMs };
}