Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* #6317 (B4, I2): does managed sync data move? One reader behind
* `gbrain sources status`, `gbrain sources writer status --json`, the doctor
* check `managed_sync_not_moving`, the `serve` notice and
* `gbrain sources writer movement`, so no surface can call a parked sync
* healthy while another calls it stuck.
*
* Per active managed source the snapshot carries the unfinished cursor
* (`readManagedSyncBacklog`: index, total, held, the drain's last advance),
* the movement watermark (the newest `completed_at` of a committed
* `managed_sync_*` receipt in the current source incarnation, read through
* the partial index `persistence_requests_sync_watermark`, whose predicate
* repeats the read's kind test; never a count, since compaction nulls
* `intent`), the
* head request's claim stamp (`step_since` moving is progress too: a
* multi-wave group commits nothing until its last wave), the newest hold, and
* who could move it: a live drain (the per-source sync lock) or a live
* consumer on the owner host (a renewing claim, this process's own consumer,
* or a live heartbeat row).
*
* The verdict: `nothing_pending` (no unfinished cursor), `parked` (pending,
* but neither a drain nor a consumer with admitted work is live: a cursor left
* between cron runs; informational, never scored), `held` (the newest activity
* is a hold: #6298's containment, not a stall), `moving` (a commit or a step
* advance within `persistence.preparation_ceiling_ms`), `not_moving` (none for
* longer than the ceiling while something live should have produced one),
* `unknown` (#6781: the backlog read failed, so whether work is pending is not
* known; `unknown.reason` is `backlog_read_failed`; never `nothing_pending`).
* `data_moving` is true/false for the live states and null otherwise;
* `not_moving_since` is the last watermark. `judgeMovement` compares two
* snapshots over a window for the standalone command (`moved`,
* `within_allowance`, `held`, `parked`, `not_moving`, `nothing_pending`,
* `unknown`).
* `startMovementWatch` is the serve's unref'd ticker that prints one
* `[gbrain notice managed_sync_not_moving]` when a source flips to not moving
* and one line when it moves again.
*
* #6712 PR 4b (design v3.2 §3.7): a non-company source has a pass row, not a
* cursor, and the backlog reader returns it as an entry with `pass` set; it
* lands in the additive `pass` field (`cursor` stays the cursor's). The pass
* row's last write is progress. Pending means not `nothing_pending`
* (`passHealth`: converged at HEAD and the newest pass ended `synced` or
* `up_to_date`). A pass that keeps stopping (`blocked`, a `not_converged`
* streak, a `settle_mismatch` streak) is `not_moving` whatever else is live:
* every tick may commit pages, yet the source never converges.
*/
import type { BrainEngine } from '../engine.ts';
import { readManagedSyncBacklog, type ManagedSyncBacklog } from './sync-drain.ts';
import { claimStateOf, type ClaimState } from './claim-phase.ts';
import { readPreparationPolicy } from './switches.ts';
import { listHostConsumers } from './consumer-heartbeat.ts';
import type { DrainHolder } from './sync-lease.ts';
export type MovementState = 'moving' | 'held' | 'parked' | 'not_moving' | 'nothing_pending' | 'unknown';
/** #6781: why a source's movement is `unknown`; `error` is the failed read's message, bounded. */
export interface MovementUnknown { reason: 'backlog_read_failed'; error: string }
export interface MovementHead {
request_id: string;
state: string;
phase: ClaimState['phase'] | null;
step: string | null;
step_since: string | null;
step_age_ms: number | null;
waiting_on: ClaimState['waiting_on'] | null;
owner: ClaimState['owner'];
last_sql: ClaimState['last_sql'];
lapsed: boolean;
}
export interface MovementHolds { count: number; stalled: number; fences: number; concurrent: number; last_hold_at: string | null }
export interface SourceMovement {
source_id: string;
sampled_at: string;
movement_state: MovementState;
data_moving: boolean | null;
not_moving_since: string | null;
/** The newest committed managed receipt of this incarnation. */
last_commit_at: string | null;
/** The watermark the verdict used: the newest of a commit, a head step advance and the drain's cursor advance. */
last_progress_at: string | null;
cursor: (Pick<ManagedSyncBacklog, 'index' | 'total' | 'remaining' | 'held' | 'last_progress_at' | 'resume_command' | 'resume_args'>) | null;
/** #6712 PR 4b: the source's pending pass row (the backlog's pass entry), null when it has none or nothing is pending. */
pass: (Pick<ManagedSyncBacklog, 'index' | 'total' | 'remaining' | 'held' | 'last_progress_at' | 'resume_command' | 'resume_args'> & NonNullable<ManagedSyncBacklog['pass']>) | null;
head: MovementHead | null;
holds: MovementHolds;
/** Managed requests admitted on the worktree and not yet settled (queued or running). */
admitted: number;
/** #6712 PR 2 (#6552): when the oldest of those was admitted (null when none), so a queue stranded before its first commit still ages. */
admitted_since: string | null;
/** #6706: `drain_job_id` is the active `sync` Minions job with a live lock that drives this source, when no CLI drain holds the sync lock. */
live: { drain: boolean; drain_pid: number | null; drain_job_id: number | null; consumer: 'claim' | 'local' | 'heartbeat' | null };
ceiling_ms: number;
owner_is_this_host: boolean | null;
writer_status_command: string;
/** Set only when `movement_state` is `unknown`. */
unknown?: MovementUnknown;
}
export const MOVEMENT_WATCH_MS = 60_000;
const iso = (value: unknown): string | null => {
if (value === null || value === undefined) return null;
const ms = value instanceof Date ? value.getTime() : typeof value === 'number' ? value : Date.parse(String(value));
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
};
const newest = (...values: Array<string | null | undefined>): string | null => values.reduce<string | null>((best, value) => value && (!best || value > best) ? value : best, null);
export function writerStatusCommand(sourceId: string): string { return `gbrain sources writer status --source ${sourceId} --json`; }
/** $1 worktree, $2 source incarnation. The kind test matches the `persistence_requests_sync_watermark` predicate text, so the planner can prove the partial index applies. */
export const MOVEMENT_WATERMARK_SQL = `SELECT
(SELECT r.completed_at FROM persistence_requests r WHERE r.worktree_id=$1::uuid AND r.state='committed' AND r.source_incarnation=$2::uuid
AND COALESCE(r.intent->>'kind','') LIKE 'managed_sync_%' ORDER BY r.completed_at DESC LIMIT 1) AS last_commit_at,
(SELECT count(*) FROM persistence_requests r WHERE r.worktree_id=$1::uuid AND r.state IN ('queued','running') AND COALESCE(r.intent->>'kind','') LIKE 'managed_sync_%') AS admitted,
(SELECT min(r.created_at) FROM persistence_requests r WHERE r.worktree_id=$1::uuid AND r.state IN ('queued','running') AND COALESCE(r.intent->>'kind','') LIKE 'managed_sync_%') AS admitted_since`;
interface BindingRow { source_id: string; worktree_id: string; source_incarnation: string; owner_host_id: string | null }
interface WatermarkRow { last_commit_at: unknown; admitted: number | string; admitted_since?: unknown }
interface HeadRow { request_id: string; state: string; claim_phase: unknown; execution_token: string | null; claim_lapsed: boolean | null; publication_started: boolean | null }
interface HoldRow { count: number | string | null; stalled: number | string | null; fences: number | string | null; concurrent: number | string | null; updated_at: unknown }
/**
* #6706: the id of the active `sync` Minions job with a live lock that is draining `sourceId`, or null. Never rejects.
* #6712 PR 1: a job drains under the source's drain lease, so its holder row names the job first; the job-table read
* stays for a job an older gbrain started (it takes no lease). #7596: a holder that stopped renewing the lease is not live (a
* killed job's lock row names it until a takeover); `liveHolder` is the live holder a caller already read that way, or null.
*/
export async function liveSyncJob(engine: BrainEngine, sourceId: string, liveHolder?: DrainHolder | null): Promise<number | null> {
const holder = liveHolder !== undefined ? liveHolder
: await (await import('./sync-lease.ts')).readDrainLease(engine, sourceId).then(lease => lease?.live ? lease.holder : null, () => null);
if (holder?.kind === 'job' && holder.id && /^\d+$/.test(holder.id)) return Number(holder.id);
// The lease's beacon row (sync-lease.ts) is for older binaries; this build reads the lease holder above.
const [row] = await engine.executeRaw<{ id: number | string }>(`SELECT id FROM minion_jobs WHERE name='sync' AND status='active' AND lock_until > now() AND queue <> 'gbrain-drain-lease'
AND (COALESCE(data->>'sourceId', data->>'source_id')=$1 OR (COALESCE(data->>'sourceId', data->>'source_id') IS NULL
AND (data->>'repoPath' IS NULL AND $1='default' OR data->>'repoPath'=(SELECT local_path FROM sources WHERE id=$1)))) ORDER BY started_at LIMIT 1`, [sourceId]).catch(() => []);
return row ? Number(row.id) : null;
}
/** The movement of every active managed source (or of `sourceIds`), one bounded read each; sources without a binding are omitted. */
export async function readSourceMovement(engine: BrainEngine, opts: { sourceIds?: string[]; now?: number } = {}): Promise<SourceMovement[]> {
const now = opts.now ?? Date.now();
const bindings = await engine.executeRaw<BindingRow>(`SELECT b.source_id, b.worktree_id::text AS worktree_id, b.source_incarnation::text AS source_incarnation, w.owner_host_id::text AS owner_host_id
FROM persistence_source_bindings b JOIN sources s ON s.id=b.source_id AND s.incarnation=b.source_incarnation
JOIN persistence_worktrees w ON w.id=b.worktree_id WHERE ($1::text[] IS NULL OR b.source_id=ANY($1::text[])) ORDER BY b.source_id`, [opts.sourceIds ?? null]);
if (!bindings.length) return [];
let unknown: MovementUnknown | null = null;
let backlogRows: ManagedSyncBacklog[] = [];
try { backlogRows = await readManagedSyncBacklog(engine, bindings.map(b => b.source_id)); } catch (error) {
unknown = { reason: 'backlog_read_failed', error: (error instanceof Error ? error.message : String(error)).slice(0, 300) };
}
const backlog = new Map(backlogRows.map(b => [b.source_id, b]));
const policy = await readPreparationPolicy(engine);
const { hostIdFor } = await import('./identity.ts');
const { persistenceConsumerStatus } = await import('./service.ts');
const { liveSyncStatus } = await import('../db-lock.ts');
const host = hostIdFor(engine);
const localConsumer = persistenceConsumerStatus(engine).state === 'open';
let heartbeat: Promise<boolean> | undefined;
const liveHeartbeat = () => heartbeat ??= listHostConsumers(engine, host).then(rows => rows.some(row => (row.mode === 'full' || row.mode === 'promoted') && row.liveness === 'live'), () => false);
const out: SourceMovement[] = [];
for (const binding of bindings) {
const [mark] = await engine.executeRaw<WatermarkRow>(MOVEMENT_WATERMARK_SQL, [binding.worktree_id, binding.source_incarnation]);
const [headRow] = await engine.executeRaw<HeadRow>(`SELECT request_id::text AS request_id, state, claim_phase, execution_token::text AS execution_token,
(claim_expires_at IS NOT NULL AND claim_expires_at < now()) AS claim_lapsed, publication_started
FROM persistence_requests WHERE worktree_id=$1::uuid AND state IN ('queued','running','recovering') ORDER BY (state <> 'running'), sequence LIMIT 1`, [binding.worktree_id]);
const [holdRow] = await engine.executeRaw<HoldRow>(`SELECT (completed_keys->0->>'count')::int AS count, (completed_keys->0->>'stalled')::int AS stalled,
(completed_keys->0->>'fences')::int AS fences, (completed_keys->0->>'concurrent')::int AS concurrent, updated_at
FROM op_checkpoints WHERE op='sync-hold-summary' AND completed_keys->0->>'source_id'=$1 AND completed_keys->0->>'incarnation'=$2 LIMIT 1`, [binding.source_id, binding.source_incarnation]);
const claim = headRow ? claimStateOf({ state: headRow.state, claim_phase: headRow.claim_phase, execution_token: headRow.execution_token, claim_lapsed: headRow.claim_lapsed, publication_started: headRow.publication_started }, now) : null;
const stampSince = (() => {
const raw = typeof headRow?.claim_phase === 'string' ? safeJson(headRow.claim_phase) : headRow?.claim_phase;
const stamp = raw && typeof raw === 'object' ? raw as Record<string, unknown> : null;
return stamp && headRow && stamp.token === headRow.execution_token ? iso(stamp.step_since ?? stamp.since) : null;
})();
const head: MovementHead | null = headRow ? { request_id: headRow.request_id, state: headRow.state, phase: claim?.phase ?? null, step: claim?.step ?? null, step_since: stampSince,
step_age_ms: claim?.step_age_ms ?? null, waiting_on: claim?.waiting_on ?? null, owner: claim?.owner ?? null, last_sql: claim?.last_sql ?? null, lapsed: headRow.claim_lapsed === true } : null;
const drain = await liveSyncStatus(engine, binding.source_id);
const jobDrain = drain ? null : await liveSyncJob(engine, binding.source_id);
const ownerHere = binding.owner_host_id === null ? null : binding.owner_host_id === host;
const claimLive = !!claim && (claim.phase === 'preparing' || claim.phase === 'publishing');
const consumer: SourceMovement['live']['consumer'] = claimLive ? 'claim' : ownerHere !== false && localConsumer ? 'local' : ownerHere !== false && await liveHeartbeat() ? 'heartbeat' : null;
const entry = backlog.get(binding.source_id) ?? null;
const cursor = entry && !entry.pass ? entry : null;
const pass = entry?.pass ? { index: entry.index, total: entry.total, remaining: entry.remaining, held: entry.held, last_progress_at: entry.last_progress_at,
resume_command: entry.resume_command, resume_args: entry.resume_args, ...entry.pass } : null;
const holds: MovementHolds = { count: Number(holdRow?.count ?? 0), stalled: Number(holdRow?.stalled ?? 0), fences: Number(holdRow?.fences ?? 0), concurrent: Number(holdRow?.concurrent ?? 0), last_hold_at: iso(holdRow?.updated_at) };
const lastCommitAt = iso(mark?.last_commit_at);
const admitted = Number(mark?.admitted ?? 0);
const progressAt = newest(lastCommitAt, claimLive ? head?.step_since : null, cursor?.last_progress_at, pass?.last_progress_at);
const sampledAt = new Date(now).toISOString();
const base = { source_id: binding.source_id, sampled_at: sampledAt, last_commit_at: lastCommitAt, last_progress_at: progressAt,
cursor: cursor ? { index: cursor.index, total: cursor.total, remaining: cursor.remaining, held: cursor.held, last_progress_at: cursor.last_progress_at, resume_command: cursor.resume_command, resume_args: cursor.resume_args } : null,
pass, head, holds, admitted, admitted_since: iso(mark?.admitted_since), live: { drain: !!drain || !!jobDrain, drain_pid: drain?.holder_pid ?? null, drain_job_id: jobDrain, consumer }, ceiling_ms: policy.ceilingMs, owner_is_this_host: ownerHere,
writer_status_command: writerStatusCommand(binding.source_id) };
out.push(unknown ? { ...base, movement_state: 'unknown', data_moving: null, not_moving_since: null, unknown } : { ...base, ...verdict(base, now) });
}
return out;
}
function verdict(m: Omit<SourceMovement, 'movement_state' | 'data_moving' | 'not_moving_since'>, now: number): Pick<SourceMovement, 'movement_state' | 'data_moving' | 'not_moving_since'> {
if (!m.cursor && !m.pass) return { movement_state: 'nothing_pending', data_moving: null, not_moving_since: null };
if (m.pass?.stuck) return { movement_state: 'not_moving', data_moving: false, not_moving_since: m.pass.stuck_since };
const driving = m.live.drain || (m.admitted > 0 && m.live.consumer !== null);
if (!driving) return { movement_state: 'parked', data_moving: null, not_moving_since: null };
if (m.holds.last_hold_at && (!m.last_commit_at || m.holds.last_hold_at > m.last_commit_at) && m.last_progress_at && m.holds.last_hold_at >= m.last_progress_at) {
return { movement_state: 'held', data_moving: true, not_moving_since: null };
}
const since = m.last_progress_at ? Date.parse(m.last_progress_at) : NaN;
if (Number.isFinite(since) && now - since > m.ceiling_ms) return { movement_state: 'not_moving', data_moving: false, not_moving_since: m.last_progress_at };
return { movement_state: 'moving', data_moving: true, not_moving_since: null };
}
function safeJson(text: string): unknown { try { return JSON.parse(text); } catch { return null; } }
/** One human line per source for `gbrain sources status`; null for a source with nothing pending. */
export function formatSourceMovement(m: SourceMovement): string | null {
if (m.movement_state === 'nothing_pending') return null;
if (m.movement_state === 'unknown') {
return `${m.source_id}: managed sync movement UNKNOWN: the sync backlog could not be read (${m.unknown?.reason ?? 'backlog_read_failed'}: ${m.unknown?.error ?? 'no detail'}); this is not a report that nothing is pending. Recheck: gbrain sources writer movement ${m.source_id}`;
}
const where = m.head?.step ? ` head at step ${m.head.step}${m.head.waiting_on && m.head.waiting_on !== 'unknown' ? ` (waiting on ${m.head.waiting_on})` : ''}${m.head.owner ? `, ${m.head.owner.kind} pid ${m.head.owner.pid}` : ''};` : '';
const work = m.cursor ?? m.pass!;
if (m.pass?.stuck) return `${m.source_id}: managed sync NOT CONVERGING: every pass since ${m.pass.stuck_since} stopped ${m.pass.stuck}${m.pass.stop_reason ? ` (${m.pass.stop_reason})` : ''}, `
+ `${m.pass.remaining} path(s) unsettled; inspect: gbrain sync status --source ${m.source_id} --json`;
switch (m.movement_state) {
case 'not_moving': return `${m.source_id}: managed sync NOT MOVING since ${m.not_moving_since} (no committed page or step advance for longer than ${Math.round(m.ceiling_ms / 1000)}s while a ${m.live.drain ? 'sync' : 'consumer'} is live);${where} inspect: ${m.writer_status_command}`;
case 'parked': return `${m.source_id}: managed sync parked at ${work.index}/${work.total} (no sync or consumer is live; informational). Resume: ${work.resume_command}`;
case 'held': return `${m.source_id}: managed sync advancing by holds only (${m.holds.count} held, newest ${m.holds.last_hold_at}); the containment is working, not a stall.`;
default: return `${m.source_id}: managed sync moving (last progress ${m.last_progress_at ?? 'unknown'}, ${work.remaining} remaining).`;
}
}
/** The command's verdict over a window: what changed between two snapshots of one source. */
export type MovementVerdict = 'moved' | 'within_allowance' | 'held' | 'parked' | 'not_moving' | 'nothing_pending' | 'unknown';
export function judgeMovement(before: SourceMovement | undefined, after: SourceMovement): MovementVerdict {
if (after.movement_state === 'nothing_pending' || after.movement_state === 'unknown') return after.movement_state;
if (before?.movement_state === 'unknown') return 'unknown';
if (after.pass?.stuck) return 'not_moving';
if (!before) return after.movement_state === 'moving' ? 'moved' : after.movement_state;
if (after.last_commit_at && (!before.last_commit_at || after.last_commit_at > before.last_commit_at)) return 'moved';
if (after.holds.last_hold_at && (!before.holds.last_hold_at || after.holds.last_hold_at > before.holds.last_hold_at)) return 'held';
if (after.movement_state === 'parked') return 'parked';
const stepAdvanced = after.head?.step_since && (!before.head || before.head.request_id !== after.head.request_id || before.head.step_since !== after.head.step_since);
const passAdvanced = after.pass && before.pass && (after.pass.pass_id !== before.pass.pass_id || after.pass.last_progress_at !== before.pass.last_progress_at);
if (stepAdvanced || passAdvanced || (after.cursor && before.cursor && after.cursor.index > before.cursor.index)) return 'within_allowance';
return 'not_moving';
}
export interface MovementWatch { stop(): void; tick(): Promise<void> }
/**
* The serve's ticker: reads movement every `everyMs` and logs one notice when a source flips to `not_moving`
* (and one recovery line when it moves again). Read failures are swallowed, and a source whose backlog read failed
* (`unknown`, #6781) keeps its flip state, so a failed read neither repeats nor clears the notice; the timer never keeps
* the process alive.
*/
export function startMovementWatch(engine: BrainEngine, opts: { log: (line: string) => void; everyMs?: number; setInterval?: typeof setInterval; clearInterval?: typeof clearInterval } ): MovementWatch {
const flipped = new Set<string>();
let inflight = false;
const tick = async () => {
if (inflight) return;
inflight = true;
try {
for (const m of await readSourceMovement(engine)) {
if (m.movement_state === 'unknown') continue;
if (m.movement_state === 'not_moving' && !flipped.has(m.source_id)) {
flipped.add(m.source_id);
opts.log(`[gbrain notice managed_sync_not_moving kind=degraded] ${formatSourceMovement(m)}\nnext: run ${m.writer_status_command} (read-only) and act on the running claim's next step.`);
} else if (m.movement_state !== 'not_moving' && flipped.delete(m.source_id)) {
opts.log(`[gbrain notice managed_sync_not_moving kind=info] ${m.source_id}: managed sync data is moving again (${m.movement_state}).`);
}
}
} catch { /* a diagnostic read never disturbs the serve */ } finally { inflight = false; }
};
const every = opts.everyMs ?? MOVEMENT_WATCH_MS;
const timer = (opts.setInterval ?? setInterval)(() => { void tick(); }, every);
(timer as { unref?: () => void }).unref?.();
return { tick, stop: () => { (opts.clearInterval ?? clearInterval)(timer); } };
}