Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/** `gbrain jobs stats` (dispatched by runJobs in src/commands/jobs.ts). */
import { hasFlag, parseFlag, type JobsCommandContext } from './shared.ts';
import { formatNice, getEffectiveNiceness } from '../../core/minions/niceness.ts';
import { DREAM_INLINE_PRIVATE_QUEUE_PREFIX, deriveWedgeSignal } from '../../core/minions/queue.ts';
import type { QueueFamilyStats } from '../../core/minions/queue-families.ts';
export async function runJobsStats({ args, engine, queue }: JobsCommandContext): Promise<void> {
try { await queue.ensureSchema(); }
catch (e) { console.error(e instanceof Error ? e.message : String(e)); process.exit(1); }
const statsQueue = parseFlag(args, '--queue') ?? 'default';
const stats = await queue.getStats({ queue: statsQueue });
// Divergence detection: intake (created in window) vs USEFUL drain
// (drained_completed — cancellations are outflow, not work; a naive
// combined drain self-inflates while the TTL sweep shreds backlog).
// Same env-threshold pattern as the wedge line below.
const divergenceRatio = (() => {
const raw = Number(process.env.GBRAIN_QUEUE_DIVERGENCE_RATIO ?? '');
return Number.isFinite(raw) && raw > 0 ? raw : 2;
})();
const divergenceMinWaiting = (() => {
const raw = parseInt(process.env.GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING ?? '', 10);
return Number.isFinite(raw) && raw > 0 ? raw : 50;
})();
const divergent = stats.by_type.filter(t =>
t.waiting_now > divergenceMinWaiting &&
t.total > divergenceRatio * Math.max(t.drained_completed, 1));
// Waiting-TTL cancellations in the window (admission sweep visibility —
// derived from the reason prefix cancelJobs writes; no extra storage).
let ttlCancelled: Array<{ name: string; count: number }> = [];
try {
const { TTL_REASON_PREFIX } = await import('../../core/minions/admission.ts');
const ttlRows = await engine.executeRaw<{ name: string; count: string }>(
`SELECT name, count(*)::text AS count FROM minion_jobs
WHERE status = 'cancelled' AND error_text LIKE $1
AND finished_at > now() - interval '24 hours'
GROUP BY name ORDER BY count(*) DESC`,
[`${TTL_REASON_PREFIX}%`],
);
ttlCancelled = ttlRows.map(r => ({ name: r.name, count: parseInt(r.count, 10) }));
} catch { /* best-effort */ }
// Job names originate from the MCP-exposed submit surface — strip
// control/ANSI bytes + cap before echoing into the terminal screams
// (same hygiene as frontmatter-derived type names). Names embedded in
// COPY-PASTEABLE command hints get the stricter safeConfigSegment gate:
// display-sanitize keeps shell metacharacters.
const { sanitizeTypeForDisplay: sanitizeName } = await import('../../core/schema-pack/type-usage.ts');
const { safeConfigSegment } = await import('../../core/minions/admission.ts');
const { queueWorkerAlive } = await import('../../core/minions/no-worker.ts');
const signal = deriveWedgeSignal(stats.wedge, { workerAlive: queueWorkerAlive(stats.wedge.queue) });
if (hasFlag(args, '--json')) {
console.log(JSON.stringify({
queue: statsQueue,
...stats,
wedged: signal.wedged,
no_worker: signal.no_worker,
divergent: divergent.map(t => ({
name: t.name,
intake_24h: t.total,
drained_completed_24h: t.drained_completed,
waiting_now: t.waiting_now,
oldest_waiting_minutes: t.oldest_waiting_minutes,
})),
ttl_cancelled_24h: ttlCancelled,
}, null, 2));
return;
}
console.log('Job Stats (last 24h):');
if (stats.by_type.length > 0) {
console.log(` ${'Type'.padEnd(14)} ${'Total'.padEnd(7)} ${'Done'.padEnd(7)} ${'Failed'.padEnd(8)} ${'Dead'.padEnd(6)} ${'Drained'.padEnd(9)} ${'Waiting'.padEnd(9)} Avg Time`);
for (const t of stats.by_type) {
const avgTime = t.avg_duration_ms != null ? `${(t.avg_duration_ms / 1000).toFixed(1)}s` : '—';
// Drained = terminal outflow in-window, completed-first with the
// rest bracketed so TTL-cancel storms can't masquerade as work.
const drained = `${t.drained_completed}${(t.drained_failed + t.drained_dead + t.drained_cancelled) > 0 ? `(+${t.drained_failed + t.drained_dead + t.drained_cancelled})` : ''}`;
console.log(` ${sanitizeName(t.name).padEnd(14)} ${String(t.total).padEnd(7)} ${String(t.completed).padEnd(7)} ${String(t.failed).padEnd(8)} ${String(t.dead).padEnd(6)} ${drained.padEnd(9)} ${String(t.waiting_now).padEnd(9)} ${avgTime}`);
}
console.log(` (Drained = completed in-window, +N = failed/dead/cancelled outflow; Waiting = now, all queues)`);
} else {
console.log(' No jobs in the last 24 hours.');
}
console.log(`\n Queue health: ${stats.queue_health.waiting} waiting, ${stats.queue_health.active} active, ${stats.queue_health.stalled} stalled`);
printQueueFamilies(stats.by_queue, sanitizeName);
// DIVERGENT-queue scream: intake structurally exceeds useful drain and a
// real backlog is sitting there. This is the default-on protection layer
// (quota ships config-only), so it must carry the opt-in hint.
for (const t of divergent) {
const perDay = t.drained_completed; // window is 24h
const etaDays = perDay > 0 ? Math.round(t.waiting_now / perDay) : null;
const eta = etaDays != null ? `~${etaDays}d backlog at current drain` : 'backlog never drains at current rate';
const ttl = ttlCancelled.find(c => c.name === t.name);
const ttlNote = ttl ? ` Waiting-TTL is cancelling ~${ttl.count}/day of it.` : '';
console.log(
`\n ⚠ DIVERGENT QUEUE type '${sanitizeName(t.name)}': intake ${t.total}/24h vs ${t.drained_completed} completed/24h, ` +
`${t.waiting_now} waiting (${eta}).${ttlNote}\n` +
` Reduce intake, raise drain, or cap admission:\n` +
` gbrain config set minions.quota_max_waiting.${safeConfigSegment(t.name) ?? '<job-name>'} <n>`,
);
}
if (ttlCancelled.length > 0) {
const parts = ttlCancelled.map(c => `${sanitizeName(c.name)}: ${c.count}`).join(', ');
console.log(
`\n ⚠ Waiting-TTL cancelled ${ttlCancelled.reduce((a, c) => a + c.count, 0)} job(s) in the last 24h (${parts}).\n` +
` These waited past their TTL without ever being claimed. Tune:\n` +
` gbrain config set minions.ttl_waiting_hours.<name> <hours|0>`,
);
}
// Scheduling priority (niceness, issue #1815). Best-effort: measures live
// workers from the registry + the supervisor (if running) — silently skips
// when nothing is reniced/running, so default stats output stays clean.
try {
const { readWorkers } = await import('../../core/minions/worker-registry.ts');
const { readSupervisorPid } = await import('../../core/minions/supervisor-pid.ts');
const { DEFAULT_PID_FILE } = await import('../../core/minions/supervisor.ts');
const liveWorkers = readWorkers();
const sup = readSupervisorPid(DEFAULT_PID_FILE);
const supNice = sup.running && sup.pid !== null ? getEffectiveNiceness(sup.pid) : null;
if (liveWorkers.length > 0 || supNice !== null) {
console.log(`\n Scheduling priority (nice):`);
if (supNice !== null) console.log(` supervisor (pid ${sup.pid}): ${formatNice(supNice)}`);
for (const w of liveWorkers) {
const diverged = w.nice_requested !== null && w.nice_now !== null && w.nice_requested !== w.nice_now
? ` ⚠ requested ${formatNice(w.nice_requested)}, not applied` : '';
console.log(` worker (pid ${w.pid}, queue ${w.queue}): ${w.nice_now !== null ? formatNice(w.nice_now) : '?'}${diverged}`);
}
}
} catch {
// Registry/import failure is best-effort; skip silently.
}
// issue #1801 — wedged-queue signature (queue-scoped): a worker is alive
// but claiming nothing while work waits. `active_healthy` (live-lock only)
// means an expired-lock active row doesn't mask it. Loud line so the
// operator/agent catches a silent halt in `jobs stats`, not 15h later.
{
const w = stats.wedge;
const mins = w.minutes_since_completion;
// Shared derivation (queue.ts deriveWedgeSignal) so this line, the
// doctor wedged_queue check, and the get_job_stats op agree (#1801).
const { wedged, no_worker, wedge_threshold_minutes: wedgeMins, private_queue } = signal;
// Parent-owned dream-inline queue: no shared worker can EVER claim it,
// so the supervisor-restart advice below would be a dead end (the
// incident bug class). Gate the ABANDONED line on the SAME classifier
// recovery uses — a healthy mid-drain queue (active_healthy 0 in a
// claim gap) classifies live and must not scream.
const privateVerdict = private_queue && w.active_healthy === 0 && w.waiting > 0
? await queue.classifyPrivateQueueForRecovery(w.queue)
: null;
if (privateVerdict === 'orphan' || privateVerdict === 'unowned') {
const since = mins === null ? 'no completions on record' : `${mins}m since last completion`;
console.log(
`\n ⚠ ABANDONED PRIVATE QUEUE '${w.queue}': ${w.waiting} waiting, 0 active (live-lock), ${since}.\n` +
` This dream-inline queue is parent-owned; restarting a worker cannot consume it.\n` +
(privateVerdict === 'orphan'
? ` Auto-recovery cancels it at the next worker spawn or dream-cycle start.`
: ` Legacy unowned queue: preview \`gbrain dream retriage --help\` before manual cancellation.`),
);
} else if (no_worker) {
await printNoWorker(engine.kind, w.queue, w.waiting);
} else if (wedged) {
const since = mins === null ? 'no completions on record' : `${mins}m since last completion`;
console.log(
`\n ⚠ WEDGED QUEUE '${w.queue}': ${w.waiting} waiting, 0 active (live-lock), ${since}.\n` +
` A worker may be alive but stuck (dead DB pool / stuck handler). Fix:\n` +
` gbrain jobs supervisor stop && gbrain jobs supervisor start # rebuild a fresh pool\n` +
` gbrain jobs retry <id> # for dead-lettered jobs`,
);
}
// Backpressure visibility: maxPending suppression keeps `waiting` at 0
// while a job is in flight, which silences the waiting>0 wedge line
// above — the exact operator-confusion cost of the duplicate-cycle
// incident. Surface the last 24h of coalesce events (per name, this
// queue) from the backpressure audit JSONL, plus a hint naming the
// in-flight job when a name shows suppression with zero waiting rows
// and a stale live-lock active. Best-effort: unreadable audit files
// simply omit the line.
try {
const { readRecentCoalesceCounts } = await import('../../core/minions/backpressure-audit.ts');
const coalesceCounts = readRecentCoalesceCounts({ queue: statsQueue, windowMs: 24 * 3600_000 });
if (coalesceCounts.size > 0) {
// Sort once, reuse for the summary AND the hint slice — slicing
// insertion order would let low-volume early-in-file names crowd
// out the highest-volume (most likely wedged) ones the summary
// line just highlighted.
const sortedCoalesces = [...coalesceCounts.entries()]
.sort((a, b) => b[1].count - a[1].count);
const parts = sortedCoalesces.map(([name, s]) => `${name}: ${s.count}`);
console.log(`\n Backpressure (24h): submissions coalesced onto in-flight jobs — ${parts.join(', ')}`);
// Hint loop is bounded: names come from the 24h audit window
// (normally a handful), capped defensively — this is an
// operator-invoked diagnostic, not a hot path. Each hint is
// driven by the LATEST coalesce target for the name (the audit's
// returned_job_id), scoped to that job's source — a name-wide
// aggregate would let source A's waiting row mask source B's
// wedge, or name A's job for B's coalesce (multi-source brains).
const hints = sortedCoalesces.slice(0, 10);
for (const [name, summary] of hints) {
if (summary.last_returned_job_id == null) continue;
// The target CTE re-checks name+queue: the audit dir is shared
// across brains in one GBRAIN_HOME, so an id from another
// brain's audit trail must fail the match here rather than
// name an unrelated job as the suppressor.
const rows = await engine.executeRaw<{ waiting: string; live_id: string | null; age_min: string | null }>(
`WITH target AS (
SELECT id, started_at, status, lock_until,
COALESCE(data->>'sourceId', data->>'source_id') AS scope
FROM minion_jobs WHERE id = $3 AND name = $1 AND queue = $2
)
SELECT (SELECT count(*)::text FROM minion_jobs m, target t
WHERE m.name = $1 AND m.queue = $2 AND m.status = 'waiting'
AND COALESCE(m.data->>'sourceId', m.data->>'source_id') IS NOT DISTINCT FROM t.scope) AS waiting,
(SELECT id::text FROM target WHERE status = 'active' AND lock_until > now()) AS live_id,
(SELECT floor(EXTRACT(EPOCH FROM (now() - started_at)) / 60)::text FROM target
WHERE status = 'active' AND lock_until > now()) AS age_min`,
[name, statsQueue, summary.last_returned_job_id],
);
const r = rows[0];
const ageMin = r?.age_min != null ? parseInt(r.age_min, 10) : null;
if (r && parseInt(r.waiting ?? '0', 10) === 0 && r.live_id != null && ageMin != null && ageMin > wedgeMins) {
console.log(
` ${name}: dispatch suppressed by in-flight job #${r.live_id} (age ${ageMin}m) — check \`gbrain jobs get ${r.live_id}\``,
);
}
}
}
} catch {
// Audit read is advisory; never break stats.
}
}
// v0.41 Bug 2 / Eng D8 — surface lease pressure to the operator.
// Reads minion_lease_pressure_log windowed at 1h. Best-effort: pre-v93
// brains (no table) silently skip; the queue_health line above is the
// operator's primary signal in that case.
try {
const lpRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_lease_pressure_log
WHERE bounced_at > now() - interval '1 hour'`,
);
const lpCount = parseInt(lpRows[0]?.count ?? '0', 10);
if (lpCount > 0) {
// Also surface whether any of those bounces stalled forward progress.
// Bounces with rising completed counts = healthy backpressure; bounces
// with zero completes = real blocker (matches doctor's subagent_health).
const completedRows = await engine.executeRaw<{ count: string }>(
`SELECT count(*)::text AS count FROM minion_jobs
WHERE finished_at > now() - interval '1 hour'
AND status = 'completed' AND name = 'subagent'`,
).catch(() => [{ count: '0' }]);
const completed = parseInt(completedRows[0]?.count ?? '0', 10);
const tag = completed > 0
? `(${completed} subagent job${completed === 1 ? '' : 's'} completed, throughput healthy)`
: `(no subagent jobs completed — cap may be too tight; \`export GBRAIN_ANTHROPIC_MAX_INFLIGHT=64\`)`;
console.log(` Lease pressure (1h): ${lpCount} bounce${lpCount === 1 ? '' : 's'} ${tag}`);
} else {
console.log(` Lease pressure (1h): 0 bounces`);
}
} catch {
// Pre-v93 brain — no table. Silent skip.
}
// v0.41 D3 — error clustering. Optional via --cluster-errors flag so
// operators only see the breakdown when triaging a fail-heavy batch
// (default stats output stays scannable). Pulls last 24h of dead +
// failed jobs, classifies by error-classify.ts buckets, sorts by
// count, surfaces top 5 with paste-ready retry hints.
if (hasFlag(args, '--cluster-errors')) {
try {
const { clusterErrors } = await import('../../core/minions/error-classify.ts');
const errRows = await engine.executeRaw<{ id: number; last_error: string | null }>(
`SELECT id, error_text AS last_error FROM minion_jobs
WHERE status IN ('dead', 'failed')
AND updated_at > now() - interval '24 hours'`,
);
if (errRows.length === 0) {
console.log(`\n Error clusters (24h): no dead/failed jobs`);
} else {
const clusters = clusterErrors(errRows);
console.log(`\n Error clusters (24h):`);
for (const c of clusters.slice(0, 5)) {
const sample = c.sample_ids.length > 0
? ` (e.g. \`gbrain jobs get ${c.sample_ids[0]}\`)` : '';
console.log(` ${String(c.count).padStart(4)} × ${c.cluster.padEnd(22)}${sample}`);
}
if (clusters.length > 5) {
console.log(` + ${clusters.length - 5} more cluster${clusters.length - 5 === 1 ? '' : 's'}`);
}
}
} catch (e) {
// error-classify import or SQL fail. Don't block stats output.
if (process.env.GBRAIN_DEBUG === '1') {
console.error(`[jobs stats] cluster-errors skipped: ${e instanceof Error ? e.message : String(e)}`);
}
}
}
}
/** Queue honesty (agent-first operator wave E5): waiting work with no worker is not a wedge; name the fix. */
async function printNoWorker(engineKind: string, queue: string, waiting: number): Promise<void> {
const { runWaitingJobsFix } = await import('../../core/minions/no-worker.ts');
const { shellQuote } = await import('../../core/agent-output.ts');
const fix = runWaitingJobsFix(engineKind, queue);
console.log(
`\n ⚠ NO WORKER for queue '${queue}': ${waiting} waiting and no worker is running` +
`${engineKind === 'pglite' ? ' (PGLite has no background worker)' : ''}; nothing is stuck, nothing is running them.\n` +
` Fix: ${shellQuote(fix.argv ?? [])} # ${fix.why}`,
);
}
/** #7241: one line per queue; the per-run dream-inline-* queues share one line (`--queue <name>` scopes the wedge lines to one). */
function printQueueFamilies(families: QueueFamilyStats[], sanitizeName: (name: string) => string): void {
for (const q of families) {
const label = `${q.queue}-` === DREAM_INLINE_PRIVATE_QUEUE_PREFIX ? `${q.queue} (${q.queues} queue${q.queues === 1 ? '' : 's'})` : sanitizeName(q.queue);
console.log(` ${label.padEnd(26)} ${q.waiting_now} waiting, ${q.active_now} active, ${q.total} total, ${q.completed} done, ${q.failed} failed, ${q.dead} dead`);
}
}