Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.42 — Per-day extract-event rollup writer (best-effort cache).
*
* Sister module to receipt-writer.ts. Where receipts are first-class
* brain pages (long-term audit trail), the rollup table is a fast-read
* cache the doctor `extract_health` check reads to keep latency under
* 100ms regardless of audit volume.
*
* F-OUT-19 dual-write posture:
* - JSONL audit (~/.gbrain/audit/...) is the SOURCE OF TRUTH
* (forensic, append-only, crash-safe).
* - This DB rollup is BEST-EFFORT. Failures bump a counter
* (rollup_write_failures) inside the table itself and stderr-warn;
* they NEVER fail the parent extraction operation.
* - When persistent failures accumulate (>10/hr per future doctor
* rule), an auto-rebuild from JSONL self-heals the cache.
*
* Schema (migration v106):
* extract_rollup_7d (kind, source_id, day, cost_usd, halt_count,
* eval_fail_count, eval_pass_count,
* round_completed_count, rollup_write_failures,
* updated_at, PK(kind, source_id, day))
*
* Concurrency: PostgreSQL INSERT ... ON CONFLICT DO UPDATE is
* concurrency-safe for the per-(kind, source_id, day) PK. Multiple
* parallel extractions on the same day land cleanly without a lock
* (UPSERT is atomic per row).
*/
import type { BrainEngine } from '../engine.ts';
import { isMissingSchemaObject } from '../utils.ts';
/**
* #5495: why a round halted, as a closed vocabulary stored per reason in
* `extract_rollup_7d.halt_reasons` (migration v239) next to `halt_count`.
* No provider text and no paths ever enter the table: a caller passes one of
* these, or `true` for a halt it cannot classify, which reads back as
* `unknown` (`halt_count - sum(halt_reasons)`, computed on read, so rows
* written before the column and by older writers need no backfill).
*/
export const HALT_REASONS = Object.freeze([
'provider_auth',
'provider_billing',
'provider_rate_limit',
'provider_model_not_found',
'no_pricing',
'integrity_guard',
'item_error',
] as const);
export type HaltReason = (typeof HALT_REASONS)[number];
/** The read-side bucket for halts no reason was recorded for; never written. */
export const UNKNOWN_HALT_REASON = 'unknown';
/** The provider error classes a phase's global LLM halt tracker reports, as halt reasons. */
export function providerHaltReason(cls: 'auth' | 'billing' | 'rate_limit' | 'model_not_found' | null | undefined): HaltReason | null {
switch (cls) {
case 'auth': return 'provider_auth';
case 'billing': return 'provider_billing';
case 'rate_limit': return 'provider_rate_limit';
case 'model_not_found': return 'provider_model_not_found';
default: return null;
}
}
/** Reasons recorded on 7-day rows plus the read-side `unknown` bucket for the halts no reason covers. */
export function haltReasonsWithUnknown(haltCount: number, recorded: Record<string, number | string>): Record<string, number> {
const reasons: Record<string, number> = {};
let classified = 0;
for (const [reason, n] of Object.entries(recorded)) {
const count = Number(n) || 0;
if (count <= 0) continue;
reasons[reason] = count;
classified += count;
}
const unknown = haltCount - classified;
if (unknown > 0) reasons[UNKNOWN_HALT_REASON] = unknown;
return reasons;
}
/**
* The halt reasons recorded over the last 7 days, summed per kind. A brain
* whose schema is behind v239 has no column: every halt then reads as
* `unknown` (a backfill never runs; `unknown` is computed on read).
*/
export async function readRecordedHaltReasons7d(engine: Pick<BrainEngine, 'executeRaw'>): Promise<Map<string, Record<string, number>>> {
const recorded = new Map<string, Record<string, number>>();
try {
const rows = await engine.executeRaw<{ kind: string; reason: string; n: number | string }>(
`SELECT r.kind, e.key AS reason, SUM(e.value::int) AS n
FROM extract_rollup_7d r, jsonb_each_text(r.halt_reasons) e
WHERE r.day >= CURRENT_DATE - 7
GROUP BY r.kind, e.key`, []);
for (const row of rows) {
const perKind = recorded.get(row.kind) ?? {};
perKind[row.reason] = Number(row.n) || 0;
recorded.set(row.kind, perKind);
}
} catch (err) {
if (!isMissingSchemaObject(err, 'column', 'halt_reasons')) throw err;
}
return recorded;
}
/** The most frequent halt reason, `unknown` included, or null without halts. */
export function topHaltReason(reasons: Record<string, number>): { reason: string; count: number } | null {
let top: { reason: string; count: number } | null = null;
for (const [reason, count] of Object.entries(reasons)) {
if (!top || count > top.count || (count === top.count && reason < top.reason)) top = { reason, count };
}
return top;
}
/**
* One UPSERT increments per audit event. All counters default to 0 so
* callers only specify the deltas they care about (e.g. a round-completed
* event passes round_completed_delta=1; an eval-fail event passes
* eval_fail_delta=1; a halt event passes halt_delta=1). cost_delta is the
* cumulative cost ADD for the period this event represents.
*/
export interface RollupUpsertInput {
kind: string;
source_id: string;
/** ISO YYYY-MM-DD. Defaults to today (UTC) if omitted. */
day?: string;
cost_delta?: number;
halt_delta?: number;
eval_fail_delta?: number;
eval_pass_delta?: number;
round_completed_delta?: number;
/** #4482: EXPECTED-limit stop (per-source budget / walltime cap working as
* designed — partial progress banked, more on the next run). Orthogonal to
* halt_delta so doctor's failure rate can exclude self-imposed capacity
* limits. Column added by migration v141. */
expected_limit_delta?: number;
/** Increment rollup_write_failures inside the table (used by the
* self-healing path that records its own write failures forensically). */
failure_delta?: number;
/** #5495: the reason `halt_delta` halts are counted under in `halt_reasons`; omitted halts read as `unknown`. */
halt_reason?: HaltReason;
}
/**
* #4482: shared classification of how an extract run stopped, so every
* extractor records the same three-way split instead of hand-folding its
* flags into one `halted` boolean:
* - error (provider/auth/global abort, per-page failures) → halt_delta=1 —
* the failure-oriented signal doctor warns on, unchanged from today;
* - expected cap only (budget/deadline, no error) → expected_limit_delta=1
* — a capacity signal, observable but NOT a failure;
* - clean completion → round_completed_delta=1.
* An error present alongside a cap counts as an error (the cap didn't cause
* the failed work).
*/
export function classifyRunStop(stop: {
budget_exhausted?: boolean;
deadline_hit?: boolean;
/** `true` for an unclassified error halt (reads as `unknown`); a HaltReason names it (#5495). A cap is never a halt reason. */
error?: boolean | HaltReason;
}): { round_completed_delta: number; halt_delta: number; expected_limit_delta: number; halt_reason?: HaltReason } {
if (stop.error === true || (typeof stop.error === 'string' && (HALT_REASONS as readonly string[]).includes(stop.error))) {
return { round_completed_delta: 0, halt_delta: 1, expected_limit_delta: 0, ...(typeof stop.error === 'string' ? { halt_reason: stop.error } : {}) };
}
if (stop.budget_exhausted === true || stop.deadline_hit === true) {
return { round_completed_delta: 0, halt_delta: 0, expected_limit_delta: 1 };
}
return { round_completed_delta: 1, halt_delta: 0, expected_limit_delta: 0 };
}
function today(): string {
// ISO YYYY-MM-DD in UTC. Matches Postgres CURRENT_DATE behavior for
// UTC servers; for local-time servers there's a slight drift at midnight
// but rollup is best-effort cache so the drift is acceptable.
return new Date().toISOString().slice(0, 10);
}
/**
* UPSERT one rollup event. Best-effort: catches all errors, logs to
* stderr once per (kind, day, error-class) so it doesn't spam, returns
* a boolean indicating success.
*
* Caller composes multiple deltas in one call (round_completed + cost
* together is typical) so the rollup row is one UPSERT per audit event,
* not N UPSERTs.
*/
export async function upsertExtractRollup(
engine: BrainEngine,
input: RollupUpsertInput,
): Promise<{ ok: boolean; error?: string }> {
const day = input.day ?? today();
const cost = input.cost_delta ?? 0;
const halts = input.halt_delta ?? 0;
const evalFails = input.eval_fail_delta ?? 0;
const evalPasses = input.eval_pass_delta ?? 0;
const completed = input.round_completed_delta ?? 0;
const expectedLimits = input.expected_limit_delta ?? 0;
const failures = input.failure_delta ?? 0;
// #5495: the reason bucket this event's halts land in (null: unclassified,
// read back as `unknown`). Bound as text and keyed in SQL: never JSON text
// cast to jsonb (the #2339 class), and only vocabulary words reach the table.
const reason = halts > 0 && input.halt_reason && (HALT_REASONS as readonly string[]).includes(input.halt_reason) ? input.halt_reason : null;
try {
await engine.executeRaw(
`INSERT INTO extract_rollup_7d (
kind, source_id, day,
cost_usd, halt_count, eval_fail_count, eval_pass_count,
round_completed_count, expected_limit_count, rollup_write_failures, halt_reasons, updated_at
)
VALUES ($1, $2, $3::date, $4, $5, $6, $7, $8, $9, $10,
CASE WHEN $11::text IS NULL THEN '{}'::jsonb ELSE jsonb_build_object($11::text, $5::int) END, now())
ON CONFLICT (kind, source_id, day) DO UPDATE SET
cost_usd = extract_rollup_7d.cost_usd + EXCLUDED.cost_usd,
halt_count = extract_rollup_7d.halt_count + EXCLUDED.halt_count,
eval_fail_count = extract_rollup_7d.eval_fail_count + EXCLUDED.eval_fail_count,
eval_pass_count = extract_rollup_7d.eval_pass_count + EXCLUDED.eval_pass_count,
round_completed_count = extract_rollup_7d.round_completed_count + EXCLUDED.round_completed_count,
expected_limit_count = extract_rollup_7d.expected_limit_count + EXCLUDED.expected_limit_count,
rollup_write_failures = extract_rollup_7d.rollup_write_failures + EXCLUDED.rollup_write_failures,
halt_reasons = CASE WHEN $11::text IS NULL THEN extract_rollup_7d.halt_reasons
ELSE jsonb_set(extract_rollup_7d.halt_reasons, ARRAY[$11::text],
to_jsonb(COALESCE((extract_rollup_7d.halt_reasons->>$11::text)::int, 0) + $5::int)) END,
updated_at = now()`,
[input.kind, input.source_id, day, cost, halts, evalFails, evalPasses, completed, expectedLimits, failures, reason],
);
return { ok: true };
} catch (err) {
const msg = (err as Error).message || String(err);
// #5495 (no fallback by design): a brain whose schema is behind v239 has no
// halt_reasons column. The rollup is a best-effort cache, so the row is
// lost until the migration runs; say which fix restores it.
if (isMissingSchemaObject(err, 'column', 'halt_reasons')) {
rollupErrorLogOnce(input.kind, day, `${msg} (the schema is behind: doctor reports migrations_pending; run gbrain apply-migrations --yes --no-autopilot-install)`);
return { ok: false, error: msg };
}
// #4482 back-compat: a brain that hasn't applied migration v141 yet has
// no expected_limit_count column. Rather than losing the WHOLE rollup
// write (best-effort would swallow it), retry the pre-v141 statement —
// the cap stop degrades to the old conflated halt accounting until the
// migration runs.
if (isMissingSchemaObject(err, 'column', 'expected_limit_count')) {
try {
await engine.executeRaw(
`INSERT INTO extract_rollup_7d (
kind, source_id, day,
cost_usd, halt_count, eval_fail_count, eval_pass_count,
round_completed_count, rollup_write_failures, updated_at
)
VALUES ($1, $2, $3::date, $4, $5, $6, $7, $8, $9, now())
ON CONFLICT (kind, source_id, day) DO UPDATE SET
cost_usd = extract_rollup_7d.cost_usd + EXCLUDED.cost_usd,
halt_count = extract_rollup_7d.halt_count + EXCLUDED.halt_count,
eval_fail_count = extract_rollup_7d.eval_fail_count + EXCLUDED.eval_fail_count,
eval_pass_count = extract_rollup_7d.eval_pass_count + EXCLUDED.eval_pass_count,
round_completed_count = extract_rollup_7d.round_completed_count + EXCLUDED.round_completed_count,
rollup_write_failures = extract_rollup_7d.rollup_write_failures + EXCLUDED.rollup_write_failures,
updated_at = now()`,
[input.kind, input.source_id, day, cost, halts + expectedLimits, evalFails, evalPasses, completed, failures],
);
return { ok: true };
} catch { /* fall through to the normal failure record */ }
}
// Don't spam: log once per process per (kind, day) error class.
rollupErrorLogOnce(input.kind, day, msg);
return { ok: false, error: msg };
}
}
const _loggedRollupErrors = new Set<string>();
function rollupErrorLogOnce(kind: string, day: string, msg: string): void {
// Classify by error first 80 chars to dedupe "lock timeout" vs
// "connection refused" but not by exact text.
const klass = msg.slice(0, 80);
const key = `${kind}|${day}|${klass}`;
if (_loggedRollupErrors.has(key)) return;
_loggedRollupErrors.add(key);
console.error(
`[extract-rollup] write failed (best-effort; audit JSONL remains source of truth): ${msg}`,
);
}