Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* v0.31.2 — facts:absorb writer.
*
* D5 contract from /plan-ceo-review: every absorbed failure in the facts
* extraction pipeline writes one row to the existing ingest_log table.
* Cross-process visible (doctor + admin dashboard read the same query),
* scoped per source (codex P1 #3 — migration v50 added source_id), grouped
* by stable reason codes so future tooling can categorize failures.
*
* Mirrors the eval_capture_failures precedent (CLAUDE.md cites this exact
* shape for the eval_capture doctor check). No new infrastructure; one
* helper + a stable reason-code constant set.
*
* Reasons:
* - 'gateway_error' — HTTP 429/5xx, timeout, network blip on chat() or embed().
* - 'parse_failure' — LLM returned malformed JSON, all 4 parser fallback strategies failed.
* - 'queue_overflow' — getFactsQueue() cap hit; oldest entry dropped.
* - 'queue_shutdown' — queue rejected the enqueue because shutdown is in progress.
* - 'embed_failure' — gateway down on embedOne; row inserts with NULL embedding.
* - 'pipeline_error' — anything else absorbed inside runFactsBackstop's catch.
* - 'gateway_auth' — provider authentication/authorization failed.
* - 'gateway_billing' — provider credit, quota, or billing hard limit failed.
* - 'gateway_rate_limit' — provider rate limit; retry policy remains with the caller.
* - 'write_refused' — the write path refused the facts write (an OperationError); the detail starts with its code.
* - 'write_pending_accepted' — the facts write was accepted and its durable receipt is still queued, running or
* recovering after the publish wait; it commits later. Information, not a failure.
* - 'write_deferred_unadmitted' — the canonical writer was busy before admission (writer_lock_unavailable, no
* receipt): nothing was accepted and the job retries.
* - eligibility_skip is intentionally NOT logged (high cardinality, low signal).
*
* The writer is best-effort — a failure to log SHOULDN'T blow up the
* caller's actual work. Errors here are caught and stderr-warned; the
* caller proceeds.
*/
import type { BrainEngine } from '../engine.ts';
import { classifyGlobalLlmError } from '../ai/errors.ts';
import { GBrainError } from '../types.ts';
export const FACTS_ABSORB_REASONS = [
'gateway_error',
'parse_failure',
'queue_overflow',
'queue_shutdown',
'embed_failure',
'pipeline_error',
// Extraction-outcome codes (keyed-but-failing states; keyless-expected
// states deliberately write NO row — see backstop.ts
// surfaceExtractionFailure). Doctor's facts_extraction_health groups by
// split_part(summary,':',1), so new codes surface with zero schema change.
'chat_unavailable',
'refusal',
'content_filter',
'malformed_output',
'non_terminal_stop',
'truncated_output',
'gateway_auth',
'gateway_billing',
'gateway_rate_limit',
'gateway_model_not_found',
'write_refused',
'write_pending_accepted',
'write_deferred_unadmitted',
] as const;
/**
* #5362: write-path refusals that no retry can change until an operator acts
* (a claimed canonical worktree before activation, a missing grant, a bad
* request, an unregistered writer). The facts-absorb job dead-letters them on
* the first attempt instead of re-running inference before the same refusal.
*/
export const DETERMINISTIC_WRITE_REFUSALS: readonly string[] = [
'writer_coordinator_required', 'permission_denied', 'invalid_params', 'writer_registration_required',
];
/** #6778: reasons for facts writes that are not refusals; doctor facts_extraction_health keeps them out of the failure threshold. */
export const WRITE_PENDING_ACCEPTED = 'write_pending_accepted';
export const WRITE_DEFERRED_UNADMITTED = 'write_deferred_unadmitted';
/** The OperationError code of a write-path refusal, or null. Name check keeps this module import-light. */
export function writeRefusalCode(err: unknown): string | null {
const code = (err as { code?: unknown } | null)?.code;
return err instanceof Error && err.name === 'OperationError' && typeof code === 'string' ? code : null;
}
// v0.39.3.0 WARN-4 + CV13 — module-scoped flag so the first-occurrence
// diagnostic log fires ONCE per process. Subsequent occurrences of the
// same 'No database connection' class are suppressed (keeps captures
// quiet) but the first one prints enough context to triage why the
// facts subsystem is calling logIngest on a disconnected engine.
// Exported as a test seam so the assertion can reset between runs.
let _hasLoggedDisconnectedFactsAbsorb = false;
/** @internal — test seam */
export function _resetFactsAbsorbDisconnectedFlagForTests(): void {
_hasLoggedDisconnectedFactsAbsorb = false;
}
export type FactsAbsorbReason = typeof FACTS_ABSORB_REASONS[number];
/**
* Write one row to ingest_log for a facts:absorb event. The row's shape:
*
* source_type = 'facts:absorb'
* source_id = caller's brain source id (default 'default')
* source_ref = page slug or session id the failure was tied to
* summary = `<reason>: <terse detail truncated to 240 chars>`
* pages_updated = []
*
* Best-effort: any error here is caught and stderr-warned; the caller's
* pipeline keeps running. The doctor's facts_extraction_health check
* (PR1 commit 12) reads these rows and warns when any reason exceeds the
* threshold over a 24h window.
*/
export async function writeFactsAbsorbLog(
engine: BrainEngine,
ref: string,
reason: FactsAbsorbReason,
detail: string,
sourceId: string = 'default',
): Promise<void> {
try {
const cleanedDetail = (detail ?? '').toString().slice(0, 240);
await engine.logIngest({
source_id: sourceId,
source_type: 'facts:absorb',
source_ref: ref,
pages_updated: [],
summary: `${reason}: ${cleanedDetail}`,
});
} catch (e) {
// v0.39.3.0 WARN-4 + CQ1 — typed access via instanceof + .problem field
// (NOT string-match on e.message). The 'No database connection' class
// fires after every `gbrain capture` invocation because the facts
// subsystem opens its own engine handle that isn't connected in the
// CLI capture path. Per-capture noise is suppressed; CV13 prints
// ONE first-occurrence stack trace so the next user reporting it
// gives us the call site to fix in v0.38.4.
if (e instanceof GBrainError && e.problem === 'No database connection') {
if (!_hasLoggedDisconnectedFactsAbsorb) {
_hasLoggedDisconnectedFactsAbsorb = true;
// eslint-disable-next-line no-console
console.warn(
`[facts:absorb] suppressed: 'No database connection' fires on a separate engine handle ` +
`(known WARN-4 in v0.38; subsequent occurrences silent this process). ` +
`First-occurrence trace for v0.38.4 diagnosis:\n${e.stack ?? '<no stack>'}`,
);
}
// Subsequent occurrences silent — the page write itself succeeded;
// the facts:absorb log is a courtesy that the doctor health check
// reads. A connection-bound log site already filed the v0.38.4 TODO
// (see TODOS.md).
return;
}
// All other failures keep the loud warn. Don't let logging failures
// cascade — observability can't break the runtime path — but DO let
// the operator see real subsystem errors (PgBouncer crash, schema
// drift, etc.) instead of suppressing them globally.
// eslint-disable-next-line no-console
console.warn(
`[facts:absorb] failed to log ${reason} for ${ref}: ${e instanceof Error ? e.message : String(e)}`,
);
}
}
/**
* Persist an extraction failure without copying provider response bodies,
* keys, or request payloads into ingest_log. A write-path refusal records its
* own code (`write_refused`); global provider failures get stable typed reason
* codes; all other failures retain the existing classifier.
*/
export async function writeFactsAbsorbFailure(
engine: BrainEngine,
ref: string,
err: unknown,
sourceId: string = 'default',
): Promise<void> {
const errorType = err instanceof Error && err.name ? err.name : 'Error';
const refusal = writeRefusalCode(err);
if (refusal) {
const message = (err as Error).message;
if (refusal === 'write_pending') {
const pending = await pendingReceipt(engine, err);
if (pending) {
await writeFactsAbsorbLog(engine, ref, WRITE_PENDING_ACCEPTED,
`${refusal} (${errorType}) request_id ${pending.request_id} (${pending.state}): ${message}`, sourceId);
return;
}
} else if (refusal === 'writer_lock_unavailable' && (err as { writeRequest?: unknown }).writeRequest === undefined) {
await writeFactsAbsorbLog(engine, ref, WRITE_DEFERRED_UNADMITTED, `${refusal} (${errorType}): ${message}`, sourceId);
return;
}
await writeFactsAbsorbLog(engine, ref, 'write_refused', `${refusal} (${errorType}): ${message}`, sourceId);
return;
}
const globalClass = classifyGlobalLlmError(err);
const reason: FactsAbsorbReason = globalClass === 'auth'
? 'gateway_auth'
: globalClass === 'billing'
? 'gateway_billing'
: globalClass === 'rate_limit'
? 'gateway_rate_limit'
: globalClass === 'model_not_found'
? 'gateway_model_not_found'
: classifyFactsAbsorbError(err);
await writeFactsAbsorbLog(engine, ref, reason, `${reason.startsWith('gateway_') ? 'provider request failed' : 'extraction failed'} (${errorType})`, sourceId);
}
/**
* #6778: the durable receipt behind a `write_pending` error, read back from the journal, when it is still queued,
* running or recovering; null when the error names no receipt, the row is gone or terminal, or the read fails.
*/
async function pendingReceipt(engine: BrainEngine, err: unknown): Promise<{ request_id: string; state: string } | null> {
const { writeRequest, receiptFields } = err as { writeRequest?: { request_id?: unknown }; receiptFields?: { principal_kind?: unknown; principal_id?: unknown } };
const requestId = writeRequest?.request_id;
if (typeof requestId !== 'string' || typeof receiptFields?.principal_kind !== 'string' || typeof receiptFields.principal_id !== 'string') return null;
try {
const [row] = await engine.executeRaw<{ request_id: string; state: string }>(
`SELECT request_id::text AS request_id, state FROM persistence_requests
WHERE principal_kind=$1 AND principal_id=$2 AND request_id=$3::uuid AND state IN ('queued','running','recovering')`,
[receiptFields.principal_kind, receiptFields.principal_id, requestId]);
return row ?? null;
} catch {
return null;
}
}
/**
* Classify an arbitrary error into one of the stable reason codes. Heuristic
* pattern match on error name + message; falls back to 'pipeline_error' when
* nothing matches. Public so callers can route the same way the helper does
* (e.g. tests, future telemetry consumers).
*/
export function classifyFactsAbsorbError(err: unknown): FactsAbsorbReason {
if (!err) return 'pipeline_error';
const msg = err instanceof Error ? err.message : String(err);
const name = err instanceof Error ? err.name : '';
// Typed extraction failures carry their reason — map precisely instead of
// pattern-matching the message (a 401/invalid-model provider_error would
// otherwise fall through to the generic 'pipeline_error'). instanceof via
// name check: the class lives in extract.ts and this module must stay
// import-light; the name is stable and set in the constructor.
if (name === 'FactsExtractionError') {
const reason = (err as { reason?: string }).reason;
if (reason === 'provider_error') return 'gateway_error';
if (reason && (FACTS_ABSORB_REASONS as readonly string[]).includes(reason)) {
return reason as FactsAbsorbReason;
}
return 'pipeline_error';
}
// Anthropic / OpenAI / Voyage all surface 4xx/5xx + timeouts in similar shapes.
if (/timeout|timed?\s?out|ETIMEDOUT/i.test(msg)) return 'gateway_error';
if (/429|rate[\s-]?limit|too many requests/i.test(msg)) return 'gateway_error';
if (/5\d\d|server error|internal server|bad gateway|service unavail/i.test(msg)) return 'gateway_error';
if (/ECONNRESET|ECONNREFUSED|EAI_AGAIN|getaddrinfo/i.test(msg)) return 'gateway_error';
// Parser failures from extract.ts's 4-strategy fallback.
if (/JSON.parse|unexpected token|invalid json|not valid JSON/i.test(msg)) return 'parse_failure';
// Queue counter increments — currently bubble through specific paths but
// could surface here if a caller routes them.
if (name === 'QueueOverflowError' || /queue.*overflow|cap.*hit/i.test(msg)) return 'queue_overflow';
if (name === 'QueueShutdownError' || /queue.*shutdown|shutting down/i.test(msg)) return 'queue_shutdown';
// Embed-specific: extract.ts catches embedOne errors and stores NULL embedding.
// If a caller surfaces it explicitly, route to embed_failure.
if (/embed/i.test(msg) && /(fail|error)/i.test(msg)) return 'embed_failure';
return 'pipeline_error';
}