Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Automatic facts drain (Foundations 2, Lane D). PGLite has no background job
* worker, so `facts-absorb` jobs (one per eligible page write) wait until
* something runs them. This module runs them inside the processes that
* already own a PGLite brain: stdio `gbrain serve`, `gbrain serve --http` and
* the dream/autopilot cycle (`facts_drain` phase). One shared implementation:
*
* - one job at a time, with a yield between jobs, a wall-clock budget per
* run and an AbortSignal (shutdown) forwarded into the job handler;
* - spend bounded per run (`facts.drain_budget_usd`, default $1.00), per
* rolling 24 hours (`facts.drain_daily_budget_usd`, default $5.00) and by
* job count (`facts.drain_max_jobs`, default 50); unpriced models warn and
* run under the default caps, and refuse with the register-the-rate fix
* under a cap the user set;
* - every stop that is not "queue empty" is a deferral that keeps the jobs
* queued without counting an attempt (budget used up, no provider key,
* shutdown, provider halt);
* - status (last run, backlog, last error with its fix) lives in the config
* row `facts.drain_state`, read by doctor `facts_drain` and the MCP
* readiness entry `facts_drain`.
*
* Opt out with `gbrain config set facts.extraction_enabled false`. Logging goes
* to stderr only (stdio serve speaks JSON-RPC on stdout).
*/
import { randomUUID } from 'node:crypto';
import type { BrainEngine } from '../engine.ts';
import type { Action, Notice } from '../agent-output.ts';
import type { MinionJob } from '../minions/types.ts';
import { FACTS_DRAIN_KEYS, parseFactsDrainKey, type FactsDrainKey } from './drain-config.ts';
export { FACTS_DRAIN_KEYS, validateFactsDrainConfigValue, type FactsDrainKey } from './drain-config.ts';
export const FACTS_DRAIN_JOB = 'facts-absorb';
export const FACTS_DRAIN_QUEUE = 'default';
export const FACTS_DRAIN_STATE_KEY = 'facts.drain_state';
export const FACTS_DRAIN_OPT_OUT_ARGV = ['gbrain', 'config', 'set', 'facts.extraction_enabled', 'false'] as const;
export const FACTS_DRAIN_DOCS = 'docs/guides/facts-drain.md';
/** One run's default wall-clock budget. */
export const FACTS_DRAIN_WALL_MS = 5 * 60_000;
/** Tokens the extractor adds around the page text (system prompt + framing). */
const PROMPT_OVERHEAD_TOKENS = 1_500;
const RUN_HISTORY = 20;
export interface FactsDrainSettings {
enabled: boolean;
budgetUsd: number;
dailyBudgetUsd: number;
maxJobs: number;
/** A spend cap was set by the user: an unpriced model then refuses instead of warn-and-run. */
explicitCap: boolean;
}
export async function factsDrainSettings(engine: BrainEngine): Promise<FactsDrainSettings> {
const { isFactsExtractionEnabled } = await import('./extract.ts');
const values = {} as Record<FactsDrainKey, number>;
let explicitCap = false;
for (const key of Object.keys(FACTS_DRAIN_KEYS) as FactsDrainKey[]) {
const raw = await engine.getConfig(key).catch(() => null);
const parsed = raw == null || raw.trim() === '' ? null : parseFactsDrainKey(key, raw);
values[key] = parsed ?? FACTS_DRAIN_KEYS[key].fallback;
if (parsed !== null && key !== 'facts.drain_max_jobs') explicitCap = true;
}
return {
enabled: await isFactsExtractionEnabled(engine),
budgetUsd: values['facts.drain_budget_usd'],
dailyBudgetUsd: values['facts.drain_daily_budget_usd'],
maxJobs: values['facts.drain_max_jobs'],
explicitCap,
};
}
export type FactsDrainOwner = 'serve' | 'serve_http' | 'cycle';
export type FactsDrainOutcome =
| 'drained' | 'idle' | 'disabled' | 'not_applicable' | 'max_jobs' | 'deadline' | 'aborted'
| 'no_key' | 'extraction_unavailable' | 'budget_exhausted' | 'daily_budget_exhausted' | 'job_over_budget'
| 'no_pricing' | 'provider_halted' | 'error';
/** Outcomes that leave work queued for a reason the agent can act on. */
export const FACTS_DRAIN_DEFERRALS: ReadonlySet<FactsDrainOutcome> = new Set([
'no_key', 'extraction_unavailable', 'budget_exhausted', 'daily_budget_exhausted', 'job_over_budget', 'no_pricing', 'provider_halted', 'error',
]);
export interface FactsDrainRunRecord {
owner: FactsDrainOwner;
started_at: string;
finished_at: string | null;
outcome: FactsDrainOutcome | 'running';
completed: number;
failed: number;
deferred: number;
facts_inserted: number;
spent_usd: number;
unpriced_calls: number;
backlog_before: number;
backlog_after: number | null;
model: string | null;
error?: { code: string; reason: string; message: string; fix?: Action };
}
interface FactsDrainState { version: 1; runs: FactsDrainRunRecord[]; first_run_reported_at?: string }
export async function readFactsDrainState(engine: BrainEngine): Promise<FactsDrainState> {
try {
const raw = await engine.getConfig(FACTS_DRAIN_STATE_KEY);
const parsed = raw ? JSON.parse(raw) as Partial<FactsDrainState> : null;
if (parsed && Array.isArray(parsed.runs)) return { version: 1, runs: parsed.runs, first_run_reported_at: parsed.first_run_reported_at };
} catch { /* unreadable state starts fresh */ }
return { version: 1, runs: [] };
}
async function writeFactsDrainState(engine: BrainEngine, state: FactsDrainState): Promise<void> {
const cutoff = Date.now() - 48 * 3600_000;
const runs = state.runs.filter((r, i) => i === state.runs.length - 1 || Date.parse(r.started_at) >= cutoff).slice(-RUN_HISTORY);
await engine.setConfig(FACTS_DRAIN_STATE_KEY, JSON.stringify({ ...state, runs }));
}
/** Priced spend of drain runs that started in the last 24 hours. */
export function dailySpentUsd(state: FactsDrainState, now = Date.now()): number {
return state.runs.filter(r => now - Date.parse(r.started_at) < 24 * 3600_000).reduce((sum, r) => sum + (r.spent_usd || 0), 0);
}
export interface FactsDrainBacklog { waiting: number; delayed: number; active: number }
export async function factsDrainBacklog(engine: BrainEngine): Promise<FactsDrainBacklog> {
const rows = await engine.executeRaw<{ status: string; n: number }>(
`SELECT status, count(*)::int AS n FROM minion_jobs
WHERE name = $1 AND queue = $2 AND status IN ('waiting', 'delayed', 'active') GROUP BY status`,
[FACTS_DRAIN_JOB, FACTS_DRAIN_QUEUE]);
const out: FactsDrainBacklog = { waiting: 0, delayed: 0, active: 0 };
for (const r of rows) out[r.status as keyof FactsDrainBacklog] = Number(r.n);
return out;
}
const fmtUsd = (n: number) => `$${n < 0.1 ? n.toFixed(3) : n.toFixed(2)}`;
const command = (argv: readonly string[]) => argv.join(' ');
function keySetupFix(): Action {
return {
argv: ['gbrain', 'providers', 'list'], consent: ['credentials', 'paid'], actor: 'user', requires_exclusive: false,
why: 'Automatic fact extraction needs one chat provider key (for example ANTHROPIC_API_KEY or OPENAI_API_KEY, stored with `gbrain config set anthropic_api_key <key>`); `gbrain providers list` shows what each provider needs. Queued pages run on the next drain once a key exists.',
user_message: 'Pages are waiting for automatic fact extraction, which needs a chat model API key (Anthropic or OpenAI, for example). Want to add one? It costs a little per page processed.',
verify: { argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'] },
};
}
function budgetFix(key: 'facts.drain_budget_usd' | 'facts.drain_daily_budget_usd', suggested: number): Action {
return {
argv: ['gbrain', 'config', 'set', key, suggested.toFixed(2)], consent: ['paid'], actor: 'agent', requires_exclusive: false,
why: `Raises the cap so more queued pages are extracted per ${key === 'facts.drain_budget_usd' ? 'run' : 'day'}; each page is one paid chat call. Ask the user first. Doing nothing is also fine: the jobs wait and run when budget frees up.`,
user_message: `Automatic fact extraction reached its ${key === 'facts.drain_budget_usd' ? 'per-run' : 'daily'} spending cap and queued pages are waiting. Raise the cap to ${fmtUsd(suggested)}, or let them run as budget frees up?`,
verify: { argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'] },
};
}
export interface FactsDrainRunResult extends Omit<FactsDrainRunRecord, 'outcome'> { outcome: FactsDrainOutcome; notice?: Notice }
export interface FactsDrainRunOpts {
owner: FactsDrainOwner;
signal?: AbortSignal;
wallClockMs?: number;
/** Stderr sink (default process.stderr). Never stdout. */
log?: (line: string) => void;
/** Pause between jobs so foreground work interleaves (default 25 ms). */
yieldMs?: number;
}
/** Pending notice for the next stdio MCP call (dispatch takes it); one per reason per process. */
let pendingNotice: Notice | null = null;
const noticedReasons = new Set<string>();
export function takeFactsDrainNotice(): Notice | null {
const n = pendingNotice;
pendingNotice = null;
return n;
}
/** Test-only: reset the process-wide notice memo. */
export function __resetFactsDrainNoticesForTests(): void { pendingNotice = null; noticedReasons.clear(); }
function findError<T>(err: unknown, match: (e: unknown) => e is T): T | null {
let cur: unknown = err;
for (let i = 0; i < 6 && cur; i++) {
if (match(cur)) return cur;
cur = (cur as { cause?: unknown }).cause;
}
return null;
}
/** Estimated cost of extracting the queued pages, from their text length and the model price. */
async function estimateBacklogUsd(engine: BrainEngine, model: string, backlog: number, maxTokens: number): Promise<number | null> {
const { reservationCostUsd } = await import('../budget/reservation-cost.ts');
const { MAX_TURN_TEXT_CHARS } = await import('./extract.ts');
const [row] = await engine.executeRaw<{ chars: number | null }>(
`SELECT avg(LEAST(length(p.compiled_truth), $3))::float AS chars FROM minion_jobs j
JOIN pages p ON p.slug = j.data->>'slug' AND p.source_id = COALESCE(j.data->>'sourceId', 'default')
WHERE j.name = $1 AND j.queue = $2 AND j.status IN ('waiting', 'delayed')`,
[FACTS_DRAIN_JOB, FACTS_DRAIN_QUEUE, MAX_TURN_TEXT_CHARS]).catch(() => [{ chars: null }]);
const inputTokens = Math.ceil((row?.chars ?? MAX_TURN_TEXT_CHARS / 2) / 4) + PROMPT_OVERHEAD_TOKENS;
const perJob = reservationCostUsd(model, 'chat', inputTokens, Math.min(maxTokens, 300));
return perJob === null ? null : perJob * backlog;
}
/**
* Run one bounded drain pass. Never throws: failures become the `error`
* outcome with the cause recorded in `facts.drain_state`.
*/
export async function runFactsDrain(engine: BrainEngine, opts: FactsDrainRunOpts): Promise<FactsDrainRunResult> {
const log = opts.log ?? ((line: string) => { process.stderr.write(`${line}\n`); });
const startedAt = new Date();
const result: FactsDrainRunResult = {
owner: opts.owner, started_at: startedAt.toISOString(), finished_at: null, outcome: 'idle',
completed: 0, failed: 0, deferred: 0, facts_inserted: 0, spent_usd: 0, unpriced_calls: 0,
backlog_before: 0, backlog_after: null, model: null,
};
const finish = (outcome: FactsDrainOutcome): FactsDrainRunResult => {
result.outcome = outcome;
result.finished_at = new Date().toISOString();
return result;
};
if (engine.kind !== 'pglite') return finish('not_applicable');
const { isEngineDegraded } = await import('../degraded-marker.ts');
if (isEngineDegraded(engine)) return finish('not_applicable');
let state: FactsDrainState | null = null;
let recorded = false;
const save = async () => {
if (!state) return;
const rec: FactsDrainRunRecord = { ...result, outcome: result.finished_at ? result.outcome : 'running' };
delete (rec as { notice?: unknown }).notice;
if (recorded) state.runs[state.runs.length - 1] = rec;
else { state.runs.push(rec); recorded = true; }
await writeFactsDrainState(engine, state).catch(() => undefined);
};
const defer = async (outcome: FactsDrainOutcome, reason: string, message: string, fix?: Action): Promise<FactsDrainRunResult> => {
result.error = { code: 'facts_drain_deferred', reason, message, ...(fix ? { fix } : {}) };
finish(outcome);
result.notice = { code: 'facts_drain_deferred', kind: 'degraded', why: message, ...(fix ? { fix } : {}) };
if (!noticedReasons.has(reason)) {
noticedReasons.add(reason);
log(`[gbrain notice facts_drain_deferred] ${message}${fix?.argv ? ` Next: ${command(fix.argv)}` : ''}`);
pendingNotice = result.notice;
}
await save();
return result;
};
try {
const settings = await factsDrainSettings(engine);
if (!settings.enabled) return finish('disabled');
const { MinionQueue } = await import('../minions/queue.ts');
const queue = new MinionQueue(engine);
await queue.promoteDelayed();
await queue.handleStalled();
const backlog = await factsDrainBacklog(engine);
result.backlog_before = backlog.waiting;
if (backlog.waiting === 0) { result.backlog_after = 0; return finish('idle'); }
const { refreshGatewayEnvFromFilePlane, reconfigureGatewayWithEngine, withBudgetTracker } = await import('../ai/gateway.ts');
refreshGatewayEnvFromFilePlane();
await reconfigureGatewayWithEngine(engine).catch(() => undefined);
const { resolveExtractionAvailability } = await import('./extraction-availability.ts');
const { getFactsExtractionMaxTokens } = await import('./extract.ts');
const { model, available } = await resolveExtractionAvailability(engine);
result.model = model;
state = await readFactsDrainState(engine);
if (!available) {
const { classifyUnavailable } = await import('./backstop.ts');
if ((await classifyUnavailable(model)) === 'keyless') {
return defer('no_key', 'no_key',
`${backlog.waiting} page(s) wait for automatic fact extraction, which needs a chat provider key (OpenAI or Anthropic). They stay queued and run on the next drain after a key is set.`,
keySetupFix());
}
return defer('extraction_unavailable', 'extraction_unavailable',
`The facts extraction model ${model} is not servable (its provider is not configured); ${backlog.waiting} page(s) stay queued.`,
{ argv: ['gbrain', 'config', 'set', 'facts.extraction_model', '<provider:model>'], consent: ['paid'], actor: 'agent', requires_exclusive: false,
why: 'Points extraction at a model whose provider has a key, or set that provider\'s key instead.',
inputs: [{ name: 'provider:model', how: 'Pick a chat model whose provider key is configured (gbrain models doctor --json lists them).' }],
verify: { argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'] } });
}
const maxTokens = await getFactsExtractionMaxTokens(engine);
if (!state.first_run_reported_at) {
const est = await estimateBacklogUsd(engine, model, backlog.waiting, maxTokens);
const line = `[gbrain notice facts_drain] Automatic fact extraction is on: ${backlog.waiting} queued page(s), about ${est === null ? 'an unknown amount (unpriced model)' : fmtUsd(est)} with ${model}, ` +
`at most ${fmtUsd(settings.budgetUsd)} per run and ${fmtUsd(settings.dailyBudgetUsd)} per day. To opt out: ${command(FACTS_DRAIN_OPT_OUT_ARGV)}`;
log(line);
noticedReasons.add('first_run');
pendingNotice = { code: 'facts_drain_first_run', kind: 'info', why: line.replace('[gbrain notice facts_drain] ', ''),
fix: { argv: [...FACTS_DRAIN_OPT_OUT_ARGV], consent: [], actor: 'agent', requires_exclusive: false,
why: 'Only if the user does not want automatic paid extraction; queued pages are then left unextracted.', verify: { argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'] } } };
state.first_run_reported_at = startedAt.toISOString();
}
const dailyRemaining = settings.dailyBudgetUsd - dailySpentUsd(state, startedAt.getTime());
if (dailyRemaining <= 0) {
return defer('daily_budget_exhausted', 'daily_budget_exhausted',
`Automatic fact extraction used its ${fmtUsd(settings.dailyBudgetUsd)} daily budget; ${backlog.waiting} page(s) stay queued and run as the 24-hour window frees budget.`,
budgetFix('facts.drain_daily_budget_usd', settings.dailyBudgetUsd * 2));
}
const runCap = Math.min(settings.budgetUsd, dailyRemaining);
const capIsDaily = dailyRemaining < settings.budgetUsd;
const { BudgetTracker, BudgetExhausted, loadPricingOverrides } = await import('../budget/budget-tracker.ts');
const { reservationCostUsd } = await import('../budget/reservation-cost.ts');
const { MAX_TURN_TEXT_CHARS } = await import('./extract.ts');
const pricingOverrides = await loadPricingOverrides(engine);
const ceiling = reservationCostUsd(model, 'chat', Math.ceil(MAX_TURN_TEXT_CHARS / 4) + PROMPT_OVERHEAD_TOKENS, maxTokens, pricingOverrides);
if (ceiling === null && settings.explicitCap) {
const { noPricingGuidance, noPricingFix, noPricingMessage } = await import('../budget/no-pricing.ts');
const guidance = noPricingGuidance(model, 'chat');
return defer('no_pricing', 'no_pricing', noPricingMessage(guidance, { label: 'facts drain', capUsd: runCap }), noPricingFix(guidance));
}
const tracker = new BudgetTracker({ maxCostUsd: runCap, capSource: settings.explicitCap ? 'user' : 'default', label: 'facts:drain', pricingOverrides });
await save();
const { makeFactsAbsorbHandler } = await import('../minions/handlers/facts-absorb.ts');
const { withFactsAbsorbHaltCooldown } = await import('../minions/llm-halt-cooldown.ts');
const { JobDeferredError, UnrecoverableError } = await import('../minions/errors.ts');
const { RateLeaseUnavailableError, leaseFullBackoffMs } = await import('../minions/rate-leases.ts');
const { buildJobContext } = await import('../minions/job-context.ts');
const { authorizeJobExecution, withSubmissionAuthority } = await import('../minions/submission-authority.ts');
const { withChatPhase } = await import('../ai/chat-usage.ts');
const { calculateBackoff } = await import('../minions/backoff.ts');
const { defaultLockDurationMsFor } = await import('../minions/handler-timeouts.ts');
const handler = withFactsAbsorbHaltCooldown(makeFactsAbsorbHandler(engine));
const lockMs = defaultLockDurationMsFor(FACTS_DRAIN_JOB) ?? 120_000;
const deadline = startedAt.getTime() + (opts.wallClockMs ?? FACTS_DRAIN_WALL_MS);
let outcome: FactsDrainOutcome = 'drained';
let stopReason: { reason: string; message: string; fix?: Action } | null = null;
while (true) {
if (opts.signal?.aborted) { outcome = 'aborted'; break; }
if (result.completed + result.failed >= settings.maxJobs) { outcome = 'max_jobs'; break; }
if (Date.now() >= deadline) { outcome = 'deadline'; break; }
const spent = tracker.totalSpent;
if (ceiling !== null && spent + ceiling > runCap) {
if (result.completed + result.failed === 0 && ceiling > settings.budgetUsd) {
outcome = 'job_over_budget';
stopReason = { reason: 'job_over_budget', message: `One queued page can cost up to ${fmtUsd(ceiling)} with ${model}, above the ${fmtUsd(settings.budgetUsd)} per-run cap, so automatic fact extraction cannot start it. The pages stay queued.`,
fix: budgetFix('facts.drain_budget_usd', Math.max(1, Math.ceil(ceiling * 2 * 100) / 100)) };
} else if (capIsDaily) {
outcome = 'daily_budget_exhausted';
stopReason = { reason: 'daily_budget_exhausted', message: `Automatic fact extraction reached its ${fmtUsd(settings.dailyBudgetUsd)} daily budget; the remaining pages stay queued and run as the 24-hour window frees budget.`, fix: budgetFix('facts.drain_daily_budget_usd', settings.dailyBudgetUsd * 2) };
} else {
outcome = 'budget_exhausted';
stopReason = { reason: 'budget_exhausted', message: `Automatic fact extraction reached its ${fmtUsd(settings.budgetUsd)} per-run budget; the remaining pages stay queued for the next run.`, fix: budgetFix('facts.drain_budget_usd', settings.budgetUsd * 2) };
}
break;
}
const lockToken = randomUUID();
const job: MinionJob | null = await queue.claim(lockToken, lockMs, FACTS_DRAIN_QUEUE, [FACTS_DRAIN_JOB]);
if (!job) { outcome = 'drained'; break; }
const abort = new AbortController();
const onShutdown = () => { if (!abort.signal.aborted) abort.abort(new Error('shutdown')); };
opts.signal?.addEventListener('abort', onShutdown, { once: true });
let renewing = false;
const renew = setInterval(() => {
if (renewing) return;
renewing = true;
queue.renewLock(job.id, lockToken, lockMs)
.then(ok => { if (!ok && !abort.signal.aborted) abort.abort(new Error('lock-lost')); }, () => undefined)
.finally(() => { renewing = false; });
}, Math.max(1000, Math.floor(lockMs / 3)));
let value: unknown;
let thrown: unknown = null;
try {
const authority = await authorizeJobExecution(engine, job);
const ctx = buildJobContext(engine, queue, job, lockToken, abort.signal, opts.signal ?? abort.signal);
value = await withBudgetTracker(tracker, () => withSubmissionAuthority(authority, () => withChatPhase(`job:${job.name}`, () => handler(ctx)), abort.signal));
} catch (e) {
thrown = e ?? new Error('facts-absorb failed');
} finally {
clearInterval(renew);
opts.signal?.removeEventListener('abort', onShutdown);
}
const budgetErr = thrown ? findError(thrown, (e): e is InstanceType<typeof BudgetExhausted> => e instanceof BudgetExhausted) : null;
if (!thrown) {
await queue.completeJob(job.id, lockToken, value != null && typeof value === 'object' ? value as Record<string, unknown> : undefined);
result.completed++;
result.facts_inserted += Number((value as { inserted?: number } | null)?.inserted ?? 0);
} else if (opts.signal?.aborted) {
await queue.deferJob(job.id, lockToken, 'deferred (shutdown): the owning process stopped mid-job', 0);
result.deferred++;
outcome = 'aborted';
} else if (thrown instanceof JobDeferredError) {
await queue.deferJob(job.id, lockToken, `deferred (${thrown.reason}): ${thrown.message}`, thrown.retryInMs);
result.deferred++;
if (thrown.reason === 'no_key') { outcome = 'no_key'; stopReason = { reason: 'no_key', message: thrown.message, fix: keySetupFix() }; }
} else if (thrown instanceof RateLeaseUnavailableError) {
await queue.deferJob(job.id, lockToken, `deferred (provider_halted): ${thrown.message}`, thrown.retryInMs ?? leaseFullBackoffMs());
result.deferred++;
outcome = 'provider_halted';
stopReason = { reason: 'provider_halted', message: `The chat provider for ${model} is refusing calls (${thrown.message}); queued pages wait for its cooldown.` };
} else if (budgetErr) {
await queue.deferJob(job.id, lockToken, `deferred (${budgetErr.reason === 'no_pricing' ? 'no_pricing' : 'budget_exhausted'}): ${budgetErr.message}`, 0);
result.deferred++;
outcome = budgetErr.reason === 'no_pricing' ? 'no_pricing' : capIsDaily ? 'daily_budget_exhausted' : 'budget_exhausted';
stopReason = { reason: outcome, message: budgetErr.message, ...(budgetErr.fix ? { fix: budgetErr.fix } : {}) };
} else if (abort.signal.aborted && String((abort.signal.reason as Error)?.message) === 'lock-lost') {
result.failed++;
} else {
const message = thrown instanceof Error ? thrown.message : String(thrown);
const dead = thrown instanceof UnrecoverableError || job.attempts_made + 1 >= job.max_attempts;
await queue.failJob(job.id, lockToken, message, dead ? 'dead' : 'delayed', dead ? 0 : calculateBackoff({ ...job, attempts_made: job.attempts_made + 1 }));
result.failed++;
result.error = { code: 'facts_drain_job_failed', reason: dead ? 'dead' : 'retrying', message: `facts-absorb job ${job.id} (${String(job.data.slug ?? '')}) failed: ${message}`,
fix: { argv: ['gbrain', 'jobs', 'get', String(job.id), '--json'], consent: [], actor: 'agent', requires_exclusive: false, why: 'Shows the job\'s error and attempts; a retrying job runs again on a later drain.' } };
}
const snap = tracker.snapshot();
result.spent_usd = snap.cumulativeCostUsd;
result.unpriced_calls = snap.models.filter(m => m.cost_usd === null).reduce((n, m) => n + m.calls, 0);
await save();
if (outcome !== 'drained') break;
await new Promise(resolve => setTimeout(resolve, opts.yieldMs ?? 25));
}
result.backlog_after = (await factsDrainBacklog(engine)).waiting;
if (stopReason) return defer(outcome, stopReason.reason, stopReason.message, stopReason.fix);
finish(outcome);
await save();
return result;
} catch (e) {
const message = e instanceof Error ? e.message : String(e);
state ??= await readFactsDrainState(engine).catch(() => ({ version: 1 as const, runs: [] }));
return defer('error', 'drain_error', `Automatic fact extraction failed before finishing its run: ${message}. Queued pages stay queued.`,
{ argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'], consent: [], actor: 'agent', requires_exclusive: false, why: 'Shows the drain status and the last error.' });
}
}
export const FACTS_DRAIN_RUN_NOW_ARGV = ['gbrain', 'dream', '--phase', 'facts_drain'] as const;
/** A backlog older than this with no run means no owner is draining it. */
const STALE_RUN_MS = 45 * 60_000;
export type FactsDrainHealth = 'not_applicable' | 'disabled' | 'idle' | 'ok' | 'deferred' | 'no_owner';
export interface FactsDrainStatus {
health: FactsDrainHealth;
message: string;
fix?: Action;
settings: FactsDrainSettings | null;
backlog: FactsDrainBacklog;
last_run: FactsDrainRunRecord | null;
daily_spent_usd: number;
}
/** The one status read for doctor `facts_drain` and the MCP readiness entry `facts_drain`. */
export async function readFactsDrainStatus(engine: BrainEngine, now = Date.now()): Promise<FactsDrainStatus> {
const empty: FactsDrainBacklog = { waiting: 0, delayed: 0, active: 0 };
if (engine.kind !== 'pglite') {
return { health: 'not_applicable', message: 'Postgres brains run facts-absorb jobs on the job worker (gbrain jobs supervisor), not the automatic drain.', settings: null, backlog: empty, last_run: null, daily_spent_usd: 0 };
}
const settings = await factsDrainSettings(engine);
const backlog = await factsDrainBacklog(engine);
const state = await readFactsDrainState(engine);
const last = state.runs.at(-1) ?? null;
const daily = dailySpentUsd(state, now);
const base = { settings, backlog, last_run: last, daily_spent_usd: daily };
const queued = backlog.waiting + backlog.delayed;
const lastText = last ? `last run ${last.finished_at ?? last.started_at} by ${last.owner}: ${last.outcome}, ${last.completed} extracted, ${fmtUsd(last.spent_usd)}` : 'no run recorded yet';
if (!settings.enabled) {
return { ...base, health: 'disabled', message: `Facts extraction is off (facts.extraction_enabled false); ${queued} facts-absorb job(s) are queued and are skipped when they run.`,
fix: { argv: ['gbrain', 'config', 'set', 'facts.extraction_enabled', 'true'], consent: ['paid', 'egress'], actor: 'agent', requires_exclusive: false,
why: 'Turns automatic fact extraction back on: one paid chat call per eligible page write, bounded per run and per day.' } };
}
if (queued === 0) return { ...base, health: 'idle', message: `No facts-absorb jobs are queued (${lastText}).` };
if (last?.error && last.outcome !== 'running' && FACTS_DRAIN_DEFERRALS.has(last.outcome as FactsDrainOutcome)) {
return { ...base, health: 'deferred', message: `${queued} facts-absorb job(s) wait: ${last.error.message}`, ...(last.error.fix ? { fix: last.error.fix } : {}) };
}
const lastAt = last ? Date.parse(last.finished_at ?? last.started_at) : 0;
if (now - lastAt > STALE_RUN_MS) {
return { ...base, health: 'no_owner', message: `${queued} facts-absorb job(s) are queued and no drain ran in the last ${STALE_RUN_MS / 60_000} minutes (${lastText}). ` +
'The drain runs inside a resident gbrain serve (stdio or --http) and the dream/autopilot cycle; with none running, run it now.',
fix: { argv: [...FACTS_DRAIN_RUN_NOW_ARGV], consent: ['paid'], actor: 'agent', requires_exclusive: true,
why: 'Runs one bounded drain (per-run and daily caps apply) in this process; each page is one paid chat call.', verify: { argv: ['gbrain', 'doctor', '--only', 'facts_drain', '--json'] } } };
}
return { ...base, health: 'ok', message: `${queued} facts-absorb job(s) queued; the drain is running them (${lastText}; ${fmtUsd(daily)} of ${fmtUsd(settings.dailyBudgetUsd)} used in 24 h).` };
}