Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
/**
* Unattended upkeep (GBRA-438): the free, never-purging maintenance a brain
* runs with nobody watching, and the runner core `gbrain maintain --safe`
* shares its step list with.
*
* One run is a list of work items: one global item (the brain-wide free
* phases, once per window, under the shared `gbrain-cycle` lease) and one
* item per source (its source-scoped free phases, under that source's lease),
* then the embedding catch-up step on a brain with an embedding key, under a
* daily dollar cap (`maintenance.embed_daily_usd`, default $1, 0 turns it
* off). Nothing here purges a page or calls a chat model; the page purge
* stays preview-first (`gbrain maintain --purge`), and every other paid phase
* stays with `gbrain dream` or an installed autopilot.
*
* Cycles run with `stamp: 'none'`, so upkeep never writes
* `last_full_cycle_at` or `last_source_cycle_at`. Its own record is
* `maintenance.last_run` in the database config, which is also the record on
* a repo-less brain. The runner never throws to its caller: every skip and
* failure comes back as an outcome with a code and a fix.
*/
import { existsSync } from 'fs';
import { isAbsolute } from 'path';
import type { BrainEngine } from '../engine.ts';
import type { Action } from '../agent-output.ts';
import { ALL_PHASES, LAST_GLOBAL_AT_KEY, PHASE_SCOPE, anyAbortSignal, resolveCyclePhases, runCycle, type CycleOpts, type CyclePhase, type CycleReport } from '../cycle.ts';
import { MANAGED_PHASE_TABLE } from '../cycle/phase-table.ts';
import { isAvailable } from '../ai/gateway.ts';
import { fenceRepairLlmEnabled } from '../fence-repair/config.ts';
import { resolveFenceRepairModel } from '../fence-repair/model.ts';
import { resolveContentRepairModel } from '../content-repair/llm.ts';
import { loadConfigSnapshot } from '../config-snapshot.ts';
import { readDecideConfig } from '../ai/decide/config.ts';
import { hasTypesafeKey } from '../ai/decide/index.ts';
import { isEngineDegraded } from '../degraded-marker.ts';
import { hasPendingMigrations } from '../migrate-pending.ts';
import { tryAcquireDbLock } from '../db-lock.ts';
import { resolveArchiveEnabled } from '../transcripts/archive-default.ts';
import type { ArchiveMaintenanceOutcome } from '../transcripts/archive-maintenance.ts';
export const MAINTENANCE_LAST_RUN_KEY = 'maintenance.last_run';
export const MAINTENANCE_AUTO_KEY = 'maintenance.auto';
export const MAINTENANCE_EMBED_DAILY_USD_KEY = 'maintenance.embed_daily_usd';
/** Unattended embedding spend per UTC day: `{ day, usd }`. */
export const MAINTENANCE_EMBED_SPEND_KEY = 'maintenance.embed_spend';
export const DEFAULT_EMBED_DAILY_USD = 1;
/** A finished round is due again after this long. */
export const UPKEEP_INTERVAL_MS = 24 * 60 * 60 * 1000;
/** A step failing with the same code this many runs in a row backs off to once a week. */
export const UPKEEP_BACKOFF_FAILURES = 3;
export const UPKEEP_BACKOFF_MS = 7 * 24 * 60 * 60 * 1000;
/**
* A budgeted (serve) run parks an item that its budget stopped this many times in one round: the round settles without
* it and budgeted runs skip it until the next round, an unbudgeted run (`gbrain maintain --unattended`, the schedule) or
* a serve's first idle run (`retryParked`) runs it again. A foreground request's abort never counts as a slice.
*/
export const UPKEEP_PARK_AFTER_SLICES = 30;
/** One runner embeds at a time, so two serves finishing a round together never both spend the day's cap. */
export const MAINTENANCE_EMBED_LOCK_ID = 'gbrain-maintenance-embed';
/** An autopilot whose global pass is younger than this owns upkeep. */
export const AUTOPILOT_OWNS_WITHIN_MS = 48 * 60 * 60 * 1000;
/**
* The cycle phases `--safe` may run: free, and never hard-deleting. The page
* purge and every phase that calls a model stay with `gbrain dream` and
* `gbrain pages purge-deleted`.
*/
export const MAINTAIN_SAFE_PHASES: readonly CyclePhase[] = [
'lint', 'backlinks', 'sync', 'fence_repair', 'content_repair', 'unindexed_files',
'extract', 'extract_facts', 'resolve_symbol_edges', 'recompute_emotional_weight',
'consolidate', 'orphans', 'schema-suggest',
];
/** Which providers a safe phase would call on this brain; those phases are held back. */
export interface SafePhaseProviders {
embedding: boolean;
decideConflict: boolean;
fenceRepairModel: boolean;
contentRepairModel: boolean;
}
export function maintainSafePhases(sourceId: string, providers: SafePhaseProviders): { phases: CyclePhase[]; held: CyclePhase[] } {
const held: CyclePhase[] = [];
if (providers.embedding || providers.decideConflict) held.push('extract_facts');
if (providers.fenceRepairModel || providers.embedding) held.push('fence_repair');
if (providers.contentRepairModel || providers.embedding) held.push('content_repair');
const phases = resolveCyclePhases(undefined, sourceId)
.filter((phase) => MAINTAIN_SAFE_PHASES.includes(phase) && !held.includes(phase));
return { phases, held };
}
export async function readSafePhaseProviders(engine: BrainEngine): Promise<SafePhaseProviders> {
const repairLlm = await fenceRepairLlmEnabled(engine);
return {
embedding: isAvailable('embedding'),
decideConflict: readDecideConfig(await loadConfigSnapshot(engine), { typesafeKey: hasTypesafeKey() }).slots.conflict.mode !== 'off',
fenceRepairModel: repairLlm && (await resolveFenceRepairModel(engine)) !== null,
contentRepairModel: repairLlm && (await resolveContentRepairModel(engine)) !== null,
};
}
export interface UpkeepPlan {
/** Brain-wide phases, one item per run. */
global: CyclePhase[];
/** Source-scoped phases, one item per source. */
source: CyclePhase[];
/** Phases left out because they would call a provider on this brain. */
held: CyclePhase[];
/** An embedding key exists: extract_facts writes new rows without vectors and the capped embedding step adds them. */
embedding: boolean;
}
/**
* The unattended step list for this brain: the `--safe` phases plus the
* bookkeeping GC (decision C1). extract_facts stays in on a keyed brain with
* its embedding deferred; it is held only when the System One conflict slot
* would call its provider. The repair lanes stay held whenever their LLM step
* is on or an embedding key exists, because their publications embed.
*/
export function unattendedPlan(providers: SafePhaseProviders): UpkeepPlan {
const held: CyclePhase[] = [];
if (providers.decideConflict) held.push('extract_facts');
if (providers.fenceRepairModel || providers.embedding) held.push('fence_repair');
if (providers.contentRepairModel || providers.embedding) held.push('content_repair');
const steps = ALL_PHASES.filter((phase) => (MAINTAIN_SAFE_PHASES.includes(phase) || phase === 'bookkeeping_gc') && !held.includes(phase));
return {
global: steps.filter((phase) => PHASE_SCOPE[phase] !== 'source'),
source: steps.filter((phase) => PHASE_SCOPE[phase] === 'source'),
held,
embedding: providers.embedding,
};
}
/** Paid-tier phases: what upkeep leaves for the person (`gbrain dream`, autopilot). */
export const PAID_PHASES: readonly CyclePhase[] = ALL_PHASES.filter((phase) => MANAGED_PHASE_TABLE[phase].tier === 'paid');
export type UpkeepStepStatus = 'ok' | 'warn' | 'fail' | 'skipped';
export interface UpkeepStep { name: string; scope: string; status: UpkeepStepStatus; code?: string; reason?: string }
export interface UpkeepFailure { step: string; scope: string; code: string; count: number; last_at: string }
export interface UpkeepRecord {
at: string;
runner: string;
status: 'ok' | 'partial' | 'failed';
steps: UpkeepStep[];
held: CyclePhase[];
/**
* Items already settled in an unfinished round; null once the round finished. `attempts` counts a budgeted run's
* unfinished slices per item, `retry` names parked items this round runs again, and `due_at` is the due time a
* round that only reruns parked items keeps.
*/
cursor: { round_started_at: string; done: string[]; attempts?: Record<string, number>; retry?: string[]; due_at?: string } | null;
/** Items budgeted runs gave up on (UPKEEP_PARK_AFTER_SLICES), with their total unfinished slices. */
parked?: Array<{ key: string; since: string; slices: number }>;
next_due_at: string;
consecutive_failures: UpkeepFailure[];
embed?: { spent_today_usd: number; cap_usd: number };
last_skip?: { at: string; code: string };
}
export interface UpkeepOutcome {
status: 'ran' | 'skipped' | 'not_due';
/** 0 for every skip; 1 when a step failed (`maintain` only; the serve never exits). */
exit: 0 | 1;
code?: string;
message: string;
fix?: Action;
record?: UpkeepRecord;
}
export interface UpkeepOpts {
runner: string;
now?: () => Date;
/** Wall-clock budget for the whole run; an unfinished round resumes on the next run. */
budgetMs?: number;
signal?: AbortSignal;
/** A serve's first idle run: parked items get one more set of UPKEEP_PARK_AFTER_SLICES slices. */
retryParked?: boolean;
/** Test seams. */
providers?: SafePhaseProviders;
capacityPaused?: (engine: BrainEngine) => Promise<{ command: string; config_key: string } | null>;
cycle?: typeof runCycle;
archive?: (engine: BrainEngine, opts: { runner: string; now: Date }) => Promise<ArchiveMaintenanceOutcome>;
}
const RUNNER_NAME = /^[a-z][a-z0-9_-]{0,31}$/;
export const isValidRunnerName = (name: string): boolean => RUNNER_NAME.test(name);
export async function readUpkeepRecord(engine: BrainEngine): Promise<UpkeepRecord | null> {
const raw = await engine.getConfig(MAINTENANCE_LAST_RUN_KEY).catch(() => null);
if (!raw) return null;
try {
const parsed = JSON.parse(raw) as UpkeepRecord;
return parsed && typeof parsed.at === 'string' && Array.isArray(parsed.steps) ? parsed : null;
} catch {
return null;
}
}
/** Whether a run would do work now: no record yet, a round left unfinished, or past next_due_at. */
export function upkeepDue(record: UpkeepRecord | null, now: Date): boolean {
return !record || !!record.cursor || !(Date.parse(record.next_due_at) > now.getTime());
}
export async function upkeepEnabled(engine: BrainEngine): Promise<boolean> {
const raw = (await engine.getConfig(MAINTENANCE_AUTO_KEY).catch(() => null))?.trim().toLowerCase();
return !(raw === 'false' || raw === '0' || raw === 'off' || raw === 'no');
}
export async function readEmbedDailyCapUsd(engine: BrainEngine): Promise<number> {
const raw = (await engine.getConfig(MAINTENANCE_EMBED_DAILY_USD_KEY).catch(() => null))?.trim();
if (!raw) return DEFAULT_EMBED_DAILY_USD;
const value = Number(raw);
return Number.isFinite(value) && value >= 0 ? value : DEFAULT_EMBED_DAILY_USD;
}
const utcDay = (at: Date) => at.toISOString().slice(0, 10);
export async function readEmbedSpentToday(engine: BrainEngine, now: Date): Promise<number> {
const raw = await engine.getConfig(MAINTENANCE_EMBED_SPEND_KEY).catch(() => null);
if (!raw) return 0;
try {
const parsed = JSON.parse(raw) as { day?: string; usd?: number };
return parsed.day === utcDay(now) && typeof parsed.usd === 'number' && Number.isFinite(parsed.usd) ? parsed.usd : 0;
} catch {
return 0;
}
}
/** Whether an autopilot keeps this brain maintained: its brain-wide pass stamped recently (any host). */
export async function autopilotOwnsUpkeep(engine: BrainEngine, now: Date): Promise<string | null> {
const raw = await engine.getConfig(LAST_GLOBAL_AT_KEY).catch(() => null);
if (!raw) return null;
const at = Date.parse(raw);
return Number.isFinite(at) && now.getTime() - at < AUTOPILOT_OWNS_WITHIN_MS ? raw : null;
}
async function defaultCapacityPaused(engine: BrainEngine): Promise<{ command: string; config_key: string } | null> {
const { cachedCapacitySnapshot } = await import('../persistence/capacity-snapshot.ts');
const snapshot = await cachedCapacitySnapshot(engine as never).catch(() => null);
const row = snapshot?.paused[0];
return row ? { command: row.command, config_key: row.config_key } : null;
}
const STATUS_VERIFY = { argv: ['gbrain', 'maintain', '--status', '--json'] };
function skip(code: string, message: string, fix?: Action): UpkeepOutcome {
return { status: 'skipped', exit: 0, code, message, ...(fix ? { fix } : {}) };
}
/** Why upkeep cannot run right now, or null. Checked in this order before any work. */
async function preflight(engine: BrainEngine, opts: UpkeepOpts, now: Date): Promise<UpkeepOutcome | null> {
if (!await upkeepEnabled(engine)) {
return skip('maintenance_disabled', 'Unattended upkeep is off (maintenance.auto is false); nothing ran.');
}
if (isEngineDegraded(engine)) {
return skip('engine_degraded', 'The brain database is unreachable right now (degraded engine), so upkeep skipped this run.',
{ consent: [], actor: 'agent', requires_exclusive: false, why: 'Upkeep waits for the database to come back; doctor names the connection problem.', verify: { argv: ['gbrain', 'doctor', '--json'] } });
}
if (await hasPendingMigrations(engine)) {
return skip('migrations_pending', 'Schema migrations are pending on this brain, so upkeep skipped this run.',
{ argv: ['gbrain', 'apply-migrations', '--yes'], consent: [], actor: 'agent', requires_exclusive: true,
why: 'Applies the pending schema migrations (the same ones every command applies on connect); upkeep runs on its next turn.', verify: STATUS_VERIFY });
}
const owner = await autopilotOwnsUpkeep(engine, now);
if (owner) {
return skip('maintenance_owned_by_autopilot', `An autopilot keeps this brain maintained (its last brain-wide pass ran at ${owner}), so free upkeep left it alone.`);
}
const paused = await (opts.capacityPaused ?? defaultCapacityPaused)(engine).catch(() => null);
if (paused) {
return skip('write_capacity_backpressure', `A cumulative write cap (${paused.config_key}) passed the background pause line, so upkeep skipped this run.`,
{ argv: paused.command.split(' '), consent: [], actor: 'host_admin', requires_exclusive: false,
why: 'Raises the write cap on the brain host; background producers, upkeep included, resume below the pause line.', verify: { argv: ['gbrain', 'doctor', '--only', 'persistence_capacity', '--json'] } });
}
return null;
}
interface WorkItem { key: string; sourceId?: string; phases: CyclePhase[]; archive?: true }
/** The conversation archive's item (GST-2768): last in a round, only while the archive resolves on. */
export const ARCHIVE_ITEM_KEY = 'transcript_archive';
function phaseCode(phase: CycleReport['phases'][number]): string | undefined {
return phase.code ?? phase.error?.code;
}
function stepsFrom(report: CycleReport, scope: string): UpkeepStep[] {
return report.phases.map((phase) => {
const code = phase.status === 'fail' ? phaseCode(phase) ?? 'phase_failed' : phase.status === 'skipped' ? undefined : phaseCode(phase);
const reason = phase.status === 'skipped' ? String(phase.details?.reason ?? '') || undefined : undefined;
return { name: phase.phase, scope, status: phase.status, ...(code ? { code } : {}), ...(reason ? { reason } : {}) };
});
}
async function brainDirFor(engine: BrainEngine, sourceId: string, localPath: string | null | undefined): Promise<string | null> {
if (!localPath || !isAbsolute(localPath) || !existsSync(localPath)) return null;
const { syncContentDirectory } = await import('../sync-applicability.ts');
return (await syncContentDirectory(engine, { sourceId }).catch(() => null)) ? null : localPath;
}
function backedOff(failures: UpkeepFailure[], step: string, scope: string, now: Date): boolean {
const entry = failures.find((f) => f.step === step && f.scope === scope);
return !!entry && entry.count >= UPKEEP_BACKOFF_FAILURES && now.getTime() - Date.parse(entry.last_at) < UPKEEP_BACKOFF_MS;
}
/** One embed step at a time per brain, so overlapping runs cannot each spend the remaining daily cap. */
async function runEmbedStep(engine: BrainEngine, now: Date, signal: AbortSignal | undefined): Promise<{ step: UpkeepStep; embed: { spent_today_usd: number; cap_usd: number } }> {
const lock = await tryAcquireDbLock(engine, MAINTENANCE_EMBED_LOCK_ID, 60);
if (!lock) {
return { step: { name: 'embed', scope: 'global', status: 'skipped', reason: 'embed_running' },
embed: { spent_today_usd: await readEmbedSpentToday(engine, now), cap_usd: await readEmbedDailyCapUsd(engine) } };
}
try {
return await embedWithinDailyCap(engine, now, signal);
} finally {
await lock.release();
}
}
/** Reserves the rest of the day's cap in the ledger before any spend and settles it to the actual spend afterwards, so a run killed mid-step cannot spend the cap again. */
async function embedWithinDailyCap(engine: BrainEngine, now: Date, signal: AbortSignal | undefined): Promise<{ step: UpkeepStep; embed: { spent_today_usd: number; cap_usd: number } }> {
const cap = await readEmbedDailyCapUsd(engine);
const spentBefore = await readEmbedSpentToday(engine, now);
const embed = (spent: number) => ({ spent_today_usd: Math.round(spent * 1e6) / 1e6, cap_usd: cap });
if (cap <= 0) return { step: { name: 'embed', scope: 'global', status: 'skipped', reason: 'embed_cap_off' }, embed: embed(spentBefore) };
if (await engine.getConfig('embedding_disabled').catch(() => null) === 'true') {
return { step: { name: 'embed', scope: 'global', status: 'skipped', reason: 'embedding_disabled' }, embed: embed(spentBefore) };
}
const remaining = cap - spentBefore;
if (remaining <= 0) return { step: { name: 'embed', scope: 'global', status: 'skipped', reason: 'daily_cap_reached' }, embed: embed(spentBefore) };
const [{ embedCostCap }, { loadPricingOverrides }, { runEmbedCore }, { embedStaleFacts }] = await Promise.all([
import('../embed-cost-cap.ts'), import('../budget/budget-tracker.ts'), import('../../commands/embed.ts'), import('../embed-facts.ts')]);
const costCap = embedCostCap(remaining, 'user', await loadPricingOverrides(engine).catch(() => undefined))!;
await engine.setConfig(MAINTENANCE_EMBED_SPEND_KEY, JSON.stringify({ day: utcDay(now), usd: cap }));
const combined = anyAbortSignal(signal ? [costCap.signal, signal] : [costCap.signal]);
let spent = 0;
let status: UpkeepStepStatus = 'ok';
let code: string | undefined;
try {
const result = await costCap.metered(() => runEmbedCore(engine, { stale: true, quiet: true, signal: combined.signal }));
if (result.failures > 0 && !result.lock_skipped) status = 'warn';
spent = costCap.spent();
if (!costCap.hit() && !combined.signal.aborted) {
for (const source of await engine.listAllSources()) {
const left = remaining - spent;
if (left <= 0 || combined.signal.aborted) break;
try {
const facts = await embedStaleFacts(engine, { sourceId: source.id, yes: true, maxCostUsd: left, signal: combined.signal });
spent += facts.cost_usd;
if (facts.failures > 0) status = 'warn';
} catch (error) {
status = 'warn';
code ??= (error as { code?: string } | null)?.code ?? 'facts_embed_refused';
}
}
}
} catch (error) {
spent = Math.max(spent, costCap.spent());
if (!costCap.hit() && !combined.signal.aborted) {
status = 'fail';
code = (error as { code?: string } | null)?.code ?? 'embed_failed';
}
} finally {
combined.dispose();
}
const total = spentBefore + spent;
await engine.setConfig(MAINTENANCE_EMBED_SPEND_KEY, JSON.stringify({ day: utcDay(now), usd: total }));
const reason = costCap.hit() ? 'daily_cap_reached' : undefined;
return { step: { name: 'embed', scope: 'global', status, ...(code ? { code } : {}), ...(reason ? { reason } : {}) }, embed: embed(total) };
}
/**
* PGLite only: a top-level CHECKPOINT once a round finishes, outside every
* transaction, so the WAL the round wrote is folded into the data files
* while the brain is quiet instead of at the next open.
*/
async function checkpointStep(engine: BrainEngine): Promise<UpkeepStep> {
try {
await engine.executeRaw('CHECKPOINT');
return { name: 'checkpoint', scope: 'global', status: 'ok' };
} catch (error) {
return { name: 'checkpoint', scope: 'global', status: 'fail', code: (error as { code?: string } | null)?.code ?? 'checkpoint_failed' };
}
}
/**
* One unattended upkeep run. Never throws: a run that cannot start returns a
* skip with its code and fix, a failed step is recorded with its own code and
* the run moves on to the next item.
*/
export async function runUnattendedMaintenance(engine: BrainEngine, opts: UpkeepOpts): Promise<UpkeepOutcome> {
const now = (opts.now ?? (() => new Date()))();
try {
const refused = await preflight(engine, opts, now);
const prior = await readUpkeepRecord(engine);
if (refused) {
if (prior && refused.code) await engine.setConfig(MAINTENANCE_LAST_RUN_KEY, JSON.stringify({ ...prior, last_skip: { at: now.toISOString(), code: refused.code } }));
return refused;
}
const budgeted = opts.budgetMs !== undefined;
const sources = (await engine.listAllSources()).sort((a, b) => a.id.localeCompare(b.id));
const archiveOn = (await resolveArchiveEnabled(engine).catch(() => null))?.enabled === true
|| (await (await import('../transcripts/archive.ts')).archiveRescopeSkipped(engine).catch(() => null))?.code === 'archive_rescope_pending';
const keys = ['global', ...sources.map((source) => `source:${source.id}`), ...(archiveOn ? [ARCHIVE_ITEM_KEY] : [])];
const parked = (prior?.parked ?? []).filter((p) => keys.includes(p.key)).map((p) => ({ ...p }));
const retryOnly = !upkeepDue(prior, now);
const retrying = !budgeted || opts.retryParked || (!retryOnly && !prior?.cursor) ? parked.map((p) => p.key) : [];
if (retryOnly && retrying.length === 0) {
return { status: 'not_due', exit: 0, message: `Upkeep already ran at ${prior!.at}; next due at ${prior!.next_due_at}.`, record: prior! };
}
const plan = unattendedPlan(opts.providers ?? await readSafePhaseProviders(engine));
const round = prior?.cursor ?? (retryOnly
? { round_started_at: now.toISOString(), done: keys.filter((key) => !retrying.includes(key)), due_at: prior!.next_due_at }
: { round_started_at: now.toISOString(), done: [] as string[] });
const attempts: Record<string, number> = { ...round.attempts };
const retry = new Set([...(round.retry ?? []), ...retrying]);
for (const key of retrying) delete attempts[key];
const done = round.done.filter((key) => !retrying.includes(key));
const failures = prior?.consecutive_failures ?? [];
const items: WorkItem[] = [
{ key: 'global', phases: plan.global },
...sources.map((source) => ({ key: `source:${source.id}`, sourceId: source.id, phases: plan.source })),
...(archiveOn ? [{ key: ARCHIVE_ITEM_KEY, phases: [], archive: true as const }] : []),
].filter((item) => !done.includes(item.key));
const deadline = budgeted ? Date.now() + opts.budgetMs! : Infinity;
const steps: UpkeepStep[] = [];
const cycled = new Set<string>();
let lockBusy = false;
for (const item of items) {
if (Date.now() >= deadline || opts.signal?.aborted) break;
const scope = item.sourceId ?? 'global';
if (budgeted && !retry.has(item.key) && parked.some((p) => p.key === item.key)) {
steps.push({ name: 'cycle', scope, status: 'skipped', reason: 'parked' });
done.push(item.key);
continue;
}
const settle = (): void => {
done.push(item.key);
delete attempts[item.key];
retry.delete(item.key);
};
if (item.archive) {
// An archive error is recorded (transcripts.archive.last_maintenance, doctor) and never fails the pass.
const archive = opts.archive ?? (async (e: BrainEngine, o: { runner: string; now: Date }) => (await import('../transcripts/archive-maintenance.ts')).runArchiveMaintenance(e, o));
const result: ArchiveMaintenanceOutcome = await archive(engine, { runner: opts.runner, now }).catch((error: unknown) => ({ status: 'ran' as const, code: (error as { code?: string } | null)?.code ?? 'archive_maintenance_failed', message: String(error) }));
const failed = result.status === 'ran' && result.record?.status !== 'ok' && !!result.code;
steps.push({ name: ARCHIVE_ITEM_KEY, scope: 'global', status: result.status === 'ran' ? (failed ? 'warn' : 'ok') : 'skipped',
...(result.code ? (result.status === 'ran' ? { code: result.code } : { reason: result.code }) : result.status === 'not_due' ? { reason: 'not_due' } : {}) });
settle();
continue;
}
const unfinished = (): void => {
if (!budgeted || opts.signal?.aborted) return;
const slices = (attempts[item.key] ?? 0) + 1;
if (slices < UPKEEP_PARK_AFTER_SLICES) { attempts[item.key] = slices; return; }
const entry = parked.find((p) => p.key === item.key);
if (entry) entry.slices += slices;
else parked.push({ key: item.key, since: now.toISOString(), slices });
steps.push({ name: 'cycle', scope, status: 'skipped', reason: 'parked' });
settle();
};
const phases = item.phases.filter((phase) => !backedOff(failures, phase, scope, now));
for (const phase of item.phases) if (!phases.includes(phase)) steps.push({ name: phase, scope, status: 'skipped', reason: 'backed_off' });
const source = item.sourceId ? sources.find((s) => s.id === item.sourceId) : undefined;
const brainDir = source ? await brainDirFor(engine, source.id, source.local_path) : null;
const budget = Number.isFinite(deadline) ? AbortSignal.timeout(Math.max(1, deadline - Date.now())) : undefined;
const signals = [opts.signal, budget].filter((s): s is AbortSignal => s !== undefined);
const combined = signals.length ? anyAbortSignal(signals) : null;
const cycleOpts: CycleOpts = {
brainDir, pull: false, phases, stamp: 'none', deferFactEmbedding: plan.embedding,
...(item.sourceId ? { sourceId: item.sourceId } : { forceGlobalOrphans: true }),
...(combined ? { signal: combined.signal } : {}),
};
try {
if (phases.length > 0) {
const report = await (opts.cycle ?? runCycle)(engine, cycleOpts);
if (report.status === 'skipped' && report.reason === 'cycle_already_running') { lockBusy = true; break; }
cycled.add(scope);
steps.push(...stepsFrom(report, scope));
if (report.reason === 'aborted') { unfinished(); break; }
if (report.reason === 'lock_stolen') break;
}
settle();
const at = parked.findIndex((p) => p.key === item.key);
if (at >= 0) parked.splice(at, 1);
} catch (error) {
if (combined?.signal.aborted) { unfinished(); break; }
steps.push({ name: 'cycle', scope, status: 'fail', code: (error as { code?: string } | null)?.code ?? 'cycle_failed' });
settle();
} finally {
combined?.dispose();
}
}
if (lockBusy && steps.length === 0) {
const out = skip('cycle_already_running', 'Another maintenance cycle holds the brain\'s cycle lock, so upkeep skipped this run.',
{ argv: STATUS_VERIFY.argv, consent: [], actor: 'provider', requires_exclusive: false, why: 'The other cycle releases the lock when it finishes; upkeep runs again on its next turn.', verify: STATUS_VERIFY });
if (prior) await engine.setConfig(MAINTENANCE_LAST_RUN_KEY, JSON.stringify({ ...prior, last_skip: { at: now.toISOString(), code: 'cycle_already_running' } }));
return out;
}
const finished = items.every((item) => done.includes(item.key));
let embed: UpkeepRecord['embed'];
if (finished && plan.embedding && !opts.signal?.aborted) {
if (Date.now() >= deadline) {
steps.push({ name: 'embed', scope: 'global', status: 'skipped', reason: 'budget_exhausted' });
} else {
const budget = Number.isFinite(deadline)
? anyAbortSignal([...(opts.signal ? [opts.signal] : []), AbortSignal.timeout(Math.max(1, deadline - Date.now()))]) : null;
try {
const result = await runEmbedStep(engine, now, budget?.signal ?? opts.signal);
steps.push(result.step);
embed = result.embed;
} finally {
budget?.dispose();
}
}
}
if (finished && engine.kind === 'pglite' && !opts.signal?.aborted) steps.push(await checkpointStep(engine));
const failed = steps.filter((step) => step.status === 'fail');
const planned = new Map<string, string[]>([
...sources.map((source): [string, string[]] => [source.id, [...plan.source, 'cycle']]),
['global', [...plan.global, 'cycle', ...(plan.embedding ? ['embed'] : []), ...(engine.kind === 'pglite' ? ['checkpoint'] : [])]],
]);
const nextFailures: UpkeepFailure[] = failures
.filter((f) => planned.get(f.scope)?.includes(f.step) && !(f.step === 'cycle' && cycled.has(f.scope)))
.filter((f) => !steps.some((s) => s.name === f.step && s.scope === f.scope && s.status !== 'fail' && s.reason !== 'backed_off' && s.reason !== 'parked'));
for (const step of failed) {
const entry = nextFailures.find((f) => f.step === step.name && f.scope === step.scope);
if (entry && entry.code === step.code) { entry.count += 1; entry.last_at = now.toISOString(); }
else if (entry) { entry.code = step.code!; entry.count = 1; entry.last_at = now.toISOString(); }
else nextFailures.push({ step: step.name, scope: step.scope, code: step.code!, count: 1, last_at: now.toISOString() });
}
const record: UpkeepRecord = {
at: now.toISOString(),
runner: opts.runner,
status: failed.length === 0 ? 'ok' : failed.length === steps.filter((s) => s.status !== 'skipped').length ? 'failed' : 'partial',
steps,
held: plan.held,
cursor: finished ? null : {
round_started_at: round.round_started_at, done,
...(Object.keys(attempts).length ? { attempts } : {}), ...(retry.size ? { retry: [...retry] } : {}), ...(round.due_at ? { due_at: round.due_at } : {}),
},
next_due_at: finished ? round.due_at ?? new Date(now.getTime() + UPKEEP_INTERVAL_MS).toISOString() : now.toISOString(),
consecutive_failures: nextFailures,
...(parked.length ? { parked } : {}),
...(embed ? { embed } : {}),
};
await engine.setConfig(MAINTENANCE_LAST_RUN_KEY, JSON.stringify(record));
const first = failed[0];
return {
status: 'ran',
exit: first ? 1 : 0,
...(first?.code ? { code: first.code } : {}),
message: first
? `Upkeep ran ${steps.length} step(s); ${failed.length} failed, first ${first.name} (${first.scope}) with ${first.code}.`
: `Upkeep ran ${steps.length} step(s)${finished ? '' : '; the round continues on the next run'}.`,
record,
};
} catch (error) {
const code = (error as { code?: string } | null)?.code ?? 'maintenance_failed';
return { status: 'ran', exit: 1, code, message: `Upkeep stopped: ${error instanceof Error ? error.message : String(error)}` };
}
}