Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Clear an upkeep failure streak once its step leaves the plan or its cycle runs clean

garrytan864eb5a428cc
diff --git a/src/core/maintenance/run.ts b/src/core/maintenance/run.ts
index 24018975ae..0f3af02e01 100644
--- a/src/core/maintenance/run.ts
+++ b/src/core/maintenance/run.ts
@@ -351,6 +351,7 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeep
const deadline = opts.budgetMs !== undefined ? Date.now() + opts.budgetMs : Infinity;
const steps: UpkeepStep[] = [];
const done = [...round.done];
+ const cycled = new Set<string>();
let lockBusy = false;
for (const item of items) {
if (Date.now() >= deadline || opts.signal?.aborted) break;
@@ -371,6 +372,7 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeep
if (phases.length > 0) {
const report = await (opts.cycle ?? runCycle)(engine, cycleOpts);
if (report.status === 'skipped' && report.reason === 'cycle_already_running') { lockBusy = true; break; }
+ cycled.add(scope);
steps.push(...stepsFrom(report, scope));
if (report.reason === 'aborted' || report.reason === 'lock_stolen') break;
}
@@ -397,7 +399,12 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeep
embed = result.embed;
}
const failed = steps.filter((step) => step.status === 'fail');
+ const planned = new Map<string, string[]>([
+ ...sources.map((source): [string, string[]] => [source.id, [...plan.source, 'cycle']]),
+ ['global', [...plan.global, 'cycle', ...(plan.embedding ? ['embed'] : [])]],
+ ]);
const nextFailures: UpkeepFailure[] = failures
+ .filter((f) => planned.get(f.scope)?.includes(f.step) && !(f.step === 'cycle' && cycled.has(f.scope)))
.filter((f) => !steps.some((s) => s.name === f.step && s.scope === f.scope && s.status !== 'fail' && s.reason !== 'backed_off'));
for (const step of failed) {
const entry = nextFailures.find((f) => f.step === step.name && f.scope === step.scope);
diff --git a/test/maintenance-unattended.test.ts b/test/maintenance-unattended.test.ts
index e5914bc7b6..9024280ced 100644
--- a/test/maintenance-unattended.test.ts
+++ b/test/maintenance-unattended.test.ts
@@ -261,6 +261,42 @@ for (const kind of testBackends()) {
expect(check).toMatchObject({ status: 'warn', details: { state: 'failing' } });
expect(check.message).toContain('orphans');
});
+
+ test('a cycle that threw three times reads failing until the next clean run clears it', async () => {
+ let throws = 3;
+ const flaky: typeof runCycle = async (e, o) => {
+ if (throws > 0 && !o.sourceId) { throws -= 1; throw Object.assign(new Error('connection reset'), { code: 'db_connection_reset' }); }
+ return runCycle(e, o);
+ };
+ let at = Date.now();
+ for (let i = 0; i < 3; i++) {
+ const outcome = await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: flaky, now: () => new Date(at) });
+ expect(outcome).toMatchObject({ status: 'ran', exit: 1, code: 'db_connection_reset' });
+ at += UPKEEP_INTERVAL_MS + HOUR;
+ }
+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'warn', details: { state: 'failing' } });
+ const clean = await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: flaky, now: () => new Date(at) });
+ expect(clean).toMatchObject({ status: 'ran', exit: 0 });
+ expect(clean.record!.consecutive_failures).toEqual([]);
+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'ok', details: { state: 'ok' } });
+ });
+
+ test('a failing step that is held back afterwards no longer keeps doctor failing', async () => {
+ const failing: typeof runCycle = async (e, o) => {
+ const report = await runCycle(e, o);
+ return { ...report, phases: report.phases.map((p) => p.phase === 'fence_repair' ? { ...p, status: 'fail' as const, code: 'fence_probe_failed' } : p) };
+ };
+ let at = Date.now();
+ for (let i = 0; i < 3; i++) {
+ await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: failing, now: () => new Date(at) });
+ at += UPKEEP_INTERVAL_MS + HOUR;
+ }
+ expect((await readUpkeepRecord(engine))!.consecutive_failures).toEqual([expect.objectContaining({ step: 'fence_repair', count: 3 })]);
+ const held = await upkeep(engine, { runner: 'test', providers: { ...KEYLESS, fenceRepairModel: true }, now: () => new Date(at) });
+ expect(held.record!.held).toContain('fence_repair');
+ expect(held.record!.consecutive_failures).toEqual([]);
+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'ok', details: { state: 'ok' } });
+ });
});
describe('decision D: the embedding cap', () => {