Garry's Opinionated OpenClaw/Hermes Agent Brain
TypeScript
Clear an upkeep failure streak once its step leaves the plan or its cycle runs clean
garrytan864eb5a428cc
diff --git a/src/core/maintenance/run.ts b/src/core/maintenance/run.tsindex 24018975ae..0f3af02e01 100644--- a/src/core/maintenance/run.ts+++ b/src/core/maintenance/run.ts@@ -351,6 +351,7 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeepconst deadline = opts.budgetMs !== undefined ? Date.now() + opts.budgetMs : Infinity;const steps: UpkeepStep[] = [];const done = [...round.done];+ const cycled = new Set<string>();let lockBusy = false;for (const item of items) {if (Date.now() >= deadline || opts.signal?.aborted) break;@@ -371,6 +372,7 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeepif (phases.length > 0) {const report = await (opts.cycle ?? runCycle)(engine, cycleOpts);if (report.status === 'skipped' && report.reason === 'cycle_already_running') { lockBusy = true; break; }+ cycled.add(scope);steps.push(...stepsFrom(report, scope));if (report.reason === 'aborted' || report.reason === 'lock_stolen') break;}@@ -397,7 +399,12 @@ export async function runUnattendedMaintenance(engine: BrainEngine, opts: Upkeepembed = result.embed;}const failed = steps.filter((step) => step.status === 'fail');+ const planned = new Map<string, string[]>([+ ...sources.map((source): [string, string[]] => [source.id, [...plan.source, 'cycle']]),+ ['global', [...plan.global, 'cycle', ...(plan.embedding ? ['embed'] : [])]],+ ]);const nextFailures: UpkeepFailure[] = failures+ .filter((f) => planned.get(f.scope)?.includes(f.step) && !(f.step === 'cycle' && cycled.has(f.scope))).filter((f) => !steps.some((s) => s.name === f.step && s.scope === f.scope && s.status !== 'fail' && s.reason !== 'backed_off'));for (const step of failed) {const entry = nextFailures.find((f) => f.step === step.name && f.scope === step.scope);diff --git a/test/maintenance-unattended.test.ts b/test/maintenance-unattended.test.tsindex e5914bc7b6..9024280ced 100644--- a/test/maintenance-unattended.test.ts+++ b/test/maintenance-unattended.test.ts@@ -261,6 +261,42 @@ for (const kind of testBackends()) {expect(check).toMatchObject({ status: 'warn', details: { state: 'failing' } });expect(check.message).toContain('orphans');});++ test('a cycle that threw three times reads failing until the next clean run clears it', async () => {+ let throws = 3;+ const flaky: typeof runCycle = async (e, o) => {+ if (throws > 0 && !o.sourceId) { throws -= 1; throw Object.assign(new Error('connection reset'), { code: 'db_connection_reset' }); }+ return runCycle(e, o);+ };+ let at = Date.now();+ for (let i = 0; i < 3; i++) {+ const outcome = await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: flaky, now: () => new Date(at) });+ expect(outcome).toMatchObject({ status: 'ran', exit: 1, code: 'db_connection_reset' });+ at += UPKEEP_INTERVAL_MS + HOUR;+ }+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'warn', details: { state: 'failing' } });+ const clean = await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: flaky, now: () => new Date(at) });+ expect(clean).toMatchObject({ status: 'ran', exit: 0 });+ expect(clean.record!.consecutive_failures).toEqual([]);+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'ok', details: { state: 'ok' } });+ });++ test('a failing step that is held back afterwards no longer keeps doctor failing', async () => {+ const failing: typeof runCycle = async (e, o) => {+ const report = await runCycle(e, o);+ return { ...report, phases: report.phases.map((p) => p.phase === 'fence_repair' ? { ...p, status: 'fail' as const, code: 'fence_probe_failed' } : p) };+ };+ let at = Date.now();+ for (let i = 0; i < 3; i++) {+ await upkeep(engine, { runner: 'test', providers: KEYLESS, cycle: failing, now: () => new Date(at) });+ at += UPKEEP_INTERVAL_MS + HOUR;+ }+ expect((await readUpkeepRecord(engine))!.consecutive_failures).toEqual([expect.objectContaining({ step: 'fence_repair', count: 3 })]);+ const held = await upkeep(engine, { runner: 'test', providers: { ...KEYLESS, fenceRepairModel: true }, now: () => new Date(at) });+ expect(held.record!.held).toContain('fence_repair');+ expect(held.record!.consecutive_failures).toEqual([]);+ expect(await checkUnattendedMaintenance(engine, { nowMs: at })).toMatchObject({ status: 'ok', details: { state: 'ok' } });+ });});describe('decision D: the embedding cap', () => {