Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript

Wakeshift drainer: a watch that dies from drain errors is ended at Wakeshift; the final jobs-work drain keeps going past a failing pass; watch_job's shared-brain refusal omits the member count

garrytanc31c923f2fe1
diff --git a/src/core/ops/wakeshift.ts b/src/core/ops/wakeshift.ts
index 3439abdafa..b406171f75 100644
--- a/src/core/ops/wakeshift.ts
+++ b/src/core/ops/wakeshift.ts
@@ -106,9 +106,8 @@ const watch_job: Operation = {
}
const runtime = wakeshiftRuntime();
if (!runtime) throw notEnabled();
- const members = await liveAccessMembers(ctx.engine);
- if (members > 0) {
- throw opError('wakeshift_shared_brain', `This brain now has ${members} live access member(s), so reporting back is paused; nothing was watched.`,
+ if (await liveAccessMembers(ctx.engine) > 0) {
+ throw opError('wakeshift_shared_brain', 'This brain now has live access members, so reporting back is paused; nothing was watched.',
`Call wait_job ${id} to wait for the job.`, { fix: waitFix(id) });
}
const owner = agentOwnerFence(ctx, 'watch_job');
diff --git a/src/core/wakeshift/drain.ts b/src/core/wakeshift/drain.ts
index 94fa266af2..1428ae761d 100644
--- a/src/core/wakeshift/drain.ts
+++ b/src/core/wakeshift/drain.ts
@@ -155,7 +155,9 @@ async function deliver(engine: BrainEngine, api: WakeshiftApi, row: WakeRow, rep
}
} catch (e) {
const dead = attempts >= WAKE_MAX_ATTEMPTS;
- await settle(engine, row.id, lease, dead ? 'dead' : 'open', { attempts, ...(dead ? {} : { delayS: retryDelayS(attempts) }), error: 'drain_error' }).catch(() => false);
+ if (dead) await endAtWakeshift(api, row);
+ const settled = await settle(engine, row.id, lease, dead ? 'dead' : 'open', { attempts, ...(dead ? {} : { delayS: retryDelayS(attempts) }), error: 'drain_error' }).catch(() => false);
+ if (settled && dead) report.dead++;
throw e;
}
}
@@ -185,12 +187,15 @@ export async function drainWakes(engine: BrainEngine, opts: { ids?: number[]; ap
/** Passes until one has nothing left to settle (at most 10): the final pass of a process about to exit (`gbrain jobs work`). */
export async function drainWakesUntilIdle(engine: BrainEngine): Promise<DrainReport> {
const total: DrainReport = { delivered: 0, retried: 0, dead: 0, ended: 0 };
+ let failure: unknown = null;
for (let i = 0; i < 10; i++) {
- const r = await drainWakes(engine);
+ let r: DrainReport;
+ try { r = await drainWakes(engine); } catch (e) { failure ??= e; continue; }
if (r.skipped) return { ...total, skipped: r.skipped };
total.delivered += r.delivered; total.retried += r.retried; total.dead += r.dead; total.ended += r.ended;
if (r.delivered + r.dead + r.ended === 0) break;
}
+ if (failure !== null) throw failure;
return total;
}
diff --git a/test/wakeshift.serial.test.ts b/test/wakeshift.serial.test.ts
index 5802eb596d..2d4f58ee47 100644
--- a/test/wakeshift.serial.test.ts
+++ b/test/wakeshift.serial.test.ts
@@ -264,6 +264,11 @@ describe('failure schedule', () => {
expect(rows[0]).toMatchObject({ target: String(a), state: 'open', attempts: 1, delay: 30, lease: false });
expect(rows[1]).toMatchObject({ target: String(b), state: 'delivered' });
expect(emits()).toHaveLength(1);
+ await engine.executeRaw("UPDATE wake_subscriptions SET attempts = $1, next_attempt_at = now() - interval '1 second' WHERE target = $2", [WAKE_MAX_ATTEMPTS - 1, String(a)]);
+ await expect(drainWakes(poisoned)).rejects.toThrow('lookup failed');
+ const [last] = await engine.executeRaw<{ state: string; last_error: string }>('SELECT state, last_error FROM wake_subscriptions WHERE target = $1', [String(a)]);
+ expect(last).toMatchObject({ state: 'dead', last_error: 'drain_error' });
+ expect(fake.calls.filter(c => c.path === '/event-subscriptions/end')).toHaveLength(1);
});
test('an emit nobody matched ends the watch instead of calling it delivered; a duplicate counts as delivered', async () => {
@@ -291,6 +296,7 @@ describe('failure schedule', () => {
expect(r.isError).toBe(true);
const env = JSON.parse(r.content[0]!.text);
expect(env.code ?? env.error).toBe('wakeshift_shared_brain');
+ expect(JSON.stringify(env)).not.toMatch(/\d+ live access member/);
expect(await drainWakes(engine)).toMatchObject({ skipped: 'shared' });
expect(emits()).toHaveLength(0);
expect(fake.calls.filter(c => c.path === '/event-subscriptions')).toHaveLength(1);