Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.
TypeScript
fix(scheduler): a board wake ran the whole turn twice (#295)
Xialie Zhuang2dc0e54acfac
diff --git a/server/src/__tests__/scheduler-post-spawn-retry.test.ts b/server/src/__tests__/scheduler-post-spawn-retry.test.tsnew file mode 100644index 0000000..f9da88c--- /dev/null+++ b/server/src/__tests__/scheduler-post-spawn-retry.test.ts@@ -0,0 +1,121 @@+/**+ * A confirmed wake delivery has to retire the retry that was covering for it.+ *+ * `wakeOne` does two things after spinning up a pod: it queues a "post-spawn+ * health check" retry, and it runs an inline replay loop that re-delivers until+ * a subscriber answers. Both exist for the same reason — kubelet can reject the+ * allocation right after scheduling, so the pod may never reach the wake+ * stream. Only the loop can CONFIRM the pod attached.+ *+ * The queued job was never retired when it did, so the same wake went out+ * twice. The comment above the schedule call says why that was believed to be+ * harmless: "if the pod is healthy, the retry just delivers a wake and the+ * inbox fingerprint makes the turn no-op."+ *+ * That premise holds for `message.new` — and `message.new` is the one wake+ * class that never enters this queue. `_shouldRetryEnsurePodFailure` admits+ * `manual` and nothing else; the only producer of `manual` in the product is a+ * board card wake (kanban-wake.ts), which always carries a `backgroundBrief`;+ * and the briefed-manual fingerprint in turn.ts embeds `new Date()`, so it can+ * never match the previous turn's. So the only wake class that reaches the+ * queue is exactly the one the premise fails for: every board action on a+ * resting agent ran the whole turn twice — re-claiming the card, re-commenting,+ * and billing a second model call.+ *+ * Run: node --import tsx --test server/src/__tests__/scheduler-post-spawn-retry.test.ts+ */+import { test, before, after } from 'node:test'+import assert from 'node:assert/strict'+import { readFile } from 'node:fs/promises'+import { pool } from '../db/pool.js'+import { _cancelWakeRetry } from '../agents/scheduler.js'+import { redis } from '../redis.js'++// The keys scheduleWakeRetry writes. Pinned literally on purpose: if they are+// renamed, _cancelWakeRetry deletes from the new names while these seeds sit in+// the old ones, and these tests fail rather than quietly passing.+const DUE_KEY = 'cumora:wake-retry:due'+const JOB_KEY = 'cumora:wake-retry:jobs'++before(async () => {+ // The shared client connects lazily and runs with enableOfflineQueue: false,+ // so a command issued before it is ready throws rather than queueing.+ if (redis.status !== 'ready') {+ await new Promise<void>((resolve, reject) => {+ const done = () => { clearTimeout(timer); resolve() }+ const timer = setTimeout(() => reject(new Error('redis did not become ready')), 10_000)+ redis.once('ready', done)+ if (redis.status === 'ready') done()+ })+ }+})++after(async () => {+ try { await pool.end() } catch { /* ignore */ }+ try {+ const { sub } = await import('../redis.js')+ redis.disconnect()+ sub.disconnect()+ } catch { /* ignore */ }+})++async function seedJob(id: string): Promise<void> {+ await redis.hset(JOB_KEY, id, JSON.stringify({ id, attempt: 1 }))+ await redis.zadd(DUE_KEY, Date.now() + 10_000, id)+}++async function queued(id: string): Promise<boolean> {+ const [job, score] = await Promise.all([redis.hget(JOB_KEY, id), redis.zscore(DUE_KEY, id)])+ return job !== null || score !== null+}++test('a confirmed delivery retires the queued post-spawn retry', async () => {+ const id = 'a-board:manual:-'+ await seedJob(id)+ assert.equal(await queued(id), true, 'seed failed')++ await _cancelWakeRetry('a-board', 'manual', null)++ assert.equal(+ await queued(id), false,+ 'the health-check retry outlived the delivery it was covering for — the board brief goes out twice',+ )+})++test('it retires only the job for that agent, reason and conversation', async () => {+ // The guard rail: this must not become "clear the queue".+ const mine = 'a-one:manual:-'+ const peer = 'a-two:manual:-'+ const otherConvo = 'a-one:manual:c-42'+ for (const id of [mine, peer, otherConvo]) await seedJob(id)++ await _cancelWakeRetry('a-one', 'manual', null)++ assert.equal(await queued(mine), false)+ assert.equal(await queued(peer), true, 'another agent\'s retry was dropped')+ assert.equal(await queued(otherConvo), true, 'a retry for a different conversation was dropped')++ for (const id of [peer, otherConvo]) {+ await redis.hdel(JOB_KEY, id)+ await redis.zrem(DUE_KEY, id)+ }+})++test('cancelling a job that is not queued is a no-op, not an error', async () => {+ await _cancelWakeRetry('a-never-queued', 'manual', null)+})++test('wakeOne retires the retry where the replay loop confirms delivery', async () => {+ // The tests above exercise the helper. They would all still pass against a+ // wakeOne that never calls it — which is the state being fixed. Read the+ // source: the cancel has to sit on the confirmed-delivery branch of the+ // replay loop, not somewhere it fires regardless.+ const source = await readFile(new URL('../agents/scheduler.ts', import.meta.url), 'utf8')+ const loop = source.slice(source.indexOf('const replayed = await deliverWake'))+ const branch = loop.slice(0, loop.indexOf('return true') + 'return true'.length)++ assert.match(+ branch, /_cancelWakeRetry\(agentId, reason, conversationId\)/,+ 'a confirmed replay no longer retires the health-check retry — the same wake will be delivered twice',+ )+})diff --git a/server/src/agents/scheduler.ts b/server/src/agents/scheduler.tsindex 9fec6cd..d6db76a 100644--- a/server/src/agents/scheduler.ts+++ b/server/src/agents/scheduler.ts@@ -183,6 +183,38 @@ async function scheduleWakeRetry(console.warn(`[scheduler] ${agentId} ${reason} wake retry scheduled in ${Math.round((dueAt - Date.now()) / 1000)}s after ${failureClass}: ${failureReason}`)}+/** Drop a queued retry whose job has already been done.+ *+ * The post-spawn health check and the inline replay loop at the end of+ * `wakeOne` are two mechanisms for one thing: make sure a freshly-spawned pod+ * actually receives this wake. Only the loop can CONFIRM it — it re-delivers+ * until a subscriber answers. Leaving the queued job behind after that+ * confirmation delivers the same wake a second time.+ *+ * For a durable `message.new` that was harmless, which is why it went+ * unnoticed: the pod's cold-start drain reads the inbox and the fingerprint+ * absorbs the repeat. But `message.new` never enters this queue —+ * `_shouldRetryEnsurePodFailure` admits `manual` and nothing else, the only+ * producer of `manual` in the product is a board card wake, and that always+ * carries a `backgroundBrief` whose fingerprint embeds `new Date()`+ * (turn.ts). So the one wake class that reaches the queue is exactly the one+ * the "fingerprint makes the turn no-op" premise does not hold for: every+ * board action on a resting agent ran the whole turn twice, re-claiming the+ * card, re-commenting, and billing a second time.+ *+ * The id is deterministic and `scheduleWakeRetry` hsets it, so there is at+ * most one job per (agent, reason, conversation) and this removes exactly the+ * one whose work just completed. */+export async function _cancelWakeRetry(+ agentId: string,+ reason: WakeReason,+ conversationId: string | null,+): Promise<void> {+ const id = wakeRetryId(agentId, reason, conversationId)+ await redis.zrem(WAKE_RETRY_DUE_KEY, id).catch(() => { /* best effort */ })+ await redis.hdel(WAKE_RETRY_JOB_KEY, id).catch(() => { /* best effort */ })+}+/** The Redis side of the retry queue, behind an interface so the drain loop can* be driven by a test without a live Redis or a live agent fleet. */export interface WakeRetryQueue {@@ -622,7 +654,13 @@ async function wakeOne(while (Date.now() < deadline) {await new Promise((resolve) => setTimeout(resolve, 500))const replayed = await deliverWake(agentId, wakePayload).catch(() => 0)- if (replayed > 0) return true+ if (replayed > 0) {+ // Confirmed: the pod attached and took this wake. The health-check+ // retry was the net for "it never attached" — drop it before it+ // delivers the same brief again.+ await _cancelWakeRetry(agentId, reason, conversationId)+ return true+ }}console.warn(`[scheduler] ${agentId} synthetic wake (${reason}) was not delivered after pod start`)return false