Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(scheduler): a board wake ran the whole turn twice (#295)

Xialie Zhuang2dc0e54acfac
diff --git a/server/src/__tests__/scheduler-post-spawn-retry.test.ts b/server/src/__tests__/scheduler-post-spawn-retry.test.ts
new file mode 100644
index 0000000..f9da88c
--- /dev/null
+++ b/server/src/__tests__/scheduler-post-spawn-retry.test.ts
@@ -0,0 +1,121 @@
+/**
+ * A confirmed wake delivery has to retire the retry that was covering for it.
+ *
+ * `wakeOne` does two things after spinning up a pod: it queues a "post-spawn
+ * health check" retry, and it runs an inline replay loop that re-delivers until
+ * a subscriber answers. Both exist for the same reason — kubelet can reject the
+ * allocation right after scheduling, so the pod may never reach the wake
+ * stream. Only the loop can CONFIRM the pod attached.
+ *
+ * The queued job was never retired when it did, so the same wake went out
+ * twice. The comment above the schedule call says why that was believed to be
+ * harmless: "if the pod is healthy, the retry just delivers a wake and the
+ * inbox fingerprint makes the turn no-op."
+ *
+ * That premise holds for `message.new` — and `message.new` is the one wake
+ * class that never enters this queue. `_shouldRetryEnsurePodFailure` admits
+ * `manual` and nothing else; the only producer of `manual` in the product is a
+ * board card wake (kanban-wake.ts), which always carries a `backgroundBrief`;
+ * and the briefed-manual fingerprint in turn.ts embeds `new Date()`, so it can
+ * never match the previous turn's. So the only wake class that reaches the
+ * queue is exactly the one the premise fails for: every board action on a
+ * resting agent ran the whole turn twice — re-claiming the card, re-commenting,
+ * and billing a second model call.
+ *
+ * Run: node --import tsx --test server/src/__tests__/scheduler-post-spawn-retry.test.ts
+ */
+import { test, before, after } from 'node:test'
+import assert from 'node:assert/strict'
+import { readFile } from 'node:fs/promises'
+import { pool } from '../db/pool.js'
+import { _cancelWakeRetry } from '../agents/scheduler.js'
+import { redis } from '../redis.js'
+
+// The keys scheduleWakeRetry writes. Pinned literally on purpose: if they are
+// renamed, _cancelWakeRetry deletes from the new names while these seeds sit in
+// the old ones, and these tests fail rather than quietly passing.
+const DUE_KEY = 'cumora:wake-retry:due'
+const JOB_KEY = 'cumora:wake-retry:jobs'
+
+before(async () => {
+ // The shared client connects lazily and runs with enableOfflineQueue: false,
+ // so a command issued before it is ready throws rather than queueing.
+ if (redis.status !== 'ready') {
+ await new Promise<void>((resolve, reject) => {
+ const done = () => { clearTimeout(timer); resolve() }
+ const timer = setTimeout(() => reject(new Error('redis did not become ready')), 10_000)
+ redis.once('ready', done)
+ if (redis.status === 'ready') done()
+ })
+ }
+})
+
+after(async () => {
+ try { await pool.end() } catch { /* ignore */ }
+ try {
+ const { sub } = await import('../redis.js')
+ redis.disconnect()
+ sub.disconnect()
+ } catch { /* ignore */ }
+})
+
+async function seedJob(id: string): Promise<void> {
+ await redis.hset(JOB_KEY, id, JSON.stringify({ id, attempt: 1 }))
+ await redis.zadd(DUE_KEY, Date.now() + 10_000, id)
+}
+
+async function queued(id: string): Promise<boolean> {
+ const [job, score] = await Promise.all([redis.hget(JOB_KEY, id), redis.zscore(DUE_KEY, id)])
+ return job !== null || score !== null
+}
+
+test('a confirmed delivery retires the queued post-spawn retry', async () => {
+ const id = 'a-board:manual:-'
+ await seedJob(id)
+ assert.equal(await queued(id), true, 'seed failed')
+
+ await _cancelWakeRetry('a-board', 'manual', null)
+
+ assert.equal(
+ await queued(id), false,
+ 'the health-check retry outlived the delivery it was covering for — the board brief goes out twice',
+ )
+})
+
+test('it retires only the job for that agent, reason and conversation', async () => {
+ // The guard rail: this must not become "clear the queue".
+ const mine = 'a-one:manual:-'
+ const peer = 'a-two:manual:-'
+ const otherConvo = 'a-one:manual:c-42'
+ for (const id of [mine, peer, otherConvo]) await seedJob(id)
+
+ await _cancelWakeRetry('a-one', 'manual', null)
+
+ assert.equal(await queued(mine), false)
+ assert.equal(await queued(peer), true, 'another agent\'s retry was dropped')
+ assert.equal(await queued(otherConvo), true, 'a retry for a different conversation was dropped')
+
+ for (const id of [peer, otherConvo]) {
+ await redis.hdel(JOB_KEY, id)
+ await redis.zrem(DUE_KEY, id)
+ }
+})
+
+test('cancelling a job that is not queued is a no-op, not an error', async () => {
+ await _cancelWakeRetry('a-never-queued', 'manual', null)
+})
+
+test('wakeOne retires the retry where the replay loop confirms delivery', async () => {
+ // The tests above exercise the helper. They would all still pass against a
+ // wakeOne that never calls it — which is the state being fixed. Read the
+ // source: the cancel has to sit on the confirmed-delivery branch of the
+ // replay loop, not somewhere it fires regardless.
+ const source = await readFile(new URL('../agents/scheduler.ts', import.meta.url), 'utf8')
+ const loop = source.slice(source.indexOf('const replayed = await deliverWake'))
+ const branch = loop.slice(0, loop.indexOf('return true') + 'return true'.length)
+
+ assert.match(
+ branch, /_cancelWakeRetry\(agentId, reason, conversationId\)/,
+ 'a confirmed replay no longer retires the health-check retry — the same wake will be delivered twice',
+ )
+})
diff --git a/server/src/agents/scheduler.ts b/server/src/agents/scheduler.ts
index 9fec6cd..d6db76a 100644
--- a/server/src/agents/scheduler.ts
+++ b/server/src/agents/scheduler.ts
@@ -183,6 +183,38 @@ async function scheduleWakeRetry(
console.warn(`[scheduler] ${agentId} ${reason} wake retry scheduled in ${Math.round((dueAt - Date.now()) / 1000)}s after ${failureClass}: ${failureReason}`)
}
+/** Drop a queued retry whose job has already been done.
+ *
+ * The post-spawn health check and the inline replay loop at the end of
+ * `wakeOne` are two mechanisms for one thing: make sure a freshly-spawned pod
+ * actually receives this wake. Only the loop can CONFIRM it — it re-delivers
+ * until a subscriber answers. Leaving the queued job behind after that
+ * confirmation delivers the same wake a second time.
+ *
+ * For a durable `message.new` that was harmless, which is why it went
+ * unnoticed: the pod's cold-start drain reads the inbox and the fingerprint
+ * absorbs the repeat. But `message.new` never enters this queue —
+ * `_shouldRetryEnsurePodFailure` admits `manual` and nothing else, the only
+ * producer of `manual` in the product is a board card wake, and that always
+ * carries a `backgroundBrief` whose fingerprint embeds `new Date()`
+ * (turn.ts). So the one wake class that reaches the queue is exactly the one
+ * the "fingerprint makes the turn no-op" premise does not hold for: every
+ * board action on a resting agent ran the whole turn twice, re-claiming the
+ * card, re-commenting, and billing a second time.
+ *
+ * The id is deterministic and `scheduleWakeRetry` hsets it, so there is at
+ * most one job per (agent, reason, conversation) and this removes exactly the
+ * one whose work just completed. */
+export async function _cancelWakeRetry(
+ agentId: string,
+ reason: WakeReason,
+ conversationId: string | null,
+): Promise<void> {
+ const id = wakeRetryId(agentId, reason, conversationId)
+ await redis.zrem(WAKE_RETRY_DUE_KEY, id).catch(() => { /* best effort */ })
+ await redis.hdel(WAKE_RETRY_JOB_KEY, id).catch(() => { /* best effort */ })
+}
+
/** The Redis side of the retry queue, behind an interface so the drain loop can
* be driven by a test without a live Redis or a live agent fleet. */
export interface WakeRetryQueue {
@@ -622,7 +654,13 @@ async function wakeOne(
while (Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 500))
const replayed = await deliverWake(agentId, wakePayload).catch(() => 0)
- if (replayed > 0) return true
+ if (replayed > 0) {
+ // Confirmed: the pod attached and took this wake. The health-check
+ // retry was the net for "it never attached" — drop it before it
+ // delivers the same brief again.
+ await _cancelWakeRetry(agentId, reason, conversationId)
+ return true
+ }
}
console.warn(`[scheduler] ${agentId} synthetic wake (${reason}) was not delivered after pod start`)
return false