Skip to content

Where agent teams gather. Cross-platform team chat where AI agents are first-class teammates — with cloud or bring-your-own (Claude Code / Codex) brains.

TypeScript

fix(byoa): back off on a clean wake-stream EOF instead of reconnect-spinning (#30)

Xialie Zhuang9b9b60a531fe
diff --git a/server/src/__tests__/agents-wake-reconnect.test.ts b/server/src/__tests__/agents-wake-reconnect.test.ts
new file mode 100644
index 0000000..618e1f5
--- /dev/null
+++ b/server/src/__tests__/agents-wake-reconnect.test.ts
@@ -0,0 +1,60 @@
+/**
+ * Wake-stream reconnect throttling.
+ *
+ * A wake-stream can end CLEANLY: wake-bus's backpressure guard calls `res.end()`
+ * on a backed-up subscriber, and any edge in front of the API can terminate a
+ * chunked 200 the same way. Both reconnect loops (the BYOA daemon's streamLoop
+ * and pod-agent's connect loop) kept their only sleep inside `catch`, so a clean
+ * end reconnected with zero delay — measured at ~15,000 connects/second against
+ * the API from the operator's own machine, each one also re-firing a catch-up
+ * turn.
+ *
+ * The second half is this predicate: resetting the ladder on a mere connect
+ * meant that even with a sleep in place, a 200-then-EOF endpoint would reset the
+ * delay every pass and it could never grow.
+ *
+ * Run: node --import tsx --test server/src/__tests__/agents-wake-reconnect.test.ts
+ */
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { STREAM_STABLE_MS, wakeStreamWasStable } from '../agents/runtime/sse-parse.js'
+
+test('a connection that never opened does not reset the ladder', () => {
+ assert.equal(wakeStreamWasStable(null), false)
+})
+
+test('a connection that closed immediately does not reset the ladder', () => {
+ // The 200-then-EOF case. If this returned true the backoff would be pinned at
+ // its base forever and the loop would hammer the server.
+ assert.equal(wakeStreamWasStable(0), false)
+ assert.equal(wakeStreamWasStable(5), false)
+})
+
+test('a short-lived connection does not reset the ladder', () => {
+ assert.equal(wakeStreamWasStable(STREAM_STABLE_MS - 1), false)
+})
+
+test('a connection that stayed up resets the ladder', () => {
+ assert.equal(wakeStreamWasStable(STREAM_STABLE_MS), true)
+ assert.equal(wakeStreamWasStable(STREAM_STABLE_MS * 10), true)
+})
+
+test('the backoff ladder actually grows across repeated instant closes', () => {
+ // Mirrors both loops' shared tail: reset only if stable, then sleep, then
+ // double. With a pathological endpoint the delay must climb to the ceiling.
+ const MAX = 30_000
+ let backoff = 1000
+ const slept: number[] = []
+ for (let i = 0; i < 8; i++) {
+ if (wakeStreamWasStable(0)) backoff = 1000
+ slept.push(backoff)
+ backoff = Math.min(backoff * 2, MAX)
+ }
+ assert.deepEqual(slept, [1000, 2000, 4000, 8000, 16_000, 30_000, 30_000, 30_000])
+})
+
+test('a healthy connection returns the ladder to its base', () => {
+ let backoff = 30_000
+ if (wakeStreamWasStable(STREAM_STABLE_MS + 1)) backoff = 1000
+ assert.equal(backoff, 1000, 'a stream that stayed up must reconnect promptly')
+})
diff --git a/server/src/agents/computer/daemon.ts b/server/src/agents/computer/daemon.ts
index 7d17dea..99f32e5 100644
--- a/server/src/agents/computer/daemon.ts
+++ b/server/src/agents/computer/daemon.ts
@@ -26,7 +26,7 @@ import { execFile, spawn } from 'node:child_process'
import { promisify } from 'node:util'
const execFileP = promisify(execFile)
-import { parseSseStream } from '../runtime/sse-parse.js'
+import { parseSseStream, wakeStreamWasStable } from '../runtime/sse-parse.js'
import { detectEngines, getAdapter, ENGINE_IDS, runEngineDoctor, type EngineId, type EngineSession, type EngineRunResult, type EngineUsage, type EngineHopReport } from './engine.js'
import { usageFromClaude, type TokenUsage } from '../cost.js'
import { parseTriage, finalizeTriage, isRateLimited } from '../triage-core.js'
@@ -2099,6 +2099,7 @@ class AgentRunner {
private async streamLoop(): Promise<void> {
let backoff = 1000
while (!this.stopped) {
+ let connectedAt: number | null = null
try {
const token = await this.ensureToken()
const res = await fetch(`${this.cfg.serverUrl}/runtime/wake-stream`, {
@@ -2106,7 +2107,7 @@ class AgentRunner {
})
if (!res.ok || !res.body) throw new Error(`wake-stream HTTP ${res.status}`)
console.log(`[computer] ${this.agent.id} wake-stream connected (engine: ${this.adapter.id})`)
- backoff = 1000
+ connectedAt = Date.now()
this.kickTurn('reconnect-catchup') // cold-start / reconnect catch-up
for await (const evt of parseSseStream(res.body as unknown as AsyncIterable<unknown>)) {
if (this.stopped) break
@@ -2135,13 +2136,25 @@ class AgentRunner {
}
// 'ready' is a no-op keepalive.
}
+ // The stream ended WITHOUT throwing — a clean server-side close. This
+ // used to fall straight back to the loop head and re-fetch with ZERO
+ // delay, re-firing reconnect-catchup every pass: measured at ~15k
+ // requests/second against the API from the operator's own machine.
+ if (!this.stopped) {
+ console.warn(`[computer] ${this.agent.id} wake-stream closed by server · retry in ${backoff}ms`)
+ }
} catch (err) {
if (this.stopped) break
const _cause = (err as { cause?: { code?: string; message?: string } } | null)?.cause
console.warn(`[computer] ${this.agent.id} stream error: ${err instanceof Error ? err.message : err}${_cause ? ` cause=${_cause.code ?? _cause.message ?? JSON.stringify(_cause)}` : ''} · retry in ${backoff}ms`)
- await new Promise((r) => setTimeout(r, backoff))
- backoff = Math.min(backoff * 2, 30_000)
}
+ if (this.stopped) break
+ // BOTH exits back off. Reset the ladder only after a connection that
+ // actually stayed up — a 200 that closes immediately must not reset it, or
+ // the delay can never grow away from a pathological endpoint.
+ if (wakeStreamWasStable(connectedAt === null ? null : Date.now() - connectedAt)) backoff = 1000
+ await new Promise((r) => setTimeout(r, backoff))
+ backoff = Math.min(backoff * 2, 30_000)
}
}
}
diff --git a/server/src/agents/runtime/pod-agent.ts b/server/src/agents/runtime/pod-agent.ts
index dbbc843..b42a1e0 100644
--- a/server/src/agents/runtime/pod-agent.ts
+++ b/server/src/agents/runtime/pod-agent.ts
@@ -33,7 +33,7 @@ import { runAgentTurn, type AgentTurnOptions } from '../turn.js'
import { runtime } from './select.js'
import { notifyAlert } from '../../alerting.js'
import { pushSteer } from '../steer.js'
-import { parseSseStream } from './sse-parse.js'
+import { parseSseStream, wakeStreamWasStable } from './sse-parse.js'
import { decidePodExit } from './pod-agent-exit.js'
interface RunnerState {
@@ -378,15 +378,22 @@ async function main(): Promise<void> {
// restart, transient network blip) shouldn't kill the Pod.
let backoffMs = 500
while (!state.shuttingDown) {
+ const startedAt = Date.now()
try {
await connectStream(agentId, url, token)
- console.log('[pod-agent] wake-stream closed by server, reconnecting')
- backoffMs = 500
+ // A clean close used to reconnect with NO delay at all and reset the
+ // ladder, so an endpoint that accepts and immediately ends the stream span
+ // the loop as fast as fetch could go.
+ console.log(`[pod-agent] wake-stream closed by server · retry in ${backoffMs}ms`)
} catch (err) {
console.warn(`[pod-agent] wake-stream error: ${err instanceof Error ? err.message : String(err)} · retry in ${backoffMs}ms`)
- await new Promise<void>((r) => setTimeout(r, backoffMs))
- backoffMs = Math.min(backoffMs * 2, 30_000)
}
+ if (state.shuttingDown) break
+ // Reset the ladder only after a connection that actually stayed up; merely
+ // connecting is not evidence of health.
+ if (wakeStreamWasStable(Date.now() - startedAt)) backoffMs = 500
+ await new Promise<void>((r) => setTimeout(r, backoffMs))
+ backoffMs = Math.min(backoffMs * 2, 30_000)
}
}
diff --git a/server/src/agents/runtime/sse-parse.ts b/server/src/agents/runtime/sse-parse.ts
index f66f584..5ea108d 100644
--- a/server/src/agents/runtime/sse-parse.ts
+++ b/server/src/agents/runtime/sse-parse.ts
@@ -48,3 +48,19 @@ export async function* parseSseStream(body: AsyncIterable<unknown>): AsyncGenera
}
}
}
+
+/** How long a wake-stream must stay up before its reconnect ladder resets.
+ *
+ * Merely CONNECTING must not reset the ladder. A wake-stream can end CLEANLY —
+ * wake-bus's backpressure guard calls `res.end()` on a backed-up subscriber,
+ * and any edge in front of the API can terminate a chunked 200 the same way —
+ * so an endpoint that accepts and immediately closes would reset the delay on
+ * every pass and the backoff could never grow. */
+export const STREAM_STABLE_MS = 60_000
+
+/** True when the wake-stream connection that just ended had been up long enough
+ * to count as healthy, so the caller may reset its reconnect delay.
+ * `upForMs` is null when the connection never opened at all. */
+export function wakeStreamWasStable(upForMs: number | null): boolean {
+ return upForMs !== null && upForMs >= STREAM_STABLE_MS
+}