Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(relay): hold the hosted scheduler to an hourly mint budget

reviewfb9e2dc13154
diff --git a/src/core/relay/hosted-scheduler.test.ts b/src/core/relay/hosted-scheduler.test.ts
index 6459c09ae..0a651c2b4 100644
--- a/src/core/relay/hosted-scheduler.test.ts
+++ b/src/core/relay/hosted-scheduler.test.ts
@@ -54,6 +54,13 @@ function harness(mints: Array<MintResult | (() => Promise<MintResult>)>, opts: {
return { s, opened, advance, timers, delays, mintCalls: () => mintCalls }
}
const ok = (ttlMs = 120_000): MintResult => ({ ok: true, pairingToken: 'T', hostId: 'H', ttlMs })
+const okMany = (n: number): MintResult[] => Array.from({ length: n }, () => ok())
+const listener = (tok: string, ev: { onBridged(): void; onClose(): void }): Opened => {
+ const l: Opened = { bridged: false, closed: false, token: tok, ev, close() { l.closed = true } }
+ return l
+}
+const HOUR = 3_600_000
+const TIMER_MAX = 2_147_483_647
describe('hosted scheduler', () => {
it('keeps one idle listener and replaces it before the token expires', async () => {
@@ -113,7 +120,17 @@ describe('hosted scheduler', () => {
h.s.start(); await flush()
expect(h.delays).toHaveLength(1)
expect(h.delays[0]).toBeGreaterThanOrEqual(60_000)
- expect(h.delays[0]).toBeLessThanOrEqual(3_600_000)
+ expect(h.delays[0]).toBe(TIMER_MAX) // capped at setTimeout's own limit, not shortened below it
+ })
+
+ it('a 2 h Retry-After is honored in full (a 429 waits max(60 s, Retry-After), never less)', async () => {
+ const h = harness([{ ok: false, kind: 'rate-limited', status: 429, retryAfterMs: 2 * HOUR }, ok()])
+ h.s.start(); await flush()
+ await h.advance(2 * HOUR - 1)
+ expect(h.mintCalls()).toBe(1)
+ expect(h.opened).toHaveLength(0)
+ await h.advance(1)
+ expect(h.opened).toHaveLength(1)
})
it('a missing, NaN or absurd ttl never arms a refresh below 15 s or beyond the timer range', async () => {
@@ -123,7 +140,7 @@ describe('hosted scheduler', () => {
h.opened[0].ev.onBridged(); await flush()
expect(h.delays.at(-1)).toBe(15_000) // already expired → the 15 s floor
h.opened[1].ev.onBridged(); await flush()
- expect(h.delays.at(-1)).toBeLessThanOrEqual(3_600_000)
+ expect(h.delays.at(-1)).toBe(TIMER_MAX)
})
it('a bridged session ending does not advance the backoff (only a relay failure does)', async () => {
@@ -291,4 +308,102 @@ describe('hosted scheduler', () => {
await h.advance(1) // the mint at t=0 is now more than an hour old
expect(h.s.status().mintsLastHour).toBe(40)
})
+ it('a listener that bridges AND closes synchronously inside open() does not leave the scheduler dead', async () => {
+ let first = true
+ const h = harness([ok(), ok()], {
+ open: (tok, ev) => {
+ const l = listener(tok, ev)
+ h.opened.push(l)
+ if (first) { first = false; ev.onBridged(); ev.onClose() }
+ return l
+ }
+ })
+ h.s.start(); await flush()
+ expect(h.opened).toHaveLength(2)
+ expect(h.s.status()).toMatchObject({ state: 'running', idle: 1, bridged: 0 })
+ })
+
+ it("only an IDLE listener's refresh proves the registration path: a bridged one neither resets the backoff nor holds a timer", async () => {
+ let drop = false
+ const h = harness(okMany(100), {
+ open: (tok, ev) => {
+ const l = listener(tok, ev)
+ h.opened.push(l)
+ if (drop) queueMicrotask(() => ev.onClose()) // the relay refuses every NEW registration
+ return l
+ }
+ })
+ h.s.start(); await flush()
+ drop = true
+ h.opened[0].ev.onBridged(); await flush() // a teammate stays connected throughout
+ expect(h.s.status()).toMatchObject({ idle: 0, bridged: 1 })
+ // Its refresh timer went with the bridge: the only timer is the backoff for the refused newcomer.
+ expect(h.timers.map((x) => x.ms)).toEqual([1000])
+ await h.advance(600_000) // several 90 s refresh periods of the bridged listener
+ const backoffs = h.delays.filter((d) => d <= 15_000)
+ const settled = backoffs.indexOf(15_000)
+ expect(settled).toBeGreaterThan(-1)
+ // Once the backoff has climbed to its ceiling, nothing may knock it back down to 1 s.
+ expect(backoffs.slice(settled).every((d) => d === 15_000)).toBe(true)
+ expect(h.s.status().lastError).toMatch(/relay/)
+ // The bridged listener holds no refresh timer at all (only the pending backoff is armed).
+ expect(h.timers).toHaveLength(1)
+ })
+
+ it('(a) a peer that connects and leaves every 10 s cannot push the mint rate past the hourly budget', async () => {
+ const seen = new Set<string | null>()
+ const h = harness(okMany(1000), { onStatus: (st) => seen.add(st.lastError) })
+ const left = new Set<Opened>()
+ h.s.start(); await flush()
+ for (let i = 0; i < 359; i++) {
+ await h.advance(10_000)
+ const idle = [...h.opened].reverse().find((l) => !l.bridged && !l.closed && !left.has(l))
+ if (!idle) continue // the budget is holding: no listener to join right now
+ idle.ev.onBridged(); await flush()
+ idle.ev.onClose(); left.add(idle); await flush()
+ }
+ expect(h.mintCalls()).toBeLessThanOrEqual(200) // every mint so far falls inside one hour
+ expect(seen.has('mint budget')).toBe(true)
+ await h.advance(HOUR) // the budget releases: hosting comes back on its own
+ expect(h.s.status()).toMatchObject({ state: 'running', idle: 1 })
+ })
+
+ it('(b) a relay that drops every NEW registration beside one long-lived bridged session stays within budget', async () => {
+ let drop = false
+ const h = harness(okMany(1000), {
+ open: (tok, ev) => {
+ const l = listener(tok, ev)
+ h.opened.push(l)
+ if (drop) queueMicrotask(() => ev.onClose())
+ return l
+ }
+ })
+ h.s.start(); await flush()
+ drop = true
+ h.opened[0].ev.onBridged(); await flush()
+ await h.advance(HOUR)
+ expect(h.mintCalls()).toBeLessThanOrEqual(200)
+ })
+
+ it('(c) six bridged sessions on staggered phases do not multiply the re-mint rate', async () => {
+ let drop = false
+ const h = harness(okMany(2000), {
+ open: (tok, ev) => {
+ const l = listener(tok, ev)
+ h.opened.push(l)
+ if (drop) queueMicrotask(() => ev.onClose())
+ return l
+ }
+ })
+ h.s.start(); await flush()
+ for (let k = 0; k < 6; k++) {
+ await h.advance(15_000)
+ h.opened[h.opened.length - 1].ev.onBridged(); await flush() // bridge whichever listener is idle
+ }
+ expect(h.s.status()).toMatchObject({ idle: 1, bridged: 6 })
+ drop = true
+ const before = h.mintCalls()
+ await h.advance(HOUR)
+ expect(h.mintCalls() - before).toBeLessThanOrEqual(200)
+ })
})
diff --git a/src/core/relay/hosted-scheduler.ts b/src/core/relay/hosted-scheduler.ts
index b89d2977c..b763b6f5c 100644
--- a/src/core/relay/hosted-scheduler.ts
+++ b/src/core/relay/hosted-scheduler.ts
@@ -2,10 +2,16 @@
// the host's room, refresh it before its token expires, replace it the moment it bridges a peer,
// and back off on failure. A port of the desktop's rules in src/main/remote/standing-host.ts, made
// pure over injected deps so the measured incidents those rules encode are unit-tested here:
-// - the backoff resets only on PROOF THE RELAY LEG WORKS (a listener lives to its refresh, or a
-// peer bridges) — never on a successful mint. When the API is up and the relay is down every mint
-// succeeds and every socket dies at once, and a reset on mint re-minted at round-trip speed until
-// the API's per-IP limit answered 429 (relay log, 2026-09-27);
+// - the backoff resets only on PROOF THE RELAY LEG WORKS (an IDLE listener holds its registration
+// to its refresh, or a peer bridges) — never on a successful mint. When the API is up and the
+// relay is down every mint succeeds and every socket dies at once, and a reset on mint re-minted
+// at round-trip speed until the API's per-IP limit answered 429 (relay log, 2026-09-27);
+// - a BRIDGED listener's refresh is not proof: that socket registered long ago, so it says nothing
+// about whether a NEW registration works. The desktop counts it, and with one long-lived session
+// beside a relay refusing new registrations that restarted the idle backoff at 1 s every 90 s;
+// - at most MINT_BUDGET_PER_HOUR successful mints in any rolling hour, whatever asks for them. The
+// backend's free limit is 240/h, the 15 s backoff ceiling alone reaches exactly 240/h, and a peer
+// joining and leaving every 10 s asks for 360/h — so no per-path rule can hold the line on its own;
// - a 429 waits at least 60 s (longer if Retry-After says so); a 402/403 stops minting;
// - while a backoff timer is armed it owns the next mint: nothing else may mint early.
// Everything the injected deps can throw is caught: a scheduler that swallowed an exception would sit
@@ -20,9 +26,13 @@ const DEFAULT_TTL_MS = 120_000
const BACKOFF_MS = [1000, 2000, 4000, 8000, 15_000]
const RATE_LIMIT_MIN_MS = 60_000
const HOUR_MS = 3_600_000
-// Ceiling for every delay we arm. Node's setTimeout fires after ~1 ms for anything above 2^31-1 ms,
-// so an unclamped Retry-After or token lifetime from the server would become a tight re-mint loop.
-const MAX_DELAY_MS = HOUR_MS
+// Ceiling for every delay we arm: setTimeout's own limit. Node fires a timer after ~1 ms for any
+// delay above 2^31-1 ms, so an unclamped Retry-After or token lifetime would become a tight re-mint
+// loop. Nothing lower: a shorter cap would re-mint before a long Retry-After has elapsed.
+const MAX_DELAY_MS = 2_147_483_647
+// Stay clear of the backend's 240/h free limit. The count is per scheduler instance, so the headroom
+// is also what absorbs a restart, whose new instance cannot see the mints the old one made.
+const MINT_BUDGET_PER_HOUR = 200
export interface Listener {
bridged: boolean
@@ -32,7 +42,7 @@ export interface SchedulerStatus {
state: 'stopped' | 'running' | 'backend-refused'
/** The most recent failure since the relay leg was last proven to work; null once it has been. */
lastError: string | null
- /** Tokens minted in the last hour (successful mints — what the backend's hourly limit counts). */
+ /** Tokens minted in the last hour (successful mints). The scheduler holds it at 200 (MINT_BUDGET_PER_HOUR). */
mintsLastHour: number
idle: number
bridged: number
@@ -95,14 +105,17 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
lastError = null
}
- const scheduleRetry = (minMs = 0): void => {
- if (state !== 'running' || retry !== null) return
- const ms = clampDelay(Math.max(minMs, BACKOFF_MS[Math.min(attempt, BACKOFF_MS.length - 1)]))
- attempt++
+ // The ONE retry slot. Whoever arms it — backoff, 429 or the mint budget — owns the next mint.
+ const armRetry = (ms: number): void => {
retry = deps.setTimeout(() => {
retry = null
void top()
- }, ms)
+ }, clampDelay(ms))
+ }
+ const scheduleRetry = (minMs = 0): void => {
+ if (state !== 'running' || retry !== null) return
+ armRetry(Math.max(minMs, BACKOFF_MS[Math.min(attempt, BACKOFF_MS.length - 1)]))
+ attempt++
}
const drop = (e: Entry): void => {
@@ -123,16 +136,13 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
const ttl = Number.isFinite(ttlMs) ? ttlMs : DEFAULT_TTL_MS
e.refresh = deps.setTimeout(() => {
e.refresh = null
- if (state !== 'running' || !live.has(e)) return
- // This listener held its relay registration for a whole token lifetime: the relay is reachable.
+ // A bridged listener has no refresh (its timer is cleared on bridging); this is only defence.
+ // It is never cut for a refresh — the relay drops it at TTL and onClose takes it from there —
+ // and it is never re-armed: a bridged refresh would prove nothing (see the header), so a timer
+ // for it would be a timer per session that does nothing, forever for one whose close is lost.
+ if (state !== 'running' || !live.has(e) || e.bridged) return
+ // This IDLE listener held its registration for a whole token lifetime: new registrations work.
proven()
- // A listener serving a peer is never cut for a refresh — the relay drops it at TTL and onClose
- // takes it from there. Only an IDLE listener is swapped for a freshly minted one.
- if (e.bridged) {
- armRefresh(e, DEFAULT_TTL_MS)
- emit()
- return
- }
drop(e)
closeQuietly(e.listener)
emit()
@@ -149,8 +159,15 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
// Keep one idle listener registered. Never two mints at once, and never ahead of an armed backoff.
async function top(): Promise<void> {
if (state !== 'running' || opening || retry !== null || idleCount() >= 1) return
+ if (status().mintsLastHour >= MINT_BUDGET_PER_HOUR) {
+ // Out of budget: wait until the oldest mint in the window ages out (+1 ms, because the window
+ // keeps a mint exactly an hour old). The retry slot makes every other path defer to this.
+ lastError = 'mint budget'
+ armRetry(Math.max(1, mints[0] + HOUR_MS - now() + 1))
+ emit()
+ return
+ }
opening = true
- let opened = false
try {
let r: MintResult
let threw: string | null = null
@@ -183,6 +200,8 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
if (!live.has(e) || e.bridged) return // a stale listener, or the second report of one peer
e.bridged = true
if (e.listener) e.listener.bridged = true
+ if (e.refresh !== null) deps.clearTimeout(e.refresh) // a bridged listener is never refreshed
+ e.refresh = null
proven() // a completed handshake proves the relay leg end to end
emit()
void top() // this listener now serves a peer: restore a warm one
@@ -217,13 +236,14 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
closeQuietly(listener)
return
}
- opened = true
- armRefresh(e, r.ttlMs)
+ if (!e.bridged) armRefresh(e, r.ttlMs)
emit()
} finally {
opening = false
- // Still short (the new listener bridged during open()): continue, as the desktop does.
- if (opened && state === 'running' && idleCount() < 1) queueMicrotask(() => void top())
+ // Still short — the new listener bridged during open(), or bridged AND closed there (whose own
+ // top() calls were refused while we were opening). Unconditional on purpose: every path that
+ // must NOT mint again has already armed the retry slot or left 'running', and top() checks both.
+ if (state === 'running' && idleCount() < 1) queueMicrotask(() => void top())
}
}