Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.
TypeScript adhdagent-orchestrationai-agentscanvasclaude-codecodexcoding-agentsdeveloper-toolselectrongeminilinuxmacosmultiplexeropencodeparallel-agentsterminalterminal-emulatorterminal-multiplexertmuxworkspace-manager
fix(relay): a per-network 429 is E_JOIN_THROTTLED and retries in a minute (R41)
review33b23c58cc6f
diff --git a/src/core/relay/join-token.test.ts b/src/core/relay/join-token.test.tsindex 0438fd52a..ed5ac6d84 100644--- a/src/core/relay/join-token.test.ts+++ b/src/core/relay/join-token.test.ts@@ -47,9 +47,34 @@ describe('joiner tokens', () => {expect(await mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: fake(403, {}).f })).toEqual({ ok: false, kind: 'revoked' })})- it('join: 429 is a rate limit (R40) — never retried as a network failure, never a token verdict', async () => {+ // The backend's two 429s (nodeterm-server). The PER-IP limiter — `rateLimit({ windowMs: 60_000,+ // max: 30 })`, src/routes/relay.ts:104, shared by /device, /join and /host-token — answers+ // `{ error: 'rate_limited', scope: 'ip' }` (src/lib/rate-limit.ts:77) with no Retry-After, and+ // clears within a minute. The free device-mint DAMPER on /device is the one daily limit, and it+ // answers `{ error: 'rate_limited' }` with no `scope`. /join has no daily limit at all.+ const IP_429 = { error: 'rate_limited', scope: 'ip' }+ const DAILY_429 = { error: 'rate_limited' }++ it('R41: device — the per-IP 429 is `throttled` (retry in a minute); the damper\'s 429 is `rate-limited` (tomorrow)', async () => {+ const mint = (f: typeof fetch) => mintDeviceToken({ apiBase: 'a', deviceId: 'd', code, label: 'l', fetch: f })+ expect(await mint(fake(429, IP_429).f)).toEqual({ ok: false, kind: 'throttled' })+ expect(await mint(fake(429, DAILY_429).f)).toEqual({ ok: false, kind: 'rate-limited' })+ // A body that does not parse is the damper's answer on /device: never a retry that could spend mints.+ expect(await mint(fake(429, 'not json').f)).toEqual({ ok: false, kind: 'rate-limited' })+ })++ it('R41: join — every 429 is `throttled`: /join has no daily limit', async () => {const join = (f: typeof fetch) => mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: f })- expect(await join(fake(429, {}).f)).toEqual({ ok: false, kind: 'rate-limited' })+ expect(await join(fake(429, IP_429).f)).toEqual({ ok: false, kind: 'throttled' })+ expect(await join(fake(429, DAILY_429).f)).toEqual({ ok: false, kind: 'throttled' })+ expect(await join(fake(429, 'not json').f)).toEqual({ ok: false, kind: 'throttled' })+ })++ it('R41: a Retry-After, when the service sends one, rides the throttle', async () => {+ const withHeader = (async () =>+ new Response(JSON.stringify(IP_429), { status: 429, headers: { 'retry-after': '90' } })) as unknown as typeof fetch+ expect(await mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: withHeader })).toEqual({ ok: false, kind: 'throttled', retryAfterMs: 90_000 })+ expect(await mintDeviceToken({ apiBase: 'a', deviceId: 'd', code, label: 'l', fetch: withHeader })).toEqual({ ok: false, kind: 'throttled', retryAfterMs: 90_000 })})it('join: any other failure is network, never a token verdict', async () => {diff --git a/src/core/relay/join-token.ts b/src/core/relay/join-token.tsindex f6f43a878..bef005bf6 100644--- a/src/core/relay/join-token.ts+++ b/src/core/relay/join-token.ts@@ -6,19 +6,42 @@// Neither leg grants anything on its own: the host still decides who gets in.//// Each result says WHICH failure it was, because the caller reacts differently to each: only a-// `bad-token` join earns one fresh device mint, a `revoked` one never does, and `rate-limited`-// means waiting until tomorrow rather than retrying.+// `bad-token` join earns one fresh device mint, a `revoked` one never does, and the two 429s mean+// different things (the backend is nodeterm-server):+// - `throttled`: the PER-IP limiter — `rateLimit({ windowMs: 60_000, max: 30 })`,+// src/routes/relay.ts:104, shared by /device, /join and /host-token — whose body is+// `{ error: 'rate_limited', scope: 'ip' }` (src/lib/rate-limit.ts:77), sent with no Retry-After.+// It clears within a minute; a desktop behind an office NAT can hit it. Retried, once a minute.+// - `rate-limited`: the free device-mint DAMPER on /device, the one DAILY limit — the same error+// with no `scope`. Waiting for tomorrow, never retried. /join has no daily limit, so every /join+// 429 is `throttled`; a /device 429 whose body cannot be read is taken as the damper's.import type { JoinCode } from './join-code'const TIMEOUT_MS = 8000-export type DeviceMintResult = { ok: true; deviceToken: string } | { ok: false; kind: 'network' | 'refused' | 'rate-limited' }+/** The per-network limiter said to slow down; `retryAfterMs` only when it named a Retry-After. */+export type ThrottledResult = { ok: false; kind: 'throttled'; retryAfterMs?: number }+export type DeviceMintResult =+ | { ok: true; deviceToken: string }+ | { ok: false; kind: 'network' | 'refused' | 'rate-limited' }+ | ThrottledResultexport type JoinMintResult =| { ok: true; pairingToken: string; relayEndpoint: string }- | { ok: false; kind: 'bad-token' | 'revoked' | 'rate-limited' | 'network' }+ | { ok: false; kind: 'bad-token' | 'revoked' | 'network' }+ | ThrottledResult-/** A reply: its status and its parsed JSON body (`null` when the body is not JSON). */-interface Reply { status: number; ok: boolean; body: unknown }+/** A reply: its status, its parsed JSON body (`null` when the body is not JSON) and its Retry-After+ * in ms (`null` when absent or not a number of seconds). */+interface Reply { status: number; ok: boolean; body: unknown; retryAfterMs: number | null }++/** The per-IP limiter's 429 body names its scope; the device damper's does not. */+function ipScoped(body: unknown): boolean {+ return !!body && typeof body === 'object' && (body as { scope?: unknown }).scope === 'ip'+}++function throttled(res: Reply): ThrottledResult {+ return res.retryAfterMs !== null ? { ok: false, kind: 'throttled', retryAfterMs: res.retryAfterMs } : { ok: false, kind: 'throttled' }+}/** POST `body` and read the JSON reply under ONE timeout that covers the body too: a reply whose* body stalls would otherwise leave the join pending forever. `null` = no usable response (the@@ -39,7 +62,9 @@ async function post(apiBase: string, p: string, body: unknown, f: typeof fetch,} catch {if (ctrl.signal.aborted) return null}- return { status: res.status, ok: res.ok, body: parsed }+ const ra = res.headers?.get?.('retry-after')+ const retryAfterMs = ra && /^\d+$/.test(ra.trim()) ? Number(ra.trim()) * 1000 : null+ return { status: res.status, ok: res.ok, body: parsed, retryAfterMs }} catch {return null} finally {@@ -64,7 +89,7 @@ export async function mintDeviceToken(d: {d.timeoutMs ?? TIMEOUT_MS)if (!res) return { ok: false, kind: 'network' }- if (res.status === 429) return { ok: false, kind: 'rate-limited' }+ if (res.status === 429) return ipScoped(res.body) ? throttled(res) : { ok: false, kind: 'rate-limited' }// A 5xx is the service being down, not a verdict on this device.if (res.status >= 500) return { ok: false, kind: 'network' }if (!res.ok) return { ok: false, kind: 'refused' }@@ -84,9 +109,8 @@ export async function mintJoinToken(d: {// Everything else says nothing about the token, so it must not cost a device mint.if (res.status === 401) return { ok: false, kind: 'bad-token' }if (res.status === 403) return { ok: false, kind: 'revoked' }- // A rate limit is a verdict about WHEN, not a network blip: read as `network`, an unattended- // reconnect would ask again every minute for as long as the limit holds (R40).- if (res.status === 429) return { ok: false, kind: 'rate-limited' }+ // /join has no daily limit: its 429 is always the per-network limiter (see the header).+ if (res.status === 429) return throttled(res)if (!res.ok) return { ok: false, kind: 'network' }const j = res.body as { pairingToken?: unknown; relayEndpoint?: unknown } | nullif (typeof j?.pairingToken !== 'string' || !j.pairingToken || typeof j.relayEndpoint !== 'string' || !j.relayEndpoint) {diff --git a/src/main/remote/hosted-join.test.ts b/src/main/remote/hosted-join.test.tsindex 0f26dde9a..b64042e68 100644--- a/src/main/remote/hosted-join.test.ts+++ b/src/main/remote/hosted-join.test.ts@@ -8,7 +8,7 @@ import fs from 'node:fs'import os from 'node:os'import path from 'node:path'import { joinHostedTeam, connectHostedTeam, removeHostedBookmark, createHostedJoinState, HostedJoinError, type HostedJoinDeps, type HostedJoinEvents, type HostedConnectOptions, type HostedJoinFailure } from './hosted-join'-import { joinErrorCode } from '../../shared/relay-join-errors'+import { joinErrorCode, joinRetryAfterMs } from '../../shared/relay-join-errors'import { BookmarkStore, type RelayBookmark } from './relay-bookmarks'import { encodeJoinCode, type JoinCode } from '../../core/relay/join-code'import { hostIdFromPublicKeyB64 } from '../../core/relay/relay-id'@@ -278,7 +278,9 @@ describe('joinHostedTeam', () => {revoked: 'E_JOIN_REVOKED','key-locked': 'E_JOIN_KEY_LOCKED',// Another join of OURS for the same team is running: says nothing about the team (R39).- busy: 'E_JOIN_BUSY'+ busy: 'E_JOIN_BUSY',+ // The service's per-network limiter: clears within a minute, so it retries (R41).+ throttled: 'E_JOIN_THROTTLED'}for (const [kind, codeName] of Object.entries(expected) as Array<[HostedJoinFailure, string]>) {const e = new HostedJoinError(kind)@@ -426,16 +428,51 @@ describe('connectHostedTeam (the relay:client:connect leg for a join code)', ()expect(x.sessions.size).toBe(0)})- it('R40: a rate-limited /v1/relay/join is E_JOIN_RATE (never retried), and the device token is kept', async () => {- const s = setup({ routes: { device: [DEVICE_OK('KEEP')], join: [[429, {}], JOIN_OK] } })+ // nodeterm-server's per-IP limiter (src/routes/relay.ts:104, body from src/lib/rate-limit.ts:77):+ // shared by /device and /join, 30 a minute, no Retry-After. Its 429 clears within a minute.+ const IP_429: [number, unknown] = [429, { error: 'rate_limited', scope: 'ip' }]++ it('R41: a throttled /v1/relay/join is E_JOIN_THROTTLED (retryable), and the device token is kept', async () => {+ const s = setup({ routes: { device: [DEVICE_OK('KEEP')], join: [IP_429, JOIN_OK] } })const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)- expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_RATE')- // A later attempt (a person's, the next day) presents the SAME token: a rate limit costs no mint.+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')+ // The retry a minute later presents the SAME token: a throttle costs no mint.await connectHostedTeam(codeText, s.deps, io().io)expect(s.api.count('device')).toBe(1)expect(s.api.calls.filter((c) => c.route === 'join').map((c) => c.body.deviceToken)).toEqual(['KEEP', 'KEEP'])})+ it('R41: retrying a throttle is budget-safe — the mints never grow past what the service granted', async () => {+ // The device leg throttled once (no token granted, nothing to keep), then granted ONE token; the+ // join leg throttled once more. Three attempts, one granted mint, and every join presents it.+ const s = setup({ routes: { device: [IP_429, DEVICE_OK('GRANTED')], join: [IP_429, JOIN_OK] } })+ for (let i = 0; i < 2; i++) {+ const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')+ }+ await connectHostedTeam(codeText, s.deps, io().io)+ expect(s.api.count('device')).toBe(2) // one throttled (no mint), one granted+ expect(s.api.calls.filter((c) => c.route === 'join').map((c) => c.body.deviceToken)).toEqual(['GRANTED', 'GRANTED'])+ expect((await s.store.list())[0]?.deviceToken).toBe('GRANTED')+ })++ it('R41: the device damper (a 429 with no scope) stays E_JOIN_RATE — the daily limit, never retried', async () => {+ const s = setup({ routes: { device: [[429, { error: 'rate_limited' }]] } })+ const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_RATE')+ })++ it('R41: a Retry-After the service sends crosses IPC with the throttle', async () => {+ const s = setup()+ const f = (async (url: string) =>+ url.endsWith('/v1/relay/device')+ ? new Response(JSON.stringify({ error: 'rate_limited', scope: 'ip' }), { status: 429, headers: { 'retry-after': '120' } })+ : new Response('{}', { status: 500 })) as unknown as typeof fetch+ const err = await connectHostedTeam(codeText, { ...s.deps, fetch: f }, io().io).catch((e: Error) => e)+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')+ expect(joinRetryAfterMs((err as Error).message)).toBe(120_000)+ })+it('a failed join registers nothing and rejects with the human message', async () => {const s = setup({ routes: { device: [[429, {}]] } })const x = io()diff --git a/src/main/remote/hosted-join.ts b/src/main/remote/hosted-join.tsindex 0928acfd4..f8d32f527 100644--- a/src/main/remote/hosted-join.ts+++ b/src/main/remote/hosted-join.ts@@ -39,12 +39,21 @@ import type { KeyPair } from '../../core/relay/e2ee'import type { BookmarkStore, RelayBookmark } from './relay-bookmarks'import { IPC } from '../../shared/ipc'import type { RelayClosedReason } from '../../shared/types'-import type { JoinErrorCode } from '../../shared/relay-join-errors'+import { retryAfterTag, type JoinErrorCode } from '../../shared/relay-join-errors'/** What a hosted join hands the relay client: never a pin store. */export type HostedConnectOptions = Omit<ConnectRelayClientOptions, 'pins' | 'transport'>-export type HostedJoinFailure = 'invalid-code' | 'rate-limited' | 'refused' | 'network' | 'bad-token' | 'revoked' | 'key-locked' | 'busy'+export type HostedJoinFailure =+ | 'invalid-code'+ | 'rate-limited'+ | 'refused'+ | 'network'+ | 'bad-token'+ | 'revoked'+ | 'key-locked'+ | 'busy'+ | 'throttled'/** Each failure's stable code. `bad-token` is a refusal: a token the service will not accept even* freshly minted cannot be fixed by retrying, which would only spend damped mints. */@@ -56,7 +65,8 @@ const CODES: Record<HostedJoinFailure, JoinErrorCode> = {network: 'E_JOIN_NETWORK',revoked: 'E_JOIN_REVOKED','key-locked': 'E_JOIN_KEY_LOCKED',- busy: 'E_JOIN_BUSY'+ busy: 'E_JOIN_BUSY',+ throttled: 'E_JOIN_THROTTLED'}const MESSAGES: Record<HostedJoinFailure, string> = {@@ -67,15 +77,18 @@ const MESSAGES: Record<HostedJoinFailure, string> = {'bad-token': "The nodeterm service did not accept this device's token for that team.",revoked: "This device's relay access was revoked.",'key-locked': 'This device identity could not be loaded.',- busy: 'Already joining this team; wait for that attempt to finish.'+ busy: 'Already joining this team; wait for that attempt to finish.',+ throttled: 'The nodeterm service is limiting requests from this network. Try again in a minute.'}/** A join that did not reach the relay. `kind` says why; the message is `[<code>] <for the human>`,* with `detail` in place of the stock sentence when the failing step said something better. */export class HostedJoinError extends Error {readonly code: JoinErrorCode- constructor(readonly kind: HostedJoinFailure, detail?: string) {- super(`[${CODES[kind]}] ${detail ?? MESSAGES[kind]}`)+ /** @param retryAfterMs a throttle's Retry-After, carried in the message (the only part of the+ * error that crosses Electron IPC) as `[retry-after:<s>]`. */+ constructor(readonly kind: HostedJoinFailure, detail?: string, retryAfterMs?: number) {+ super(`[${CODES[kind]}] ${detail ?? MESSAGES[kind]}${retryAfterMs !== undefined ? ` ${retryAfterTag(retryAfterMs)}` : ''}`)this.code = CODES[kind]this.name = 'HostedJoinError'}@@ -240,7 +253,8 @@ async function joinWithCode(code: JoinCode, deps: HostedJoinDeps, ev: HostedJoinlabel: deps.label,fetch: deps.fetch})- if (!d.ok) throw new HostedJoinError(d.kind)+ // A throttled mint minted nothing: there is no token to keep, and the retry is budget-safe.+ if (!d.ok) throw new HostedJoinError(d.kind, undefined, d.kind === 'throttled' ? d.retryAfterMs : undefined)deviceToken = d.deviceTokenstate.tokens.set(code.hostId, code.hostPublicKeyB64, d.deviceToken)await persist()@@ -263,7 +277,7 @@ async function joinWithCode(code: JoinCode, deps: HostedJoinDeps, ev: HostedJoin// The kept token no longer verifies (it expired, or the service forgot it): one fresh mint.j = await join(await mintDevice())}- if (!j.ok) throw new HostedJoinError(j.kind)+ if (!j.ok) throw new HostedJoinError(j.kind, undefined, j.kind === 'throttled' ? j.retryAfterMs : undefined)// The client token is a bearer for the relay: never send it over plaintext to another machine,// whoever named the endpoint. Same rule a join code's own endpoint is held to.if (!allowedEndpoint(j.relayEndpoint)) {diff --git a/src/shared/relay-join-errors.test.ts b/src/shared/relay-join-errors.test.tsindex a36d22c4b..6d1163672 100644--- a/src/shared/relay-join-errors.test.ts+++ b/src/shared/relay-join-errors.test.ts@@ -1,6 +1,6 @@// src/shared/relay-join-errors.test.tsimport { describe, it, expect } from 'vitest'-import { JOIN_ERROR_CODES, joinErrorCode, joinErrorRetries } from './relay-join-errors'+import { JOIN_ERROR_CODES, joinErrorCode, joinErrorRetries, joinRetryAfterMs } from './relay-join-errors'describe('hosted join error codes', () => {it('reads each code off the message main threw, and off the one Electron hands the renderer', () => {@@ -19,11 +19,24 @@ describe('hosted join error codes', () => {expect(joinErrorCode(undefined as unknown as string)).toBeNull()})- it('only a network failure is worth retrying', () => {- expect(JOIN_ERROR_CODES.filter(joinErrorRetries)).toEqual(['E_JOIN_NETWORK'])+ it('only a network failure and the per-network throttle are worth retrying', () => {+ expect(JOIN_ERROR_CODES.filter(joinErrorRetries)).toEqual(['E_JOIN_NETWORK', 'E_JOIN_THROTTLED'])expect(joinErrorRetries(null)).toBe(false)})+ it('R41: E_JOIN_THROTTLED (the per-IP limiter, clears within a minute) retries; E_JOIN_RATE (the daily damper) does not', () => {+ expect(joinErrorCode('[E_JOIN_THROTTLED] limiting')).toBe('E_JOIN_THROTTLED')+ expect(joinErrorRetries('E_JOIN_THROTTLED')).toBe(true)+ expect(joinErrorRetries('E_JOIN_RATE')).toBe(false)+ })++ it('R41: reads a Retry-After main attached, and nothing else', () => {+ expect(joinRetryAfterMs("Error invoking remote method 'x': Error: [E_JOIN_THROTTLED] limiting [retry-after:120]")).toBe(120_000)+ expect(joinRetryAfterMs('[E_JOIN_THROTTLED] limiting')).toBeNull()+ expect(joinRetryAfterMs('[retry-after:abc]')).toBeNull()+ expect(joinRetryAfterMs(undefined as unknown as string)).toBeNull()+ })+it('E_JOIN_BUSY is a code of its own (another attempt of ours is running), and it never retries', () => {expect(JOIN_ERROR_CODES).toContain('E_JOIN_BUSY')expect(joinErrorCode('[E_JOIN_BUSY] Already joining this team; wait for that attempt to finish.')).toBe('E_JOIN_BUSY')diff --git a/src/shared/relay-join-errors.ts b/src/shared/relay-join-errors.tsindex 2cbfb7f71..94c4f17aa 100644--- a/src/shared/relay-join-errors.ts+++ b/src/shared/relay-join-errors.ts@@ -8,6 +8,9 @@// retrying it would at best repeat the refusal and at worst spend damped device-token mints.// `E_JOIN_BUSY` says nothing about the team at all: another join of OURS for the same team is still// running, and it is the one that should finish.+// `E_JOIN_THROTTLED` is the service's per-network limiter (it clears within a minute), so it is the+// one refusal worth retrying — never faster than once a minute. `E_JOIN_RATE` is the daily limit on+// device mints: tomorrow, not in a minute.export const JOIN_ERROR_CODES = ['E_JOIN_REVOKED',@@ -16,7 +19,8 @@ export const JOIN_ERROR_CODES = ['E_JOIN_BAD_CODE','E_JOIN_NETWORK','E_JOIN_KEY_LOCKED',- 'E_JOIN_BUSY'+ 'E_JOIN_BUSY',+ 'E_JOIN_THROTTLED'] as constexport type JoinErrorCode = (typeof JOIN_ERROR_CODES)[number]@@ -33,5 +37,19 @@ export function joinErrorCode(message: string): JoinErrorCode | null {/** Whether a failed join is worth retrying unattended. */export function joinErrorRetries(code: JoinErrorCode | null): boolean {- return code === 'E_JOIN_NETWORK'+ return code === 'E_JOIN_NETWORK' || code === 'E_JOIN_THROTTLED'+}++const RETRY_AFTER_RE = /\[retry-after:(\d+)\]/++/** The Retry-After (in ms) main attached to a throttled join as `[retry-after:<seconds>]`, else null. */+export function joinRetryAfterMs(message: string): number | null {+ if (typeof message !== 'string') return null+ const m = RETRY_AFTER_RE.exec(message)+ return m ? Number(m[1]) * 1000 : null+}++/** The tag main appends to a throttled join's message when the service named a Retry-After. */+export function retryAfterTag(ms: number): string {+ return `[retry-after:${Math.max(0, Math.ceil(ms / 1000))}]`}