Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(relay): a per-network 429 is E_JOIN_THROTTLED and retries in a minute (R41)

review33b23c58cc6f
diff --git a/src/core/relay/join-token.test.ts b/src/core/relay/join-token.test.ts
index 0438fd52a..ed5ac6d84 100644
--- a/src/core/relay/join-token.test.ts
+++ b/src/core/relay/join-token.test.ts
@@ -47,9 +47,34 @@ describe('joiner tokens', () => {
expect(await mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: fake(403, {}).f })).toEqual({ ok: false, kind: 'revoked' })
})
- it('join: 429 is a rate limit (R40) — never retried as a network failure, never a token verdict', async () => {
+ // The backend's two 429s (nodeterm-server). The PER-IP limiter — `rateLimit({ windowMs: 60_000,
+ // max: 30 })`, src/routes/relay.ts:104, shared by /device, /join and /host-token — answers
+ // `{ error: 'rate_limited', scope: 'ip' }` (src/lib/rate-limit.ts:77) with no Retry-After, and
+ // clears within a minute. The free device-mint DAMPER on /device is the one daily limit, and it
+ // answers `{ error: 'rate_limited' }` with no `scope`. /join has no daily limit at all.
+ const IP_429 = { error: 'rate_limited', scope: 'ip' }
+ const DAILY_429 = { error: 'rate_limited' }
+
+ it('R41: device — the per-IP 429 is `throttled` (retry in a minute); the damper\'s 429 is `rate-limited` (tomorrow)', async () => {
+ const mint = (f: typeof fetch) => mintDeviceToken({ apiBase: 'a', deviceId: 'd', code, label: 'l', fetch: f })
+ expect(await mint(fake(429, IP_429).f)).toEqual({ ok: false, kind: 'throttled' })
+ expect(await mint(fake(429, DAILY_429).f)).toEqual({ ok: false, kind: 'rate-limited' })
+ // A body that does not parse is the damper's answer on /device: never a retry that could spend mints.
+ expect(await mint(fake(429, 'not json').f)).toEqual({ ok: false, kind: 'rate-limited' })
+ })
+
+ it('R41: join — every 429 is `throttled`: /join has no daily limit', async () => {
const join = (f: typeof fetch) => mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: f })
- expect(await join(fake(429, {}).f)).toEqual({ ok: false, kind: 'rate-limited' })
+ expect(await join(fake(429, IP_429).f)).toEqual({ ok: false, kind: 'throttled' })
+ expect(await join(fake(429, DAILY_429).f)).toEqual({ ok: false, kind: 'throttled' })
+ expect(await join(fake(429, 'not json').f)).toEqual({ ok: false, kind: 'throttled' })
+ })
+
+ it('R41: a Retry-After, when the service sends one, rides the throttle', async () => {
+ const withHeader = (async () =>
+ new Response(JSON.stringify(IP_429), { status: 429, headers: { 'retry-after': '90' } })) as unknown as typeof fetch
+ expect(await mintJoinToken({ apiBase: 'a', deviceToken: 'x', fetch: withHeader })).toEqual({ ok: false, kind: 'throttled', retryAfterMs: 90_000 })
+ expect(await mintDeviceToken({ apiBase: 'a', deviceId: 'd', code, label: 'l', fetch: withHeader })).toEqual({ ok: false, kind: 'throttled', retryAfterMs: 90_000 })
})
it('join: any other failure is network, never a token verdict', async () => {
diff --git a/src/core/relay/join-token.ts b/src/core/relay/join-token.ts
index f6f43a878..bef005bf6 100644
--- a/src/core/relay/join-token.ts
+++ b/src/core/relay/join-token.ts
@@ -6,19 +6,42 @@
// Neither leg grants anything on its own: the host still decides who gets in.
//
// Each result says WHICH failure it was, because the caller reacts differently to each: only a
-// `bad-token` join earns one fresh device mint, a `revoked` one never does, and `rate-limited`
-// means waiting until tomorrow rather than retrying.
+// `bad-token` join earns one fresh device mint, a `revoked` one never does, and the two 429s mean
+// different things (the backend is nodeterm-server):
+// - `throttled`: the PER-IP limiter — `rateLimit({ windowMs: 60_000, max: 30 })`,
+// src/routes/relay.ts:104, shared by /device, /join and /host-token — whose body is
+// `{ error: 'rate_limited', scope: 'ip' }` (src/lib/rate-limit.ts:77), sent with no Retry-After.
+// It clears within a minute; a desktop behind an office NAT can hit it. Retried, once a minute.
+// - `rate-limited`: the free device-mint DAMPER on /device, the one DAILY limit — the same error
+// with no `scope`. Waiting for tomorrow, never retried. /join has no daily limit, so every /join
+// 429 is `throttled`; a /device 429 whose body cannot be read is taken as the damper's.
import type { JoinCode } from './join-code'
const TIMEOUT_MS = 8000
-export type DeviceMintResult = { ok: true; deviceToken: string } | { ok: false; kind: 'network' | 'refused' | 'rate-limited' }
+/** The per-network limiter said to slow down; `retryAfterMs` only when it named a Retry-After. */
+export type ThrottledResult = { ok: false; kind: 'throttled'; retryAfterMs?: number }
+export type DeviceMintResult =
+ | { ok: true; deviceToken: string }
+ | { ok: false; kind: 'network' | 'refused' | 'rate-limited' }
+ | ThrottledResult
export type JoinMintResult =
| { ok: true; pairingToken: string; relayEndpoint: string }
- | { ok: false; kind: 'bad-token' | 'revoked' | 'rate-limited' | 'network' }
+ | { ok: false; kind: 'bad-token' | 'revoked' | 'network' }
+ | ThrottledResult
-/** A reply: its status and its parsed JSON body (`null` when the body is not JSON). */
-interface Reply { status: number; ok: boolean; body: unknown }
+/** A reply: its status, its parsed JSON body (`null` when the body is not JSON) and its Retry-After
+ * in ms (`null` when absent or not a number of seconds). */
+interface Reply { status: number; ok: boolean; body: unknown; retryAfterMs: number | null }
+
+/** The per-IP limiter's 429 body names its scope; the device damper's does not. */
+function ipScoped(body: unknown): boolean {
+ return !!body && typeof body === 'object' && (body as { scope?: unknown }).scope === 'ip'
+}
+
+function throttled(res: Reply): ThrottledResult {
+ return res.retryAfterMs !== null ? { ok: false, kind: 'throttled', retryAfterMs: res.retryAfterMs } : { ok: false, kind: 'throttled' }
+}
/** POST `body` and read the JSON reply under ONE timeout that covers the body too: a reply whose
* body stalls would otherwise leave the join pending forever. `null` = no usable response (the
@@ -39,7 +62,9 @@ async function post(apiBase: string, p: string, body: unknown, f: typeof fetch,
} catch {
if (ctrl.signal.aborted) return null
}
- return { status: res.status, ok: res.ok, body: parsed }
+ const ra = res.headers?.get?.('retry-after')
+ const retryAfterMs = ra && /^\d+$/.test(ra.trim()) ? Number(ra.trim()) * 1000 : null
+ return { status: res.status, ok: res.ok, body: parsed, retryAfterMs }
} catch {
return null
} finally {
@@ -64,7 +89,7 @@ export async function mintDeviceToken(d: {
d.timeoutMs ?? TIMEOUT_MS
)
if (!res) return { ok: false, kind: 'network' }
- if (res.status === 429) return { ok: false, kind: 'rate-limited' }
+ if (res.status === 429) return ipScoped(res.body) ? throttled(res) : { ok: false, kind: 'rate-limited' }
// A 5xx is the service being down, not a verdict on this device.
if (res.status >= 500) return { ok: false, kind: 'network' }
if (!res.ok) return { ok: false, kind: 'refused' }
@@ -84,9 +109,8 @@ export async function mintJoinToken(d: {
// Everything else says nothing about the token, so it must not cost a device mint.
if (res.status === 401) return { ok: false, kind: 'bad-token' }
if (res.status === 403) return { ok: false, kind: 'revoked' }
- // A rate limit is a verdict about WHEN, not a network blip: read as `network`, an unattended
- // reconnect would ask again every minute for as long as the limit holds (R40).
- if (res.status === 429) return { ok: false, kind: 'rate-limited' }
+ // /join has no daily limit: its 429 is always the per-network limiter (see the header).
+ if (res.status === 429) return throttled(res)
if (!res.ok) return { ok: false, kind: 'network' }
const j = res.body as { pairingToken?: unknown; relayEndpoint?: unknown } | null
if (typeof j?.pairingToken !== 'string' || !j.pairingToken || typeof j.relayEndpoint !== 'string' || !j.relayEndpoint) {
diff --git a/src/main/remote/hosted-join.test.ts b/src/main/remote/hosted-join.test.ts
index 0f26dde9a..b64042e68 100644
--- a/src/main/remote/hosted-join.test.ts
+++ b/src/main/remote/hosted-join.test.ts
@@ -8,7 +8,7 @@ import fs from 'node:fs'
import os from 'node:os'
import path from 'node:path'
import { joinHostedTeam, connectHostedTeam, removeHostedBookmark, createHostedJoinState, HostedJoinError, type HostedJoinDeps, type HostedJoinEvents, type HostedConnectOptions, type HostedJoinFailure } from './hosted-join'
-import { joinErrorCode } from '../../shared/relay-join-errors'
+import { joinErrorCode, joinRetryAfterMs } from '../../shared/relay-join-errors'
import { BookmarkStore, type RelayBookmark } from './relay-bookmarks'
import { encodeJoinCode, type JoinCode } from '../../core/relay/join-code'
import { hostIdFromPublicKeyB64 } from '../../core/relay/relay-id'
@@ -278,7 +278,9 @@ describe('joinHostedTeam', () => {
revoked: 'E_JOIN_REVOKED',
'key-locked': 'E_JOIN_KEY_LOCKED',
// Another join of OURS for the same team is running: says nothing about the team (R39).
- busy: 'E_JOIN_BUSY'
+ busy: 'E_JOIN_BUSY',
+ // The service's per-network limiter: clears within a minute, so it retries (R41).
+ throttled: 'E_JOIN_THROTTLED'
}
for (const [kind, codeName] of Object.entries(expected) as Array<[HostedJoinFailure, string]>) {
const e = new HostedJoinError(kind)
@@ -426,16 +428,51 @@ describe('connectHostedTeam (the relay:client:connect leg for a join code)', ()
expect(x.sessions.size).toBe(0)
})
- it('R40: a rate-limited /v1/relay/join is E_JOIN_RATE (never retried), and the device token is kept', async () => {
- const s = setup({ routes: { device: [DEVICE_OK('KEEP')], join: [[429, {}], JOIN_OK] } })
+ // nodeterm-server's per-IP limiter (src/routes/relay.ts:104, body from src/lib/rate-limit.ts:77):
+ // shared by /device and /join, 30 a minute, no Retry-After. Its 429 clears within a minute.
+ const IP_429: [number, unknown] = [429, { error: 'rate_limited', scope: 'ip' }]
+
+ it('R41: a throttled /v1/relay/join is E_JOIN_THROTTLED (retryable), and the device token is kept', async () => {
+ const s = setup({ routes: { device: [DEVICE_OK('KEEP')], join: [IP_429, JOIN_OK] } })
const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)
- expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_RATE')
- // A later attempt (a person's, the next day) presents the SAME token: a rate limit costs no mint.
+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')
+ // The retry a minute later presents the SAME token: a throttle costs no mint.
await connectHostedTeam(codeText, s.deps, io().io)
expect(s.api.count('device')).toBe(1)
expect(s.api.calls.filter((c) => c.route === 'join').map((c) => c.body.deviceToken)).toEqual(['KEEP', 'KEEP'])
})
+ it('R41: retrying a throttle is budget-safe — the mints never grow past what the service granted', async () => {
+ // The device leg throttled once (no token granted, nothing to keep), then granted ONE token; the
+ // join leg throttled once more. Three attempts, one granted mint, and every join presents it.
+ const s = setup({ routes: { device: [IP_429, DEVICE_OK('GRANTED')], join: [IP_429, JOIN_OK] } })
+ for (let i = 0; i < 2; i++) {
+ const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)
+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')
+ }
+ await connectHostedTeam(codeText, s.deps, io().io)
+ expect(s.api.count('device')).toBe(2) // one throttled (no mint), one granted
+ expect(s.api.calls.filter((c) => c.route === 'join').map((c) => c.body.deviceToken)).toEqual(['GRANTED', 'GRANTED'])
+ expect((await s.store.list())[0]?.deviceToken).toBe('GRANTED')
+ })
+
+ it('R41: the device damper (a 429 with no scope) stays E_JOIN_RATE — the daily limit, never retried', async () => {
+ const s = setup({ routes: { device: [[429, { error: 'rate_limited' }]] } })
+ const err = await connectHostedTeam(codeText, s.deps, io().io).catch((e: Error) => e)
+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_RATE')
+ })
+
+ it('R41: a Retry-After the service sends crosses IPC with the throttle', async () => {
+ const s = setup()
+ const f = (async (url: string) =>
+ url.endsWith('/v1/relay/device')
+ ? new Response(JSON.stringify({ error: 'rate_limited', scope: 'ip' }), { status: 429, headers: { 'retry-after': '120' } })
+ : new Response('{}', { status: 500 })) as unknown as typeof fetch
+ const err = await connectHostedTeam(codeText, { ...s.deps, fetch: f }, io().io).catch((e: Error) => e)
+ expect(joinErrorCode((err as Error).message)).toBe('E_JOIN_THROTTLED')
+ expect(joinRetryAfterMs((err as Error).message)).toBe(120_000)
+ })
+
it('a failed join registers nothing and rejects with the human message', async () => {
const s = setup({ routes: { device: [[429, {}]] } })
const x = io()
diff --git a/src/main/remote/hosted-join.ts b/src/main/remote/hosted-join.ts
index 0928acfd4..f8d32f527 100644
--- a/src/main/remote/hosted-join.ts
+++ b/src/main/remote/hosted-join.ts
@@ -39,12 +39,21 @@ import type { KeyPair } from '../../core/relay/e2ee'
import type { BookmarkStore, RelayBookmark } from './relay-bookmarks'
import { IPC } from '../../shared/ipc'
import type { RelayClosedReason } from '../../shared/types'
-import type { JoinErrorCode } from '../../shared/relay-join-errors'
+import { retryAfterTag, type JoinErrorCode } from '../../shared/relay-join-errors'
/** What a hosted join hands the relay client: never a pin store. */
export type HostedConnectOptions = Omit<ConnectRelayClientOptions, 'pins' | 'transport'>
-export type HostedJoinFailure = 'invalid-code' | 'rate-limited' | 'refused' | 'network' | 'bad-token' | 'revoked' | 'key-locked' | 'busy'
+export type HostedJoinFailure =
+ | 'invalid-code'
+ | 'rate-limited'
+ | 'refused'
+ | 'network'
+ | 'bad-token'
+ | 'revoked'
+ | 'key-locked'
+ | 'busy'
+ | 'throttled'
/** Each failure's stable code. `bad-token` is a refusal: a token the service will not accept even
* freshly minted cannot be fixed by retrying, which would only spend damped mints. */
@@ -56,7 +65,8 @@ const CODES: Record<HostedJoinFailure, JoinErrorCode> = {
network: 'E_JOIN_NETWORK',
revoked: 'E_JOIN_REVOKED',
'key-locked': 'E_JOIN_KEY_LOCKED',
- busy: 'E_JOIN_BUSY'
+ busy: 'E_JOIN_BUSY',
+ throttled: 'E_JOIN_THROTTLED'
}
const MESSAGES: Record<HostedJoinFailure, string> = {
@@ -67,15 +77,18 @@ const MESSAGES: Record<HostedJoinFailure, string> = {
'bad-token': "The nodeterm service did not accept this device's token for that team.",
revoked: "This device's relay access was revoked.",
'key-locked': 'This device identity could not be loaded.',
- busy: 'Already joining this team; wait for that attempt to finish.'
+ busy: 'Already joining this team; wait for that attempt to finish.',
+ throttled: 'The nodeterm service is limiting requests from this network. Try again in a minute.'
}
/** A join that did not reach the relay. `kind` says why; the message is `[<code>] <for the human>`,
* with `detail` in place of the stock sentence when the failing step said something better. */
export class HostedJoinError extends Error {
readonly code: JoinErrorCode
- constructor(readonly kind: HostedJoinFailure, detail?: string) {
- super(`[${CODES[kind]}] ${detail ?? MESSAGES[kind]}`)
+ /** @param retryAfterMs a throttle's Retry-After, carried in the message (the only part of the
+ * error that crosses Electron IPC) as `[retry-after:<s>]`. */
+ constructor(readonly kind: HostedJoinFailure, detail?: string, retryAfterMs?: number) {
+ super(`[${CODES[kind]}] ${detail ?? MESSAGES[kind]}${retryAfterMs !== undefined ? ` ${retryAfterTag(retryAfterMs)}` : ''}`)
this.code = CODES[kind]
this.name = 'HostedJoinError'
}
@@ -240,7 +253,8 @@ async function joinWithCode(code: JoinCode, deps: HostedJoinDeps, ev: HostedJoin
label: deps.label,
fetch: deps.fetch
})
- if (!d.ok) throw new HostedJoinError(d.kind)
+ // A throttled mint minted nothing: there is no token to keep, and the retry is budget-safe.
+ if (!d.ok) throw new HostedJoinError(d.kind, undefined, d.kind === 'throttled' ? d.retryAfterMs : undefined)
deviceToken = d.deviceToken
state.tokens.set(code.hostId, code.hostPublicKeyB64, d.deviceToken)
await persist()
@@ -263,7 +277,7 @@ async function joinWithCode(code: JoinCode, deps: HostedJoinDeps, ev: HostedJoin
// The kept token no longer verifies (it expired, or the service forgot it): one fresh mint.
j = await join(await mintDevice())
}
- if (!j.ok) throw new HostedJoinError(j.kind)
+ if (!j.ok) throw new HostedJoinError(j.kind, undefined, j.kind === 'throttled' ? j.retryAfterMs : undefined)
// The client token is a bearer for the relay: never send it over plaintext to another machine,
// whoever named the endpoint. Same rule a join code's own endpoint is held to.
if (!allowedEndpoint(j.relayEndpoint)) {
diff --git a/src/shared/relay-join-errors.test.ts b/src/shared/relay-join-errors.test.ts
index a36d22c4b..6d1163672 100644
--- a/src/shared/relay-join-errors.test.ts
+++ b/src/shared/relay-join-errors.test.ts
@@ -1,6 +1,6 @@
// src/shared/relay-join-errors.test.ts
import { describe, it, expect } from 'vitest'
-import { JOIN_ERROR_CODES, joinErrorCode, joinErrorRetries } from './relay-join-errors'
+import { JOIN_ERROR_CODES, joinErrorCode, joinErrorRetries, joinRetryAfterMs } from './relay-join-errors'
describe('hosted join error codes', () => {
it('reads each code off the message main threw, and off the one Electron hands the renderer', () => {
@@ -19,11 +19,24 @@ describe('hosted join error codes', () => {
expect(joinErrorCode(undefined as unknown as string)).toBeNull()
})
- it('only a network failure is worth retrying', () => {
- expect(JOIN_ERROR_CODES.filter(joinErrorRetries)).toEqual(['E_JOIN_NETWORK'])
+ it('only a network failure and the per-network throttle are worth retrying', () => {
+ expect(JOIN_ERROR_CODES.filter(joinErrorRetries)).toEqual(['E_JOIN_NETWORK', 'E_JOIN_THROTTLED'])
expect(joinErrorRetries(null)).toBe(false)
})
+ it('R41: E_JOIN_THROTTLED (the per-IP limiter, clears within a minute) retries; E_JOIN_RATE (the daily damper) does not', () => {
+ expect(joinErrorCode('[E_JOIN_THROTTLED] limiting')).toBe('E_JOIN_THROTTLED')
+ expect(joinErrorRetries('E_JOIN_THROTTLED')).toBe(true)
+ expect(joinErrorRetries('E_JOIN_RATE')).toBe(false)
+ })
+
+ it('R41: reads a Retry-After main attached, and nothing else', () => {
+ expect(joinRetryAfterMs("Error invoking remote method 'x': Error: [E_JOIN_THROTTLED] limiting [retry-after:120]")).toBe(120_000)
+ expect(joinRetryAfterMs('[E_JOIN_THROTTLED] limiting')).toBeNull()
+ expect(joinRetryAfterMs('[retry-after:abc]')).toBeNull()
+ expect(joinRetryAfterMs(undefined as unknown as string)).toBeNull()
+ })
+
it('E_JOIN_BUSY is a code of its own (another attempt of ours is running), and it never retries', () => {
expect(JOIN_ERROR_CODES).toContain('E_JOIN_BUSY')
expect(joinErrorCode('[E_JOIN_BUSY] Already joining this team; wait for that attempt to finish.')).toBe('E_JOIN_BUSY')
diff --git a/src/shared/relay-join-errors.ts b/src/shared/relay-join-errors.ts
index 2cbfb7f71..94c4f17aa 100644
--- a/src/shared/relay-join-errors.ts
+++ b/src/shared/relay-join-errors.ts
@@ -8,6 +8,9 @@
// retrying it would at best repeat the refusal and at worst spend damped device-token mints.
// `E_JOIN_BUSY` says nothing about the team at all: another join of OURS for the same team is still
// running, and it is the one that should finish.
+// `E_JOIN_THROTTLED` is the service's per-network limiter (it clears within a minute), so it is the
+// one refusal worth retrying — never faster than once a minute. `E_JOIN_RATE` is the daily limit on
+// device mints: tomorrow, not in a minute.
export const JOIN_ERROR_CODES = [
'E_JOIN_REVOKED',
@@ -16,7 +19,8 @@ export const JOIN_ERROR_CODES = [
'E_JOIN_BAD_CODE',
'E_JOIN_NETWORK',
'E_JOIN_KEY_LOCKED',
- 'E_JOIN_BUSY'
+ 'E_JOIN_BUSY',
+ 'E_JOIN_THROTTLED'
] as const
export type JoinErrorCode = (typeof JOIN_ERROR_CODES)[number]
@@ -33,5 +37,19 @@ export function joinErrorCode(message: string): JoinErrorCode | null {
/** Whether a failed join is worth retrying unattended. */
export function joinErrorRetries(code: JoinErrorCode | null): boolean {
- return code === 'E_JOIN_NETWORK'
+ return code === 'E_JOIN_NETWORK' || code === 'E_JOIN_THROTTLED'
+}
+
+const RETRY_AFTER_RE = /\[retry-after:(\d+)\]/
+
+/** The Retry-After (in ms) main attached to a throttled join as `[retry-after:<seconds>]`, else null. */
+export function joinRetryAfterMs(message: string): number | null {
+ if (typeof message !== 'string') return null
+ const m = RETRY_AFTER_RE.exec(message)
+ return m ? Number(m[1]) * 1000 : null
+}
+
+/** The tag main appends to a throttled join's message when the service named a Retry-After. */
+export function retryAfterTag(ms: number): string {
+ return `[retry-after:${Math.max(0, Math.ceil(ms / 1000))}]`
}