Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(relay): pin the refusal streak across transient failures; clearer refusal status

reviewcd856621ad13
diff --git a/src/core/relay/host-token.ts b/src/core/relay/host-token.ts
index c46e96dad..e9481f5a5 100644
--- a/src/core/relay/host-token.ts
+++ b/src/core/relay/host-token.ts
@@ -28,7 +28,8 @@ export type MintResult =
kind: 'network' | 'rate-limited' | 'refused' | 'bad-response'
retryAfterMs?: number
status?: number
- /** Set only on a terminal key-proof refusal (a `refused` 403). */
+ /** Set only on a key-proof refusal (a `refused` 403). The caller counts them: the second in a
+ * row is terminal (see hosted-scheduler.ts). */
reason?: PopRefusal
}
diff --git a/src/core/relay/hosted-scheduler.test.ts b/src/core/relay/hosted-scheduler.test.ts
index af087bbd1..87b8fbfeb 100644
--- a/src/core/relay/hosted-scheduler.test.ts
+++ b/src/core/relay/hosted-scheduler.test.ts
@@ -1,5 +1,5 @@
// src/core/relay/hosted-scheduler.test.ts
-import { describe, it, expect } from 'vitest'
+import { describe, it, expect, vi } from 'vitest'
import nacl from 'tweetnacl'
import { createHostedScheduler, type MintResult, type SchedulerStatus } from './hosted-scheduler'
import { mintHostToken } from './host-token'
@@ -421,8 +421,12 @@ describe('hosted scheduler', () => {
async (reason) => {
const h = harness([popRefused(reason), popRefused(reason)])
h.s.start(); await flush()
- // The first is transient (a POP_SECRET rotation mid challenge→mint looks exactly like this).
- expect(h.s.status()).toMatchObject({ state: 'running', lastError: 'network (403)' })
+ // The first is transient (a POP_SECRET rotation mid challenge→mint looks exactly like this),
+ // and says so: an operator reading `team status` must not see a bare `network (403)`.
+ expect(h.s.status()).toMatchObject({
+ state: 'running',
+ lastError: `key proof refused once (${reason}) — retrying with a fresh challenge`
+ })
expect(h.timers).toHaveLength(1) // the backoff owns the next mint (a fresh challenge)
await h.advance(1000)
expect(h.mintCalls()).toBe(2)
@@ -431,6 +435,60 @@ describe('hosted scheduler', () => {
}
)
+ it.each<[string, MintResult]>([
+ ['a 503', { ok: false, kind: 'network', status: 503 }],
+ ['a 429', { ok: false, kind: 'rate-limited', status: 429 }],
+ ['a bad response', { ok: false, kind: 'bad-response' }]
+ ])('a transient failure between two refusals does not reset the count: refusal, %s, refusal stops', async (_label, transient) => {
+ // Only a successful mint (or start()) proves the key is accepted. Resetting on a transient
+ // failure would let a backend that refuses every proof, behind a flaky challenge, loop forever.
+ const h = harness([popRefused(), transient, popRefused()])
+ h.s.start(); await flush()
+ expect(h.s.status().state).toBe('running')
+ await h.advance(1000) // the backoff → the transient failure
+ expect(h.mintCalls()).toBe(2)
+ expect(h.s.status().state).toBe('running')
+ await h.advance(60_000) // the next backoff step (or the 429's 60 s floor) → refused again
+ expect(h.mintCalls()).toBe(3)
+ expect(h.s.status()).toMatchObject({ state: 'backend-refused', lastError: POP_REFUSED_MESSAGE })
+ expect(h.timers).toHaveLength(0)
+ })
+
+ it('each key-proof refusal is logged with its kind: warn on the first, error on the terminal one', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ try {
+ const h = harness([popRefused('pop_required'), popRefused('pop_invalid')])
+ h.s.start(); await flush()
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(String(warn.mock.calls[0][0])).toContain('pop_required')
+ expect(error).not.toHaveBeenCalled()
+ await h.advance(1000)
+ expect(h.s.status().state).toBe('backend-refused')
+ expect(warn).toHaveBeenCalledTimes(1)
+ expect(error).toHaveBeenCalledTimes(1)
+ expect(String(error.mock.calls[0][0])).toContain('pop_invalid')
+ } finally {
+ warn.mockRestore()
+ error.mockRestore()
+ }
+ })
+
+ it('a refusal that is not a key proof (402) logs nothing', async () => {
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {})
+ const error = vi.spyOn(console, 'error').mockImplementation(() => {})
+ try {
+ const h = harness([{ ok: false, kind: 'refused', status: 402 }])
+ h.s.start(); await flush()
+ expect(h.s.status()).toMatchObject({ state: 'backend-refused', lastError: 'refused (402)' })
+ expect(warn).not.toHaveBeenCalled()
+ expect(error).not.toHaveBeenCalled()
+ } finally {
+ warn.mockRestore()
+ error.mockRestore()
+ }
+ })
+
it('one PoP refusal then a successful mint: hosting keeps running', async () => {
const h = harness([popRefused(), ok()])
h.s.start(); await flush()
diff --git a/src/core/relay/hosted-scheduler.ts b/src/core/relay/hosted-scheduler.ts
index f1b9ac75d..905be6854 100644
--- a/src/core/relay/hosted-scheduler.ts
+++ b/src/core/relay/hosted-scheduler.ts
@@ -18,12 +18,14 @@
// - a key-proof refusal stops minting only on the SECOND in a row: a POP_SECRET rotation, or a
// secret mismatch between backend instances, inside ONE challenge→mint pair refuses an honest
// host once. The first is backed off like a transient 403 (the retry fetches a fresh challenge);
-// a successful mint or start() resets the count;
+// only a successful mint or start() resets the count — a transient failure in between does not,
+// or a backend refusing every proof behind a flaky challenge would retry forever. Both refusals
+// are logged with their kind (warn, then error), so an operator can tell the two apart;
// - while a backoff timer is armed it owns the next mint: nothing else may mint early.
// Everything the injected deps can throw is caught: a scheduler that swallowed an exception would sit
// in 'running' with no listener and no timer, i.e. hosting silently dead until a restart.
import type { MintResult } from './host-token'
-import { POP_REFUSED_MESSAGE } from './relay-pop'
+import { POP_REFUSED_MESSAGE, type PopRefusal } from './relay-pop'
export type { MintResult } from './host-token'
const REFRESH_LEAD_MS = 30_000
@@ -182,6 +184,8 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
try {
let r: MintResult
let threw: string | null = null
+ /** Set when this mint is the FIRST key-proof refusal in a row, which is retried, not terminal. */
+ let refusedOnce: PopRefusal | null = null
try {
r = await deps.mint()
} catch (err) {
@@ -193,13 +197,23 @@ export function createHostedScheduler(deps: SchedulerDeps, now: () => number) {
popRefusals = 0
} else if (r.kind === 'refused' && r.reason && ++popRefusals < 2) {
// The first key-proof refusal in a row is transient (see the header): back off, re-challenge.
+ refusedOnce = r.reason
r = { ok: false, kind: 'network', status: 403 }
}
if (state !== 'running') return
if (!r.ok) {
// A key-proof refusal says what to do about it (update, or `team rotate-key`); `refused (403)` would not.
- lastError = threw ?? (r.reason ? POP_REFUSED_MESSAGE : r.kind + (r.status ? ` (${r.status})` : ''))
+ lastError =
+ threw ??
+ (refusedOnce
+ ? `key proof refused once (${refusedOnce}) — retrying with a fresh challenge`
+ : r.reason
+ ? POP_REFUSED_MESSAGE
+ : r.kind + (r.status ? ` (${r.status})` : ''))
+ // Only the refusal KIND is logged: nothing here holds key material, and nothing may.
+ if (refusedOnce) console.warn(`[hosted-team] the relay refused this host's key proof once (${refusedOnce}); retrying with a fresh challenge`)
if (r.kind === 'refused') {
+ if (r.reason) console.error(`[hosted-team] the relay refused this host's key proof twice in a row (${r.reason}); hosting stopped`)
state = 'backend-refused'
emit()
return
diff --git a/src/main/remote/standing-host.test.ts b/src/main/remote/standing-host.test.ts
index 2f492ff21..63c250956 100644
--- a/src/main/remote/standing-host.test.ts
+++ b/src/main/remote/standing-host.test.ts
@@ -707,6 +707,9 @@ describe('standing host: host key proof-of-possession on the host-token mint', (
expect(sessions).toHaveLength(0)
expect(errorBoxes).toHaveLength(1)
expect(errorBoxes[0].body).toContain(POP_REFUSED_MESSAGE_DESKTOP)
+ // The follow-up line has its own antecedent now that the message ends with "contact support".
+ expect(errorBoxes[0].body).toContain('Phone access is off. Turn it back on in Settings → Phone after updating.')
+ expect(errorBoxes[0].body).not.toContain('until then')
// `team rotate-key` exists only in the Server Edition: the desktop must not advise it.
expect(errorBoxes[0].body).not.toContain('rotate-key')
// Terminal: no reconnect timer is left armed.
@@ -776,6 +779,43 @@ describe('standing host: host key proof-of-possession on the host-token mint', (
host.stop()
})
+ it.each<[string, 'challenge' | 'host-token', Route]>([
+ ['the challenge answering 503', 'challenge', () => res(503)],
+ ['a challenge network error', 'challenge', () => Promise.reject(new TypeError('fetch failed'))],
+ ['the mint answering 503', 'host-token', () => res(503)]
+ ])(
+ 'a transient failure between two refusals does not reset the count: refusal, %s, refusal stops',
+ async (_label, where, transient) => {
+ // Only a successful mint (or start()) proves the key is accepted. Resetting on a transient
+ // failure would let a backend that refuses every proof, behind a flaky challenge, loop forever.
+ vi.useFakeTimers()
+ const pop = createTestPopServer()
+ let challenges = 0
+ let mints = 0
+ routeFetch({
+ challenge: (init) =>
+ where === 'challenge' && ++challenges === 2 ? transient(init) : challengeFrom(pop)(init),
+ hostToken: (init) =>
+ where === 'host-token' && ++mints === 2 ? transient(init) : res(403, { error: 'pop_invalid' })
+ })
+ const host = makeHost()
+ host.setEnabled(true)
+ for (let i = 0; i < 5; i++) await settle()
+ expect(errorBoxes).toHaveLength(0) // refusal 1: transient
+ await vi.advanceTimersByTimeAsync(1000) // the backoff → the transient failure
+ expect(challengeCalls()).toHaveLength(2)
+ expect(errorBoxes).toHaveLength(0)
+ await vi.advanceTimersByTimeAsync(2000) // the next backoff step → refusal 2, the second in a row
+ expect(challengeCalls()).toHaveLength(3)
+ expect(errorBoxes).toHaveLength(1)
+ expect(errorBoxes[0].body).toContain(POP_REFUSED_MESSAGE_DESKTOP)
+ await vi.advanceTimersByTimeAsync(60_000) // stopped: nothing re-mints, no second dialog
+ expect(challengeCalls()).toHaveLength(3)
+ expect(errorBoxes).toHaveLength(1)
+ host.stop()
+ }
+ )
+
it('turning access off while a mint is in flight: a refusal that lands afterwards raises nothing', async () => {
vi.useFakeTimers()
const pop = createTestPopServer()
diff --git a/src/main/remote/standing-host.ts b/src/main/remote/standing-host.ts
index fa9728c32..8ee4bb06f 100644
--- a/src/main/remote/standing-host.ts
+++ b/src/main/remote/standing-host.ts
@@ -110,7 +110,8 @@ async function mintHostToken(
const hostPublicKeyB64 = publicKeyToB64(keys.publicKey)
// Read ONCE: the proof's subject must be the exact id the body sends. On a first launch
// getDeviceId() mints a uuid and writes it asynchronously, so a second call before that write
- // lands mints another one, and the backend would refuse the pair as pop_invalid (terminal).
+ // lands mints another one, and the backend would refuse the pair as pop_invalid (a key-proof
+ // refusal, which stops phone access when it repeats).
const deviceId = entitlement ? null : getDeviceId()
const base = entitlement ? { entitlement, hostPublicKeyB64 } : { deviceId, hostPublicKeyB64 }
const ctrl = new AbortController()
@@ -151,7 +152,7 @@ async function mintHostToken(
method: 'POST',
headers: { 'content-type': 'application/json' },
// popChallenge and popProof travel together or not at all: the backend treats a half-present
- // proof as invalid, and invalid is terminal.
+ // proof as pop_invalid, which would repeat on every retry and so stop phone access.
body: JSON.stringify({ ...base, ...(proof ?? {}) }),
signal: ctrl.signal
})
@@ -214,7 +215,7 @@ function reportPopRefused(kind: PopRefusal): void {
try {
dialog.showErrorBox(
'Remote access stopped',
- `${POP_REFUSED_MESSAGE_DESKTOP}\n\nPhone access is off until then. Turn it back on in Settings → Phone.`
+ `${POP_REFUSED_MESSAGE_DESKTOP}\n\nPhone access is off. Turn it back on in Settings → Phone after updating.`
)
} catch {
// No dialog available (headless / very early boot): the console line is the fallback.