The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
WebSocket churn: failed selects back off; offline and gone machines stop redialling (relay, Desktop, Mobile) (#2027)
0xken158013e40f2d21
diff --git a/cli/src/lib/relayFrames.ts b/cli/src/lib/relayFrames.tsindex 678f5826a..01782a314 100644--- a/cli/src/lib/relayFrames.ts+++ b/cli/src/lib/relayFrames.ts@@ -56,7 +56,8 @@ export const GATEWAY_REQUEST_TYPES = new Set(['device_e2ee_pair', 'phone_pair',/** A window's connection to another machine could not be made: why, and the close code to tell it. Here and* not with the relay pool, so the local socket can tell one apart without loading the pool. */export class RelayConnectError extends Error {- constructor(message: string, readonly closeCode?: number) {+ /** `retryAfterMs`: how long the backend asked not to be asked again (its MACHINE_OFFLINE refusal of a relay). */+ constructor(message: string, readonly closeCode?: number, readonly retryAfterMs?: number) {super(message)this.name = 'RelayConnectError'}diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.tsindex 4fae3d3b0..a2e5a61e7 100644--- a/cli/src/lib/remoteRelay.spec.ts+++ b/cli/src/lib/remoteRelay.spec.ts@@ -822,14 +822,18 @@ describe('RemoteRelayPool keeps background (isolated) clients cheap', () => {}})- it('the window pool has no cooldown: a person clicking again always gets a fresh dial', async () => {+ it('the window pool backs a failed machine off, but an explicit redial always gets a fresh dial', async () => {const pool = new RemoteRelayPool(...ctorArgs) as unknown as {connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>dial: (...args: unknown[]) => Promise<unknown>}const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('relay connect timed out'))await expect(pool.connect('m1', 'env', select)).rejects.toThrow()+ // The app's next select a moment later is answered from the failure (remoteRelayRefusal.spec.ts).await expect(pool.connect('m1', 'env', select)).rejects.toThrow()+ expect(dial).toHaveBeenCalledTimes(1)+ // A redial the app asks for explicitly is dialled at once.+ await expect(pool.connect('m1', 'env', { ...select, payload: { ...select.payload, forceReconnect: true } })).rejects.toThrow()expect(dial).toHaveBeenCalledTimes(2)})})diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.tsindex 9bac07c5d..b03751a8e 100644--- a/cli/src/lib/remoteRelay.ts+++ b/cli/src/lib/remoteRelay.ts@@ -49,9 +49,17 @@ const LINGER_MS = 30_000const ISOLATED_LINGER_MS = 3 * 60_000// After a FAILED background dial, how long the same machine is answered from that failure instead of// dialled again. A poller that retries on every tick against an unlinked or unreachable machine would-// otherwise turn one bad machine into a relay dial per tick, forever. Never applied to the window's-// pool: a person clicking again deserves a fresh attempt.+// otherwise turn one bad machine into a relay dial per tick, forever. The window's pool has the shorter backoff+// below instead, which an explicit redial skips: a person clicking again deserves a fresh attempt.const ISOLATED_DIAL_COOLDOWN_MS = 30_000+// After a machine was refused or could not be reached, every pool answers its next selects from that failure+// for 1 s, then 2, 4 … up to this, or for as long as the backend asked (its MACHINE_OFFLINE retry-after, 30 s),+// whichever is longer. Prod, 2026-10-28: 2,304 of 2,308 web-ws opens in 15 minutes were daemon relays living a+// median 0.4 s, open, sign in, select, refused, closed, because the window's pool dialled again on every+// select its local clients made: the app's retry, a TUI, a CLI command.+const REFUSED_BACKOFF_MAX_MS = 30_000+// A retry-after longer than this is read as this: a wrong answer must not shut a machine out for long.+const RETRY_AFTER_MAX_MS = 5 * 60_000// Warm background sessions kept per machine. Each concurrent background client needs its own (they// must not see each other's frames), so the cap is "how many pollers may overlap on one machine and// still find a warm session"; beyond it the extra session is torn down on detach as it always was.@@ -260,8 +268,16 @@ export interface RemoteRelayPoolOptions {expectsTrust?: (machineId: string, pub: string) => boolean/** Shared record of recent denials per machine, for the above. */denials?: Map<string, number[]>+ /** Shared record of the machines being backed off after a failed dial (REFUSED_BACKOFF_MAX_MS). */+ refusals?: Map<string, Refusal>+ /** The clock the backoff reads; a test's own. */+ now?: () => number}+/** A machine whose last dials failed `count` times in a row: selects of it before `until` get `error` back.+ * `asked`: the backend set `until` (its retry-after), which even an explicit redial waits out. */+interface Refusal { until: number; error: unknown; count: number; asked: boolean }+const DENIED_TRIES = 3const DENIED_WINDOW_MS = 60_000@@ -278,6 +294,8 @@ export class RemoteRelayPool {private readonly onSessionReady: ((machineId: string) => void) | nullprivate readonly expectsTrust: ((machineId: string, pub: string) => boolean) | nullprivate readonly denials: Map<string, number[]>+ private readonly refusals: Map<string, Refusal>+ private readonly now: () => number/** Warm background pools per machine, each holding one lingering session nobody is attached to —* see acquireIsolated(). A pool is either here (idle) or in a client's hands, never both. */private readonly idleIsolated = new Map<string, RemoteRelayPool[]>()@@ -300,6 +318,8 @@ export class RemoteRelayPool {this.lastDialFailure = opts.dialFailures ?? new Map()this.expectsTrust = opts.expectsTrust ?? nullthis.denials = opts.denials ?? new Map()+ this.refusals = opts.refusals ?? new Map()+ this.now = opts.now ?? Date.now}/** `machineId` denied this machine's key. Drop the pin — unless the device key log says it should@@ -334,7 +354,7 @@ export class RemoteRelayPool {const shelf = this.idleIsolated.get(machineId) ?? []const pool = shelf.pop() ?? new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers,{ p2p: false, lingerMs: ISOLATED_LINGER_MS, dialCooldownMs: ISOLATED_DIAL_COOLDOWN_MS, dialFailures: this.lastDialFailure,- ...(this.expectsTrust ? { expectsTrust: this.expectsTrust } : {}), denials: this.denials })+ ...(this.expectsTrust ? { expectsTrust: this.expectsTrust } : {}), denials: this.denials, refusals: this.refusals, now: this.now })const session = await pool.acquire(machineId, autonomousEnv, selectFrame, {...sink,sendFrame: frame => sink.sendFrame(frame.type === 'connected'@@ -431,6 +451,11 @@ export class RemoteRelayPool {}private connect(machineId: string, autonomousEnv: string, selectFrame: Frame): Promise<Entry> {+ // An explicit redial (`forceReconnect`: the app saw its session die, or a person asked again) is not made to+ // wait out the backoff, only what the backend itself asked for.+ const redial = (selectFrame.payload as { forceReconnect?: unknown } | undefined)?.forceReconnect === true+ const refused = this.refusals.get(machineId)+ if (refused && this.now() < refused.until && (refused.asked || !redial)) return Promise.reject(refused.error)const failed = this.lastDialFailure.get(machineId)if (failed && this.dialCooldownMs > 0 && Date.now() - failed.at < this.dialCooldownMs) {return Promise.reject(failed.error)@@ -450,6 +475,7 @@ export class RemoteRelayPool {(error) => { this.lastDialFailure.set(machineId, { at: Date.now(), error }) },)}+ promise.then(() => { this.refusals.delete(machineId) }, (error) => { this.backOff(machineId, error) })this.pending.set(machineId, promise)// `.finally()` re-throws on rejection, producing a SECOND promise distinct from the one returned// below (which callers already await/catch) — left un-caught, every failed dial (e.g. NO_PEER_LINK@@ -460,6 +486,17 @@ export class RemoteRelayPool {return promise}+ /** A dial of `machineId` failed: hold its next selects for the backoff's next step, or the backend's retry-after.+ * An unlinked machine is not held: nothing was dialled, and a link made a moment later must work at once. */+ private backOff(machineId: string, error: unknown): void {+ if (error instanceof RelayConnectError && error.message === 'NO_PEER_LINK') return+ const count = (this.refusals.get(machineId)?.count ?? 0) + 1+ const backoff = Math.min(REFUSED_BACKOFF_MAX_MS, 1_000 * 2 ** (count - 1))+ const asked = error instanceof RelayConnectError && typeof error.retryAfterMs === 'number' && Number.isFinite(error.retryAfterMs)+ ? Math.min(Math.max(error.retryAfterMs, 0), RETRY_AFTER_MAX_MS) : 0+ this.refusals.set(machineId, { until: this.now() + Math.max(backoff, asked), error, count, asked: asked > backoff })+ }+private async dial(machineId: string, autonomousEnv: string, selectFrame: Frame, forceRefresh: boolean): Promise<Entry> {const peer = this.peers.get(machineId)if (!peer) throw new RelayConnectError('NO_PEER_LINK')@@ -560,7 +597,11 @@ export class RemoteRelayPool {if (frame.type === 'machine_select_error' && payload?.machineId === machineId) {if (!settled) {settled = true; clearTimeout(timeout)- reject(new RelayConnectError(typeof payload.error === 'string' ? payload.error : 'machine_select_error'))+ const code = typeof payload.error === 'string' ? payload.error : 'machine_select_error'+ const retryAfterMs = (payload as { retryAfterMs?: unknown }).retryAfterMs+ // Offline closes the window's socket as the node_status answer below does (1013, try again later).+ reject(new RelayConnectError(code, code === 'MACHINE_OFFLINE' ? 1013 : undefined,+ typeof retryAfterMs === 'number' ? retryAfterMs : undefined))}return}diff --git a/cli/src/lib/remoteRelayRefusal.spec.ts b/cli/src/lib/remoteRelayRefusal.spec.tsnew file mode 100644index 000000000..9a66eb66b--- /dev/null+++ b/cli/src/lib/remoteRelayRefusal.spec.ts@@ -0,0 +1,100 @@+import { afterEach, describe, expect, it } from 'vitest'+import { WebSocketServer } from 'ws'+import type { AddressInfo } from 'node:net'+import { RemoteRelayPool } from './remoteRelay.js'+import { MachinePeerStore } from './e2ee/machinePeers.js'+import { b64e, newIdentity } from './e2ee/core.js'+import type { LocalClientSink } from '../backendSocket.js'++/**+ * A machine the backend refuses must not cost a backend socket on every select a local client makes. Prod,+ * 2026-10-28: 2,304 of 2,308 web-ws opens in 15 minutes came from daemon relays, median life 0.4 s — open, sign+ * in, select, refused, closed — because the window's pool dialled again on every select (the app's retry, a+ * TUI, a CLI command), and nothing read the backend's retry-after.+ */+const machineId = 'abcdabcdabcdabcdabcdabcdabcdabcd'+const sink: LocalClientSink = { sendFrame: () => true, sendBinary: () => true }+const select = { type: 'machine_select', payload: { machineId } }+const cleanups: Array<() => Promise<void>> = []+afterEach(async () => { for (const cleanup of cleanups.splice(0)) await cleanup() })++/** A backend that answers every select the way webWs.ts answers a relay's: `refusal` for this machine. */+async function refusingBackend(refusal: Record<string, unknown>) {+ let dials = 0+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1' })+ await new Promise<void>(resolve => wss.once('listening', resolve))+ wss.on('connection', ws => {+ dials+++ ws.send(JSON.stringify({ type: 'connected', payload: { userId: 'user-1' } }))+ ws.on('message', raw => {+ const frame = JSON.parse(raw.toString()) as { type: string }+ if (frame.type === 'machine_select') ws.send(JSON.stringify({ type: 'machine_select_error', payload: { machineId, ...refusal } }))+ })+ })+ let now = 1_000_000+ const peers = new MachinePeerStore(); peers.pin(machineId, b64e(newIdentity().pub), 'test remote')+ const pool = new RemoteRelayPool(+ { accessToken: async () => 'fixture-token' } as never,+ `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`,+ newIdentity(), peers, { p2p: false, now: () => now },+ )+ cleanups.push(async () => {+ pool.close()+ for (const ws of wss.clients) ws.terminate()+ await new Promise<void>(resolve => wss.close(() => resolve()))+ })+ return {+ pool,+ dials: () => dials,+ advance(ms: number) { now += ms },+ window: () => pool.acquire(machineId, 'prod', select, sink, () => {}),+ redial: () => pool.acquire(machineId, 'prod', { ...select, payload: { ...select.payload, forceReconnect: true } }, sink, () => {}),+ background: () => pool.acquireIsolated(machineId, 'prod', select, sink, () => {}),+ }+}++describe('a machine the backend refuses', () => {+ it('is not dialled again, by the window or a background client, until the retry-after has passed', async () => {+ const b = await refusingBackend({ error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 })+ const offline = { message: 'MACHINE_OFFLINE', closeCode: 1013 }+ await expect(b.window()).rejects.toMatchObject(offline)+ expect(b.dials()).toBe(1)++ // The app's retry a second later, a CLI command, a monitor pane: answered here, from the refusal.+ b.advance(1_000)+ await expect(b.window()).rejects.toMatchObject(offline)+ await expect(b.background()).rejects.toMatchObject(offline)+ b.advance(28_000)+ await expect(b.window()).rejects.toMatchObject(offline)+ expect(b.dials()).toBe(1)++ // Once the backend's 30 s are up, the machine is asked again.+ b.advance(1_000)+ await expect(b.window()).rejects.toMatchObject(offline)+ expect(b.dials()).toBe(2)+ })++ it('makes even an explicit redial wait out the retry-after', async () => {+ const b = await refusingBackend({ error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 })+ await expect(b.window()).rejects.toMatchObject({ message: 'MACHINE_OFFLINE' })+ b.advance(5_000)+ await expect(b.redial()).rejects.toMatchObject({ message: 'MACHINE_OFFLINE' })+ expect(b.dials()).toBe(1)+ })++ it('without a retry-after is asked again after 1, 2, 4 … seconds, at most every 30', async () => {+ const b = await refusingBackend({ error: 'NOT_YOUR_MACHINE' })+ const refused = { message: 'NOT_YOUR_MACHINE' }+ const expected: number[] = []+ for (const wait of [1, 2, 4, 8, 16, 30, 30]) {+ await expect(b.window()).rejects.toMatchObject(refused)+ expected.push(b.dials())+ // A select just before the wait is up is answered from the refusal.+ b.advance(wait * 1_000 - 1)+ await expect(b.window()).rejects.toMatchObject(refused)+ expect(b.dials(), `${wait} s`).toBe(expected.length)+ b.advance(1)+ }+ expect(expected).toEqual([1, 2, 3, 4, 5, 6, 7])+ })+})diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dartindex 225320803..1fb391dfe 100644--- a/desktop/lib/state/app_state.dart+++ b/desktop/lib/state/app_state.dart@@ -5347,6 +5347,10 @@ class AppNotifier extends ChangeNotifier {/// What a machine hears the moment its socket is up — first connect, or a/// reconnect after its daemon restarted, which has forgotten all of it.+ /// [machineId]'s socket, if it has one: the test's stand-in, or the pool's.+ WsConn? _connectionOf(String machineId) =>+ connectionForTest?.call(machineId) ?? _pool?[machineId];+void _onMachineConnected(String machineId, MachineState machine) {machine._selectedConnection =connectionForTest?.call(machineId) ?? _pool?[machineId];@@ -8721,9 +8725,45 @@ class AppNotifier extends ChangeNotifier {}}+ /// The backend refused to select [machine] ([code]), straight or through this computer's daemon.+ void _selectRefused(MachineState machine, String code) {+ final name = machine.machine.displayName;+ if (code == 'NOT_YOUR_MACHINE' && machine.machine.machineShare != null) {+ // Its owner stopped sharing it: no retry brings it back, and the list drops it.+ _noteSharingEnded(machine.machine);+ _lastError =+ '${machine.machine.machineShare!.owner} stopped sharing $name with you.';+ _lastErrorRetryable = false;+ _sharingEndedError = (+ machineId: machine.machine.machineId,+ text: _lastError!,+ );+ unawaited(refreshMachines());+ } else {+ _lastError = machineSelectErrorText(code, name);+ _lastErrorRetryable =+ code != 'NOT_YOUR_MACHINE' && code != 'MACHINE_ENV_MISMATCH';+ // Removed from the account: its socket has stopped (WsConn), and the account's list, read+ // again, takes it out of the list and its tabs. Prod, 2026-10-28: a laptop removed in+ // September was still selected every few seconds.+ if (code == 'NOT_YOUR_MACHINE') unawaited(refreshMachines());+ }+ machine.connectionStatus = ConnectionStatus.disconnected;+ }+void _onLocalFailure(String machineId, int code, String reason) {final machine = machineStates[machineId];if (machine == null) return;+ // The daemon's relay closing for a refusal no redial changes (WsConn), with the backend's code.+ if (finalSelectRefusals.contains(reason)) {+ productConnections.refused(+ machineId,+ ProductHealthMeasure.connectErrorOther,+ );+ _selectRefused(machine, reason);+ notifyListeners();+ return;+ }productConnections.refused(machineId,code == 4404@@ -9649,6 +9689,11 @@ class AppNotifier extends ChangeNotifier {(state.nodeOnline == null || state.nodeOnline != reportedOnline)) {unawaited(_applyInventoryNodeStatus(state, reportedOnline));}+ // A relayed machine that was off waits for word it is back (WsConn.isParked), and this+ // list is that word, whether or not this row ever read as offline.+ if (!state.isLocalMachine && reportedOnline == true) {+ _connectionOf(machine.machineId)?.wake('listed online');+ }}// A list that arrived but does not name this computer (a stale copy from before this computer// was paired, say) still gets the daemon's own row, on the same rule as the failure path.@@ -17392,6 +17437,7 @@ class AppNotifier extends ChangeNotifier {_startOfflineRetry(machine);} else {_stopOfflineRetry(machineId);+ if (!machine.isLocalMachine) _connectionOf(machineId)?.wake('online');if (wasOnline == false) {_markSessionsUnreachable(machine,@@ -21341,26 +21387,7 @@ class AppNotifier extends ChangeNotifier {await _applyNodeStatus(machine, online, why: 'node_status');break;case 'machine_select_error':- final code = '${payload['error'] ?? ''}';- final name = machine.machine.displayName;- if (code == 'NOT_YOUR_MACHINE' &&- machine.machine.machineShare != null) {- // Its owner stopped sharing it: no retry brings it back, and the list drops it.- _noteSharingEnded(machine.machine);- _lastError =- '${machine.machine.machineShare!.owner} stopped sharing $name with you.';- _lastErrorRetryable = false;- _sharingEndedError = (- machineId: machine.machine.machineId,- text: _lastError!,- );- unawaited(refreshMachines());- } else {- _lastError = machineSelectErrorText(code, name);- _lastErrorRetryable =- code != 'NOT_YOUR_MACHINE' && code != 'MACHINE_ENV_MISMATCH';- }- machine.connectionStatus = ConnectionStatus.disconnected;+ _selectRefused(machine, '${payload['error'] ?? ''}');break;case 'agent_synced':final raw = payload['agent'];diff --git a/desktop/lib/ws/ws_conn.dart b/desktop/lib/ws/ws_conn.dartindex f5b6e42a0..134a2e467 100644--- a/desktop/lib/ws/ws_conn.dart+++ b/desktop/lib/ws/ws_conn.dart@@ -48,6 +48,18 @@ Duration reconnectDelay(int attempt, Random random) {/// every second, for good: about 54 dials a minute from one browser tab (prod, 2026-10-28).const connectionStableAfter = Duration(minutes: 1);+/// The backend's refusals of a `machine_select` that no redial can change: the machine left the+/// account or was never shared with it, it belongs to another environment, or its subscription stops+/// it (`backend/src/lib/webWs.ts`). Through this computer's daemon they arrive as the relay's close,+/// with the code for its reason (`cli/src/localWsServer.ts`). Prod, 2026-10-28: a laptop removed in+/// September was still selected every few seconds.+const finalSelectRefusals = {+ 'NOT_YOUR_MACHINE',+ 'MACHINE_ENV_MISMATCH',+ 'MACHINE_PAYMENT_PENDING',+ 'MACHINE_SUBSCRIPTION_REQUIRED',+};+/// A `request()` call got no reply within its timeout — distinct from other request-level errors/// (an explicit `{error: ...}` response) so callers can tell "the peer is unresponsive" apart from/// "the peer answered with a failure."@@ -200,6 +212,56 @@ class WsConn {bool get isReady => _ready && _channel != null;+ /// Waiting for its machine, which the daemon's relay said is off ([_park]).+ bool get isParked => _parked;+ bool _parked = false;++ /// The rechecks dialled in a row on their own, each parked again: what doubles the next wait.+ int _parkedRechecks = 0;+ static const _recheck = 'recheck';++ /// The wait before a parked connection dials again on its own after [rechecks] of them in a+ /// row: 30 s, then a minute, 2, 4, 8, and 15 at most. Anything that says the machine is back+ /// [wake]s it. The first is the relay's own retry-after for an offline machine (backend+ /// `OFFLINE_RETRY_AFTER_MS`): a wake that reaches the daemon while it still answers from that+ /// refusal parks again, and a machine back online must not then wait a minute more.+ static Duration parkedRecheckAfter(int rechecks) {+ final wait = const Duration(seconds: 30) * (1 << min(max(rechecks, 0), 5));+ return wait > const Duration(minutes: 15)+ ? const Duration(minutes: 15)+ : wait;+ }++ /// The daemon's relay refused this machine as off (`MACHINE_OFFLINE`): wait for it.+ ///+ /// ⚠️ **Why (prod, 2026-10-28).** Daemon relays made 2,304 of 2,308 backend socket opens in 15+ /// minutes, living a median 0.4 s. A relayed machine that was off was redialled on the 15–30 s+ /// backoff for as long as the app ran, and each dial cost the daemon a backend socket refused at+ /// once: about 2.7 a minute for every machine that was off, in every window.+ void _park() {+ _parked = true;+ _reconnectTimer?.cancel();+ final wait = parkedRecheckAfter(_parkedRechecks);+ _reconnectTimer = Timer(wait, () => wake(_recheck));+ onStatus(ConnectionStatus.reconnecting);+ appLog.info(+ 'ws',+ 'parked $machineId · the relay says it is off; recheck in ${wait.inSeconds}s',+ );+ }++ /// Dial a parked connection now: the machine is reported back, or somebody asks for it. Its own+ /// recheck makes the next wait longer; anything else starts the waits over.+ void wake(String reason) {+ if (!_parked || _closing) return;+ _parked = false;+ _parkedRechecks = reason == _recheck ? _parkedRechecks + 1 : 0;+ _reconnectTimer?.cancel();+ _reconnectTimer = null;+ _attempt = 0;+ unawaited(connect());+ }+/// Wait for this machine's handshake without queuing a request or changing/// its timeout. Callers can then start independent RPCs with their own budgets.Future<void> waitUntilReady({required Duration timeout}) {@@ -338,6 +400,12 @@ class WsConn {Future<void> connect() async {if (_closing || _connecting) return;+ // A dial asked for while parked (a forced redial) is the recheck's too.+ if (_parked) {+ _parked = false;+ _reconnectTimer?.cancel();+ _reconnectTimer = null;+ }_connecting = true;_ready = false;onStatus(@@ -491,6 +559,7 @@ class WsConn {} catch (_) {return;}+ if (message['type'] == 'machine_select_error') _selectRefused(message);if (codec != null) {_inboundTail = _inboundTail.then((_) => _onRelayFrame(codec, message))@@ -525,6 +594,7 @@ class WsConn {void _markReady() {_observerHandshakeTimer?.cancel();+ _parkedRechecks = 0;_ready = true;_refreshedToken = null;_stableTimer?.cancel();@@ -998,6 +1068,17 @@ class WsConn {}final code = channel.closeCode;if (isLocal) {+ if (channel.closeReason == 'MACHINE_OFFLINE') {+ _park();+ return;+ }+ if (finalSelectRefusals.contains(channel.closeReason)) {+ // The relay could not select the machine, for good: told as a 4404 is, never redialled.+ _closing = true;+ onLocalFailure?.call(code ?? 1011, channel.closeReason!);+ onStatus(ConnectionStatus.disconnected);+ return;+ }if (code == 4404 || (observerShareId != null && code == 4403)) {_closing = true;// needsLink first: AppNotifier's onStatus handler reads machine.needsLink@@ -1045,6 +1126,24 @@ class WsConn {_scheduleReconnect();}+ /// The backend refused this socket's select. The app still hears it ([onEvent]); a refusal no+ /// redial can change also ends the connection, which would otherwise sit open and unselected.+ void _selectRefused(Map<String, dynamic> message) {+ final payload = message['payload'];+ if (payload is! Map || payload['machineId'] != machineId) return;+ if (_closing || _ready || !finalSelectRefusals.contains(payload['error'])) {+ return;+ }+ _closing = true;+ _reconnectTimer?.cancel();+ _reconnectTimer = null;+ _disposePlugin();+ onStatus(ConnectionStatus.disconnected);+ final channel = _channel;+ _channel = null;+ unawaited(channel?.sink.close().catchError((Object _) {}));+ }+Future<void> _refreshAndReconnect() async {onStatus(ConnectionStatus.reconnecting);try {diff --git a/desktop/test/machine_sharing_ended_test.dart b/desktop/test/machine_sharing_ended_test.dartindex e80b3a50e..bd186795c 100644--- a/desktop/test/machine_sharing_ended_test.dart+++ b/desktop/test/machine_sharing_ended_test.dart@@ -39,7 +39,7 @@ class _Discovery extends LocalCliDiscovery {}class _Connection extends WsConn {- _Connection(String machineId)+ _Connection(String machineId, {this.onWake}): super(wsBaseUrl: 'ws://fixture.invalid',autonomousEnv: 'test',@@ -49,6 +49,9 @@ class _Connection extends WsConn {onEvent: (_) {},onStatus: (_) {},);+ final void Function(String machineId)? onWake;+ @override+ void wake(String reason) => onWake?.call(machineId);@overrideFuture<void> waitUntilReady({required Duration timeout}) async {}@override@@ -61,13 +64,13 @@ class _Connection extends WsConn {/// Signed in, with this computer's daemon and its machine connections faked.class _App extends AppNotifier {- _App(_Api api)+ _App(_Api api, {void Function(String machineId)? onWake}): super(config: AppConfig.dev,authSession: AuthSession(),configStore: null,localCliDiscovery: _Discovery(),- connectionForTest: _Connection.new,+ connectionForTest: (id) => _Connection(id, onWake: onWake),) {this.api = api;}@@ -188,6 +191,86 @@ void main() {expect(app.lastErrorRetryable, isTrue);});+ // Prod, 2026-10-28: a laptop removed from the account in September was still selected every few+ // seconds. Refused as not on the account, an own machine is read again from the account's list.+ for (final (how, refuse) in <(String, Future<void> Function(AppNotifier))>[+ (+ 'the backend’s refusal',+ (app) => app.handleEventForTest('gone', {+ 'type': 'machine_select_error',+ 'payload': {'machineId': 'gone', 'error': 'NOT_YOUR_MACHINE'},+ }),+ ),+ (+ 'the daemon relay’s close',+ (app) async => app.localFailureForTest('gone', 1011, 'NOT_YOUR_MACHINE'),+ ),+ ]) {+ test('an own machine refused by $how leaves the list', () async {+ const gone = Machine(+ machineId: 'gone',+ authMode: MachineAuthMode.remote,+ name: 'M1 Laptop',+ );+ const kept = Machine(+ machineId: 'kept',+ authMode: MachineAuthMode.remote,+ name: 'Studio',+ );+ final api = _Api()..listed = const [kept];+ app = _App(api)..status = AppStatus.authenticated;+ app.machines = [gone, kept];+ for (final machine in app.machines) {+ app.machineStates[machine.machineId] = MachineState(machine)+ ..nodeOnline = true+ ..connectionStatus = ConnectionStatus.connected;+ }+ await refuse(app);+ for (var i = 0; i < 20; i++) {+ await Future<void>.delayed(Duration.zero);+ }+ final listed = app.machines.map((m) => m.machineId);+ expect(listed, contains('kept'));+ expect(listed, isNot(contains('gone')));+ expect(app.stateOf('gone'), isNull);+ expect(app.lastErrorRetryable, isFalse);+ });+ }++ // A relayed machine that was off waits parked (WsConn.isParked) until it is reported back: the+ // machine list's refresh saying it is online is that word.+ test('a machine the list reports online wakes its parked connection', () async {+ const studio = Machine(+ machineId: 'studio',+ authMode: MachineAuthMode.remote,+ name: 'Studio',+ status: 'online',+ );+ final api = _Api()..listed = const [studio];+ final woken = <String>[];+ app = _App(api, onWake: woken.add)..status = AppStatus.authenticated;+ await app.refreshMachines();+ expect(woken, contains('studio'));+ });++ // The list's word alone, for a row that already reads online: no node-status change runs, so only the list's own+ // wake can reach the parked connection (review of #2027).+ test('a row already online that the list reports online still wakes its parked connection', () async {+ const studio = Machine(+ machineId: 'studio',+ authMode: MachineAuthMode.remote,+ name: 'Studio',+ status: 'online',+ );+ final api = _Api()..listed = const [studio];+ final woken = <String>[];+ app = _App(api, onWake: woken.add)..status = AppStatus.authenticated;+ app.machines = [studio];+ app.machineStates['studio'] = MachineState(studio)..nodeOnline = true;+ await app.refreshMachines();+ expect(woken, contains('studio'));+ });+test('every refusal code reads as a sentence', () {for (final code in ['NOT_YOUR_MACHINE',diff --git a/desktop/test/ws_conn_test.dart b/desktop/test/ws_conn_test.dartindex 1fd282a22..cb06e421d 100644--- a/desktop/test/ws_conn_test.dart+++ b/desktop/test/ws_conn_test.dart@@ -17,6 +17,12 @@ class FakeHub {final HttpServer server;final bool rejectOldToken;final int? closeCodeOnSelect;++ /// The reason [closeCodeOnSelect] closes with: what the daemon's relay says, as it says it.+ String closeReasonOnSelect = 'rejected';++ /// Answers every select with this `machine_select_error` code instead, as the backend does.+ String? selectErrorOnSelect;final List<Map<String, dynamic>> frames = [];final List<String> protocols = [];final List<String?> environments = [];@@ -56,7 +62,19 @@ class FakeHub {if (frame['type'] == 'machine_select') {final machineId = (frame['payload'] as Map)['machineId'] as String;if (hub.closeCodeOnSelect != null) {- ws.close(hub.closeCodeOnSelect!, 'rejected');+ ws.close(hub.closeCodeOnSelect!, hub.closeReasonOnSelect);+ return;+ }+ if (hub.selectErrorOnSelect != null) {+ ws.add(+ jsonEncode({+ 'type': 'machine_select_error',+ 'payload': {+ 'machineId': machineId,+ 'error': hub.selectErrorOnSelect,+ },+ }),+ );return;}if (hub.rejectOldToken && protocol == 'tok-old') {@@ -509,6 +527,87 @@ void main() {expect(conn!.isClosed, isTrue);});+ // Prod, 2026-10-28: a laptop removed from the account in September was still selected every few+ // seconds. Through this computer's daemon the refusal arrives as the relay's close, with the+ // backend's code for its reason; straight from the backend (the web viewer), as the frame.+ for (final (path, local) in [('daemon relay', true), ('backend', false)]) {+ test(+ 'a machine refused as not on the account ($path) is not dialled again',+ () async {+ hub = await FakeHub.start(closeCodeOnSelect: local ? 1011 : null);+ hub.closeReasonOnSelect = 'NOT_YOUR_MACHINE';+ if (!local) hub.selectErrorOnSelect = 'NOT_YOUR_MACHINE';+ final failures = <String>[];+ final statuses = <ConnectionStatus>[];+ conn = WsConn(+ wsBaseUrl: 'ws://127.0.0.1:${hub.port}',+ autonomousEnv: 'prod',+ machineId: 'm1',+ accessTokenProvider: (_, _) async => 'sso-token',+ onAuthFailure: (_) {},+ onLocalFailure: (code, reason) => failures.add(reason),+ onEvent: (_) {},+ onStatus: statuses.add,+ transportKind: local+ ? WsTransportKind.localPlaintext+ : WsTransportKind.cloudE2ee,+ allowLocalTcpFixture: true,+ localWsUri: local+ ? Uri.parse('ws://127.0.0.1:${hub.port}/api/local-ws')+ : null,+ );+ await conn!.connect();+ // Past the first two steps of the backoff (0.5–1 s, then 1–2 s).+ await Future<void>.delayed(const Duration(milliseconds: 3200));+ expect(hub.protocols, hasLength(1));+ expect(conn!.isClosed, isTrue);+ expect(statuses.last, ConnectionStatus.disconnected);+ if (local) expect(failures, ['NOT_YOUR_MACHINE']);+ },+ );+ }++ // Prod, 2026-10-28: relays living 0.4 s made 2,304 of 2,308 backend socket opens in 15 minutes.+ // A relayed machine that is off was redialled on the 15–30 s backoff for as long as the app ran,+ // each dial refused at once: about 2.7 a minute for every machine that was off. It now waits for+ // word that the machine is back, and rechecks on its own only every 1, 2, 4, 8, then 15 minutes.+ test('a relayed machine that is off waits to be woken instead of redialling', () async {+ // The reason is what counts: a current daemon closes with 1013, an older one with 1011 (a+ // dart:io server cannot send 1013, a reserved code).+ hub = await FakeHub.start(closeCodeOnSelect: 1011);+ hub.closeReasonOnSelect = 'MACHINE_OFFLINE';+ conn = WsConn(+ wsBaseUrl: 'wss://unused.example',+ autonomousEnv: 'prod',+ machineId: 'm1',+ accessTokenProvider: (_, _) async => 'sso-token',+ onAuthFailure: (_) {},+ onEvent: (_) {},+ onStatus: (_) {},+ transportKind: WsTransportKind.localPlaintext,+ allowLocalTcpFixture: true,+ localWsUri: Uri.parse('ws://127.0.0.1:${hub.port}/api/local-ws'),+ );+ await conn!.connect();+ await Future<void>.delayed(const Duration(milliseconds: 3200));+ expect(hub.protocols, hasLength(1));+ expect(conn!.isParked, isTrue);+ expect(conn!.isClosed, isFalse);++ conn!.wake('machine online');+ await Future<void>.delayed(const Duration(milliseconds: 250));+ expect(hub.protocols, hasLength(2));+ });++ // The first recheck is the relay's own retry-after (30 s): a machine back online that a wake reached while the relay+ // still answered from its refusal parks again, and must not wait a minute more (review of #2027).+ test('a parked connection rechecks after 30 s, then 1, 2, 4, 8 and every 15 minutes', () {+ expect(+ [for (var n = 0; n < 7; n++) WsConn.parkedRecheckAfter(n).inSeconds],+ [30, 60, 120, 240, 480, 900, 900],+ );+ });+test('a fixed reconnect delay retries at that pace instead of backing off',() async {diff --git a/mobile/lib/state/app_state.dart b/mobile/lib/state/app_state.dartindex 6fda1f981..010212c71 100644--- a/mobile/lib/state/app_state.dart+++ b/mobile/lib/state/app_state.dart@@ -10334,9 +10334,20 @@ class AppNotifier extends ChangeNotifier {_wakeMachinesBackOnline(payload['statuses']);return;case 'machine_select_error':- _lastError =- 'Machine selection failed: ${payload['error'] ?? 'unknown error'}';- _lastErrorRetryable = true;+ if (payload['error'] == 'NOT_YOUR_MACHINE') {+ // Removed from the account, or no longer shared with it: no redial brings it back, and+ // its socket has stopped (WsConn._selectRefused). The account's list is read again so the+ // machine leaves the list, its tabs and the cache it came from. Prod, 2026-10-28: a laptop+ // removed in September was still selected every ~7 s.+ _lastError =+ '${machine.machine.displayName} is no longer on your account.';+ _lastErrorRetryable = false;+ unawaited(refreshMachines());+ } else {+ _lastError =+ 'Machine selection failed: ${payload['error'] ?? 'unknown error'}';+ _lastErrorRetryable = true;+ }machine.connectionStatus = ConnectionStatus.disconnected;break;case 'agent_synced':diff --git a/mobile/lib/ws/ws_conn.dart b/mobile/lib/ws/ws_conn.dartindex d93f1c5f3..5f00c5777 100644--- a/mobile/lib/ws/ws_conn.dart+++ b/mobile/lib/ws/ws_conn.dart@@ -506,6 +506,46 @@ class WsConn {unawaited(channel.sink.close().catchError((Object _) {}));}+ /// Refusals no redial can change: the machine left the account or was never shared with it, it+ /// belongs to another environment, or its subscription stops it. Every code the relay's+ /// `machine_select` answers with besides these may pass (`backend/src/lib/webWs.ts`).+ static const _finalSelectRefusals = {+ 'NOT_YOUR_MACHINE',+ 'MACHINE_ENV_MISMATCH',+ 'MACHINE_PAYMENT_PENDING',+ 'MACHINE_SUBSCRIPTION_REQUIRED',+ };++ /// The relay refused this connection's select. The app still hears the refusal ([onEvent]); the+ /// connection stops here for good when no redial can change it, or else waits out its backoff.+ ///+ /// ⚠️ **Why (prod, 2026-10-28).** Nothing here read the refusal, so [_armSelectWatchdog] took the+ /// silence after it for a relay gone quiet and redialled six seconds later, for as long as the app+ /// was open: a laptop removed from the account in September was still selected every ~7 s, about+ /// 20 dials in ten minutes from one phone, each a fresh credential, key and TLS dial.+ void _selectRefused(Map<String, dynamic> message) {+ final payload = message['payload'];+ if (payload is! Map || payload['machineId'] != machineId) return;+ if (_closing || _ready) return;+ _selectWatchdog?.cancel();+ _selectWatchdog = null;+ final code = '${payload['error'] ?? ''}';+ appLog.info('ws', 'machine_select refused for $machineId: $code');+ if (!_finalSelectRefusals.contains(code)) {+ _abandonDial(_channel);+ _scheduleReconnect();+ return;+ }+ _closing = true;+ _reconnectTimer?.cancel();+ _reconnectTimer = null;+ _disposePlugin();+ onStatus(ConnectionStatus.disconnected);+ final channel = _channel;+ _channel = null;+ unawaited(channel?.sink.close().catchError((Object _) {}));+ }+void _signOut(String message) {_closing = true;onStatus(ConnectionStatus.disconnected);@@ -623,6 +663,7 @@ class WsConn {} catch (_) {return;}+ if (message['type'] == 'machine_select_error') _selectRefused(message);if (codec != null) {_inboundTail = _inboundTail.then((_) {diff --git a/mobile/test/app_state_machine_events_test.dart b/mobile/test/app_state_machine_events_test.dartindex 01897d43e..47d171747 100644--- a/mobile/test/app_state_machine_events_test.dart+++ b/mobile/test/app_state_machine_events_test.dart@@ -705,6 +705,29 @@ void main() {);});+ test('a machine refused as not on the account is read again and leaves the list', () async {+ // Prod, 2026-10-28: a laptop removed in September was still selected every ~7 s.+ final rig = await signedInWith({+ 'm': ['a'],+ 'gone': ['b'],+ });+ addTearDown(rig.app.dispose);+ final fetches = rig.api.machineFetches;+ rig.api.onMachines = () async => [remoteMachine('m')];++ await push(rig, 'gone', 'machine_select_error', {+ 'machineId': 'gone',+ 'error': 'NOT_YOUR_MACHINE',+ });+ await settle();++ expect(rig.api.machineFetches, fetches + 1);+ expect(rig.app.stateOf('gone'), isNull);+ expect(rig.app.machines.map((m) => m.machineId), ['m']);+ expect(rig.app.lastError, 'Machine gone is no longer on your account.');+ expect(rig.app.lastErrorRetryable, isFalse);+ });+test('pushes for a machine the phone does not have are ignored', () async {final rig = await signedInWith({'m': ['a'],diff --git a/mobile/test/ws/ws_conn_refused_test.dart b/mobile/test/ws/ws_conn_refused_test.dartnew file mode 100644index 000000000..1cb494aa7--- /dev/null+++ b/mobile/test/ws/ws_conn_refused_test.dart@@ -0,0 +1,91 @@+import 'dart:async';+import 'dart:convert';++import 'package:fake_async/fake_async.dart';+import 'package:flutter_test/flutter_test.dart';++import 'package:harness_mobile/core/models.dart' show ConnectionStatus;+import 'package:harness_mobile/ws/ws_conn.dart';++import 'memory_web_socket.dart';++/// A machine the relay refuses — removed from the account (prod, 2026-10-28: a laptop deleted in+/// September, still selected every ~7 s), no longer shared, another environment, unpaid — is not+/// dialled again. The select watchdog read the silence after the refusal as a relay gone quiet and+/// redialled it after six seconds, for as long as the app was open.+void main() {+ void refused(+ String error,+ void Function(+ FakeAsync clock,+ WsConn conn,+ List<MemoryWebSocket> relay,+ List<ConnectionStatus> statuses,+ )+ body,+ ) {+ fakeAsync((clock) {+ final relay = <MemoryWebSocket>[];+ final statuses = <ConnectionStatus>[];+ final conn = WsConn(+ wsBaseUrl: 'ws://fixture.invalid',+ connectChannel: (_, {protocols}) {+ final (client, server) = MemoryWebSocket.pair();+ server.stream.listen((raw) {+ final frame = jsonDecode(raw as String) as Map<String, dynamic>;+ if (frame['type'] != 'machine_select') return;+ server.sink.add(+ jsonEncode({+ 'type': 'machine_select_error',+ 'payload': {'machineId': 'm', 'error': error},+ }),+ );+ });+ relay.add(server);+ client.accept();+ return client;+ },+ autonomousEnv: 'test',+ machineId: 'm',+ accessTokenProvider: (_, _) async => 'token',+ onAuthFailure: (_) {},+ onEvent: (_) {},+ onStatus: statuses.add,+ );+ unawaited(conn.connect());+ clock.flushMicrotasks();+ body(clock, conn, relay, statuses);+ unawaited(conn.close());+ clock.flushMicrotasks();+ });+ }++ for (final error in [+ 'NOT_YOUR_MACHINE',+ 'MACHINE_ENV_MISMATCH',+ 'MACHINE_PAYMENT_PENDING',+ 'MACHINE_SUBSCRIPTION_REQUIRED',+ ]) {+ test('$error is not redialled', () {+ refused(error, (clock, conn, relay, statuses) {+ clock.elapse(const Duration(minutes: 10));+ expect(relay, hasLength(1));+ expect(conn.isClosed, isTrue);+ expect(statuses.last, ConnectionStatus.disconnected);+ });+ });+ }++ test('a refusal it may outlive (a machine that failed to start) waits out the backoff', () {+ refused('MACHINE_START_FAILED', (clock, conn, relay, statuses) {+ // Not the six-second watchdog: the ordinary 1, 2, 4 … 30 s backoff, from the refusal on.+ clock.elapse(const Duration(milliseconds: 999));+ expect(relay, hasLength(1));+ clock.elapse(const Duration(milliseconds: 1));+ expect(relay, hasLength(2));+ clock.elapse(const Duration(seconds: 2));+ expect(relay, hasLength(3));+ expect(conn.isClosed, isFalse);+ });+ });+}