Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

WebSocket churn: failed selects back off; offline and gone machines stop redialling (relay, Desktop, Mobile) (#2027)

0xken158013e40f2d21
diff --git a/cli/src/lib/relayFrames.ts b/cli/src/lib/relayFrames.ts
index 678f5826a..01782a314 100644
--- a/cli/src/lib/relayFrames.ts
+++ b/cli/src/lib/relayFrames.ts
@@ -56,7 +56,8 @@ export const GATEWAY_REQUEST_TYPES = new Set(['device_e2ee_pair', 'phone_pair',
/** A window's connection to another machine could not be made: why, and the close code to tell it. Here and
* not with the relay pool, so the local socket can tell one apart without loading the pool. */
export class RelayConnectError extends Error {
- constructor(message: string, readonly closeCode?: number) {
+ /** `retryAfterMs`: how long the backend asked not to be asked again (its MACHINE_OFFLINE refusal of a relay). */
+ constructor(message: string, readonly closeCode?: number, readonly retryAfterMs?: number) {
super(message)
this.name = 'RelayConnectError'
}
diff --git a/cli/src/lib/remoteRelay.spec.ts b/cli/src/lib/remoteRelay.spec.ts
index 4fae3d3b0..a2e5a61e7 100644
--- a/cli/src/lib/remoteRelay.spec.ts
+++ b/cli/src/lib/remoteRelay.spec.ts
@@ -822,14 +822,18 @@ describe('RemoteRelayPool keeps background (isolated) clients cheap', () => {
}
})
- it('the window pool has no cooldown: a person clicking again always gets a fresh dial', async () => {
+ it('the window pool backs a failed machine off, but an explicit redial always gets a fresh dial', async () => {
const pool = new RemoteRelayPool(...ctorArgs) as unknown as {
connect: (machineId: string, env: string, frame: unknown) => Promise<unknown>
dial: (...args: unknown[]) => Promise<unknown>
}
const dial = vi.spyOn(pool, 'dial').mockRejectedValue(new Error('relay connect timed out'))
await expect(pool.connect('m1', 'env', select)).rejects.toThrow()
+ // The app's next select a moment later is answered from the failure (remoteRelayRefusal.spec.ts).
await expect(pool.connect('m1', 'env', select)).rejects.toThrow()
+ expect(dial).toHaveBeenCalledTimes(1)
+ // A redial the app asks for explicitly is dialled at once.
+ await expect(pool.connect('m1', 'env', { ...select, payload: { ...select.payload, forceReconnect: true } })).rejects.toThrow()
expect(dial).toHaveBeenCalledTimes(2)
})
})
diff --git a/cli/src/lib/remoteRelay.ts b/cli/src/lib/remoteRelay.ts
index 9bac07c5d..b03751a8e 100644
--- a/cli/src/lib/remoteRelay.ts
+++ b/cli/src/lib/remoteRelay.ts
@@ -49,9 +49,17 @@ const LINGER_MS = 30_000
const ISOLATED_LINGER_MS = 3 * 60_000
// After a FAILED background dial, how long the same machine is answered from that failure instead of
// dialled again. A poller that retries on every tick against an unlinked or unreachable machine would
-// otherwise turn one bad machine into a relay dial per tick, forever. Never applied to the window's
-// pool: a person clicking again deserves a fresh attempt.
+// otherwise turn one bad machine into a relay dial per tick, forever. The window's pool has the shorter backoff
+// below instead, which an explicit redial skips: a person clicking again deserves a fresh attempt.
const ISOLATED_DIAL_COOLDOWN_MS = 30_000
+// After a machine was refused or could not be reached, every pool answers its next selects from that failure
+// for 1 s, then 2, 4 … up to this, or for as long as the backend asked (its MACHINE_OFFLINE retry-after, 30 s),
+// whichever is longer. Prod, 2026-10-28: 2,304 of 2,308 web-ws opens in 15 minutes were daemon relays living a
+// median 0.4 s, open, sign in, select, refused, closed, because the window's pool dialled again on every
+// select its local clients made: the app's retry, a TUI, a CLI command.
+const REFUSED_BACKOFF_MAX_MS = 30_000
+// A retry-after longer than this is read as this: a wrong answer must not shut a machine out for long.
+const RETRY_AFTER_MAX_MS = 5 * 60_000
// Warm background sessions kept per machine. Each concurrent background client needs its own (they
// must not see each other's frames), so the cap is "how many pollers may overlap on one machine and
// still find a warm session"; beyond it the extra session is torn down on detach as it always was.
@@ -260,8 +268,16 @@ export interface RemoteRelayPoolOptions {
expectsTrust?: (machineId: string, pub: string) => boolean
/** Shared record of recent denials per machine, for the above. */
denials?: Map<string, number[]>
+ /** Shared record of the machines being backed off after a failed dial (REFUSED_BACKOFF_MAX_MS). */
+ refusals?: Map<string, Refusal>
+ /** The clock the backoff reads; a test's own. */
+ now?: () => number
}
+/** A machine whose last dials failed `count` times in a row: selects of it before `until` get `error` back.
+ * `asked`: the backend set `until` (its retry-after), which even an explicit redial waits out. */
+interface Refusal { until: number; error: unknown; count: number; asked: boolean }
+
const DENIED_TRIES = 3
const DENIED_WINDOW_MS = 60_000
@@ -278,6 +294,8 @@ export class RemoteRelayPool {
private readonly onSessionReady: ((machineId: string) => void) | null
private readonly expectsTrust: ((machineId: string, pub: string) => boolean) | null
private readonly denials: Map<string, number[]>
+ private readonly refusals: Map<string, Refusal>
+ private readonly now: () => number
/** Warm background pools per machine, each holding one lingering session nobody is attached to —
* see acquireIsolated(). A pool is either here (idle) or in a client's hands, never both. */
private readonly idleIsolated = new Map<string, RemoteRelayPool[]>()
@@ -300,6 +318,8 @@ export class RemoteRelayPool {
this.lastDialFailure = opts.dialFailures ?? new Map()
this.expectsTrust = opts.expectsTrust ?? null
this.denials = opts.denials ?? new Map()
+ this.refusals = opts.refusals ?? new Map()
+ this.now = opts.now ?? Date.now
}
/** `machineId` denied this machine's key. Drop the pin — unless the device key log says it should
@@ -334,7 +354,7 @@ export class RemoteRelayPool {
const shelf = this.idleIsolated.get(machineId) ?? []
const pool = shelf.pop() ?? new RemoteRelayPool(this.auth, this.backendWsBase, this.selfIdentity, this.peers,
{ p2p: false, lingerMs: ISOLATED_LINGER_MS, dialCooldownMs: ISOLATED_DIAL_COOLDOWN_MS, dialFailures: this.lastDialFailure,
- ...(this.expectsTrust ? { expectsTrust: this.expectsTrust } : {}), denials: this.denials })
+ ...(this.expectsTrust ? { expectsTrust: this.expectsTrust } : {}), denials: this.denials, refusals: this.refusals, now: this.now })
const session = await pool.acquire(machineId, autonomousEnv, selectFrame, {
...sink,
sendFrame: frame => sink.sendFrame(frame.type === 'connected'
@@ -431,6 +451,11 @@ export class RemoteRelayPool {
}
private connect(machineId: string, autonomousEnv: string, selectFrame: Frame): Promise<Entry> {
+ // An explicit redial (`forceReconnect`: the app saw its session die, or a person asked again) is not made to
+ // wait out the backoff, only what the backend itself asked for.
+ const redial = (selectFrame.payload as { forceReconnect?: unknown } | undefined)?.forceReconnect === true
+ const refused = this.refusals.get(machineId)
+ if (refused && this.now() < refused.until && (refused.asked || !redial)) return Promise.reject(refused.error)
const failed = this.lastDialFailure.get(machineId)
if (failed && this.dialCooldownMs > 0 && Date.now() - failed.at < this.dialCooldownMs) {
return Promise.reject(failed.error)
@@ -450,6 +475,7 @@ export class RemoteRelayPool {
(error) => { this.lastDialFailure.set(machineId, { at: Date.now(), error }) },
)
}
+ promise.then(() => { this.refusals.delete(machineId) }, (error) => { this.backOff(machineId, error) })
this.pending.set(machineId, promise)
// `.finally()` re-throws on rejection, producing a SECOND promise distinct from the one returned
// below (which callers already await/catch) — left un-caught, every failed dial (e.g. NO_PEER_LINK
@@ -460,6 +486,17 @@ export class RemoteRelayPool {
return promise
}
+ /** A dial of `machineId` failed: hold its next selects for the backoff's next step, or the backend's retry-after.
+ * An unlinked machine is not held: nothing was dialled, and a link made a moment later must work at once. */
+ private backOff(machineId: string, error: unknown): void {
+ if (error instanceof RelayConnectError && error.message === 'NO_PEER_LINK') return
+ const count = (this.refusals.get(machineId)?.count ?? 0) + 1
+ const backoff = Math.min(REFUSED_BACKOFF_MAX_MS, 1_000 * 2 ** (count - 1))
+ const asked = error instanceof RelayConnectError && typeof error.retryAfterMs === 'number' && Number.isFinite(error.retryAfterMs)
+ ? Math.min(Math.max(error.retryAfterMs, 0), RETRY_AFTER_MAX_MS) : 0
+ this.refusals.set(machineId, { until: this.now() + Math.max(backoff, asked), error, count, asked: asked > backoff })
+ }
+
private async dial(machineId: string, autonomousEnv: string, selectFrame: Frame, forceRefresh: boolean): Promise<Entry> {
const peer = this.peers.get(machineId)
if (!peer) throw new RelayConnectError('NO_PEER_LINK')
@@ -560,7 +597,11 @@ export class RemoteRelayPool {
if (frame.type === 'machine_select_error' && payload?.machineId === machineId) {
if (!settled) {
settled = true; clearTimeout(timeout)
- reject(new RelayConnectError(typeof payload.error === 'string' ? payload.error : 'machine_select_error'))
+ const code = typeof payload.error === 'string' ? payload.error : 'machine_select_error'
+ const retryAfterMs = (payload as { retryAfterMs?: unknown }).retryAfterMs
+ // Offline closes the window's socket as the node_status answer below does (1013, try again later).
+ reject(new RelayConnectError(code, code === 'MACHINE_OFFLINE' ? 1013 : undefined,
+ typeof retryAfterMs === 'number' ? retryAfterMs : undefined))
}
return
}
diff --git a/cli/src/lib/remoteRelayRefusal.spec.ts b/cli/src/lib/remoteRelayRefusal.spec.ts
new file mode 100644
index 000000000..9a66eb66b
--- /dev/null
+++ b/cli/src/lib/remoteRelayRefusal.spec.ts
@@ -0,0 +1,100 @@
+import { afterEach, describe, expect, it } from 'vitest'
+import { WebSocketServer } from 'ws'
+import type { AddressInfo } from 'node:net'
+import { RemoteRelayPool } from './remoteRelay.js'
+import { MachinePeerStore } from './e2ee/machinePeers.js'
+import { b64e, newIdentity } from './e2ee/core.js'
+import type { LocalClientSink } from '../backendSocket.js'
+
+/**
+ * A machine the backend refuses must not cost a backend socket on every select a local client makes. Prod,
+ * 2026-10-28: 2,304 of 2,308 web-ws opens in 15 minutes came from daemon relays, median life 0.4 s — open, sign
+ * in, select, refused, closed — because the window's pool dialled again on every select (the app's retry, a
+ * TUI, a CLI command), and nothing read the backend's retry-after.
+ */
+const machineId = 'abcdabcdabcdabcdabcdabcdabcdabcd'
+const sink: LocalClientSink = { sendFrame: () => true, sendBinary: () => true }
+const select = { type: 'machine_select', payload: { machineId } }
+const cleanups: Array<() => Promise<void>> = []
+afterEach(async () => { for (const cleanup of cleanups.splice(0)) await cleanup() })
+
+/** A backend that answers every select the way webWs.ts answers a relay's: `refusal` for this machine. */
+async function refusingBackend(refusal: Record<string, unknown>) {
+ let dials = 0
+ const wss = new WebSocketServer({ port: 0, host: '127.0.0.1' })
+ await new Promise<void>(resolve => wss.once('listening', resolve))
+ wss.on('connection', ws => {
+ dials++
+ ws.send(JSON.stringify({ type: 'connected', payload: { userId: 'user-1' } }))
+ ws.on('message', raw => {
+ const frame = JSON.parse(raw.toString()) as { type: string }
+ if (frame.type === 'machine_select') ws.send(JSON.stringify({ type: 'machine_select_error', payload: { machineId, ...refusal } }))
+ })
+ })
+ let now = 1_000_000
+ const peers = new MachinePeerStore(); peers.pin(machineId, b64e(newIdentity().pub), 'test remote')
+ const pool = new RemoteRelayPool(
+ { accessToken: async () => 'fixture-token' } as never,
+ `ws://127.0.0.1:${(wss.address() as AddressInfo).port}`,
+ newIdentity(), peers, { p2p: false, now: () => now },
+ )
+ cleanups.push(async () => {
+ pool.close()
+ for (const ws of wss.clients) ws.terminate()
+ await new Promise<void>(resolve => wss.close(() => resolve()))
+ })
+ return {
+ pool,
+ dials: () => dials,
+ advance(ms: number) { now += ms },
+ window: () => pool.acquire(machineId, 'prod', select, sink, () => {}),
+ redial: () => pool.acquire(machineId, 'prod', { ...select, payload: { ...select.payload, forceReconnect: true } }, sink, () => {}),
+ background: () => pool.acquireIsolated(machineId, 'prod', select, sink, () => {}),
+ }
+}
+
+describe('a machine the backend refuses', () => {
+ it('is not dialled again, by the window or a background client, until the retry-after has passed', async () => {
+ const b = await refusingBackend({ error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 })
+ const offline = { message: 'MACHINE_OFFLINE', closeCode: 1013 }
+ await expect(b.window()).rejects.toMatchObject(offline)
+ expect(b.dials()).toBe(1)
+
+ // The app's retry a second later, a CLI command, a monitor pane: answered here, from the refusal.
+ b.advance(1_000)
+ await expect(b.window()).rejects.toMatchObject(offline)
+ await expect(b.background()).rejects.toMatchObject(offline)
+ b.advance(28_000)
+ await expect(b.window()).rejects.toMatchObject(offline)
+ expect(b.dials()).toBe(1)
+
+ // Once the backend's 30 s are up, the machine is asked again.
+ b.advance(1_000)
+ await expect(b.window()).rejects.toMatchObject(offline)
+ expect(b.dials()).toBe(2)
+ })
+
+ it('makes even an explicit redial wait out the retry-after', async () => {
+ const b = await refusingBackend({ error: 'MACHINE_OFFLINE', retryAfterMs: 30_000 })
+ await expect(b.window()).rejects.toMatchObject({ message: 'MACHINE_OFFLINE' })
+ b.advance(5_000)
+ await expect(b.redial()).rejects.toMatchObject({ message: 'MACHINE_OFFLINE' })
+ expect(b.dials()).toBe(1)
+ })
+
+ it('without a retry-after is asked again after 1, 2, 4 … seconds, at most every 30', async () => {
+ const b = await refusingBackend({ error: 'NOT_YOUR_MACHINE' })
+ const refused = { message: 'NOT_YOUR_MACHINE' }
+ const expected: number[] = []
+ for (const wait of [1, 2, 4, 8, 16, 30, 30]) {
+ await expect(b.window()).rejects.toMatchObject(refused)
+ expected.push(b.dials())
+ // A select just before the wait is up is answered from the refusal.
+ b.advance(wait * 1_000 - 1)
+ await expect(b.window()).rejects.toMatchObject(refused)
+ expect(b.dials(), `${wait} s`).toBe(expected.length)
+ b.advance(1)
+ }
+ expect(expected).toEqual([1, 2, 3, 4, 5, 6, 7])
+ })
+})
diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dart
index 225320803..1fb391dfe 100644
--- a/desktop/lib/state/app_state.dart
+++ b/desktop/lib/state/app_state.dart
@@ -5347,6 +5347,10 @@ class AppNotifier extends ChangeNotifier {
/// What a machine hears the moment its socket is up — first connect, or a
/// reconnect after its daemon restarted, which has forgotten all of it.
+ /// [machineId]'s socket, if it has one: the test's stand-in, or the pool's.
+ WsConn? _connectionOf(String machineId) =>
+ connectionForTest?.call(machineId) ?? _pool?[machineId];
+
void _onMachineConnected(String machineId, MachineState machine) {
machine._selectedConnection =
connectionForTest?.call(machineId) ?? _pool?[machineId];
@@ -8721,9 +8725,45 @@ class AppNotifier extends ChangeNotifier {
}
}
+ /// The backend refused to select [machine] ([code]), straight or through this computer's daemon.
+ void _selectRefused(MachineState machine, String code) {
+ final name = machine.machine.displayName;
+ if (code == 'NOT_YOUR_MACHINE' && machine.machine.machineShare != null) {
+ // Its owner stopped sharing it: no retry brings it back, and the list drops it.
+ _noteSharingEnded(machine.machine);
+ _lastError =
+ '${machine.machine.machineShare!.owner} stopped sharing $name with you.';
+ _lastErrorRetryable = false;
+ _sharingEndedError = (
+ machineId: machine.machine.machineId,
+ text: _lastError!,
+ );
+ unawaited(refreshMachines());
+ } else {
+ _lastError = machineSelectErrorText(code, name);
+ _lastErrorRetryable =
+ code != 'NOT_YOUR_MACHINE' && code != 'MACHINE_ENV_MISMATCH';
+ // Removed from the account: its socket has stopped (WsConn), and the account's list, read
+ // again, takes it out of the list and its tabs. Prod, 2026-10-28: a laptop removed in
+ // September was still selected every few seconds.
+ if (code == 'NOT_YOUR_MACHINE') unawaited(refreshMachines());
+ }
+ machine.connectionStatus = ConnectionStatus.disconnected;
+ }
+
void _onLocalFailure(String machineId, int code, String reason) {
final machine = machineStates[machineId];
if (machine == null) return;
+ // The daemon's relay closing for a refusal no redial changes (WsConn), with the backend's code.
+ if (finalSelectRefusals.contains(reason)) {
+ productConnections.refused(
+ machineId,
+ ProductHealthMeasure.connectErrorOther,
+ );
+ _selectRefused(machine, reason);
+ notifyListeners();
+ return;
+ }
productConnections.refused(
machineId,
code == 4404
@@ -9649,6 +9689,11 @@ class AppNotifier extends ChangeNotifier {
(state.nodeOnline == null || state.nodeOnline != reportedOnline)) {
unawaited(_applyInventoryNodeStatus(state, reportedOnline));
}
+ // A relayed machine that was off waits for word it is back (WsConn.isParked), and this
+ // list is that word, whether or not this row ever read as offline.
+ if (!state.isLocalMachine && reportedOnline == true) {
+ _connectionOf(machine.machineId)?.wake('listed online');
+ }
}
// A list that arrived but does not name this computer (a stale copy from before this computer
// was paired, say) still gets the daemon's own row, on the same rule as the failure path.
@@ -17392,6 +17437,7 @@ class AppNotifier extends ChangeNotifier {
_startOfflineRetry(machine);
} else {
_stopOfflineRetry(machineId);
+ if (!machine.isLocalMachine) _connectionOf(machineId)?.wake('online');
if (wasOnline == false) {
_markSessionsUnreachable(
machine,
@@ -21341,26 +21387,7 @@ class AppNotifier extends ChangeNotifier {
await _applyNodeStatus(machine, online, why: 'node_status');
break;
case 'machine_select_error':
- final code = '${payload['error'] ?? ''}';
- final name = machine.machine.displayName;
- if (code == 'NOT_YOUR_MACHINE' &&
- machine.machine.machineShare != null) {
- // Its owner stopped sharing it: no retry brings it back, and the list drops it.
- _noteSharingEnded(machine.machine);
- _lastError =
- '${machine.machine.machineShare!.owner} stopped sharing $name with you.';
- _lastErrorRetryable = false;
- _sharingEndedError = (
- machineId: machine.machine.machineId,
- text: _lastError!,
- );
- unawaited(refreshMachines());
- } else {
- _lastError = machineSelectErrorText(code, name);
- _lastErrorRetryable =
- code != 'NOT_YOUR_MACHINE' && code != 'MACHINE_ENV_MISMATCH';
- }
- machine.connectionStatus = ConnectionStatus.disconnected;
+ _selectRefused(machine, '${payload['error'] ?? ''}');
break;
case 'agent_synced':
final raw = payload['agent'];
diff --git a/desktop/lib/ws/ws_conn.dart b/desktop/lib/ws/ws_conn.dart
index f5b6e42a0..134a2e467 100644
--- a/desktop/lib/ws/ws_conn.dart
+++ b/desktop/lib/ws/ws_conn.dart
@@ -48,6 +48,18 @@ Duration reconnectDelay(int attempt, Random random) {
/// every second, for good: about 54 dials a minute from one browser tab (prod, 2026-10-28).
const connectionStableAfter = Duration(minutes: 1);
+/// The backend's refusals of a `machine_select` that no redial can change: the machine left the
+/// account or was never shared with it, it belongs to another environment, or its subscription stops
+/// it (`backend/src/lib/webWs.ts`). Through this computer's daemon they arrive as the relay's close,
+/// with the code for its reason (`cli/src/localWsServer.ts`). Prod, 2026-10-28: a laptop removed in
+/// September was still selected every few seconds.
+const finalSelectRefusals = {
+ 'NOT_YOUR_MACHINE',
+ 'MACHINE_ENV_MISMATCH',
+ 'MACHINE_PAYMENT_PENDING',
+ 'MACHINE_SUBSCRIPTION_REQUIRED',
+};
+
/// A `request()` call got no reply within its timeout — distinct from other request-level errors
/// (an explicit `{error: ...}` response) so callers can tell "the peer is unresponsive" apart from
/// "the peer answered with a failure."
@@ -200,6 +212,56 @@ class WsConn {
bool get isReady => _ready && _channel != null;
+ /// Waiting for its machine, which the daemon's relay said is off ([_park]).
+ bool get isParked => _parked;
+ bool _parked = false;
+
+ /// The rechecks dialled in a row on their own, each parked again: what doubles the next wait.
+ int _parkedRechecks = 0;
+ static const _recheck = 'recheck';
+
+ /// The wait before a parked connection dials again on its own after [rechecks] of them in a
+ /// row: 30 s, then a minute, 2, 4, 8, and 15 at most. Anything that says the machine is back
+ /// [wake]s it. The first is the relay's own retry-after for an offline machine (backend
+ /// `OFFLINE_RETRY_AFTER_MS`): a wake that reaches the daemon while it still answers from that
+ /// refusal parks again, and a machine back online must not then wait a minute more.
+ static Duration parkedRecheckAfter(int rechecks) {
+ final wait = const Duration(seconds: 30) * (1 << min(max(rechecks, 0), 5));
+ return wait > const Duration(minutes: 15)
+ ? const Duration(minutes: 15)
+ : wait;
+ }
+
+ /// The daemon's relay refused this machine as off (`MACHINE_OFFLINE`): wait for it.
+ ///
+ /// ⚠️ **Why (prod, 2026-10-28).** Daemon relays made 2,304 of 2,308 backend socket opens in 15
+ /// minutes, living a median 0.4 s. A relayed machine that was off was redialled on the 15–30 s
+ /// backoff for as long as the app ran, and each dial cost the daemon a backend socket refused at
+ /// once: about 2.7 a minute for every machine that was off, in every window.
+ void _park() {
+ _parked = true;
+ _reconnectTimer?.cancel();
+ final wait = parkedRecheckAfter(_parkedRechecks);
+ _reconnectTimer = Timer(wait, () => wake(_recheck));
+ onStatus(ConnectionStatus.reconnecting);
+ appLog.info(
+ 'ws',
+ 'parked $machineId · the relay says it is off; recheck in ${wait.inSeconds}s',
+ );
+ }
+
+ /// Dial a parked connection now: the machine is reported back, or somebody asks for it. Its own
+ /// recheck makes the next wait longer; anything else starts the waits over.
+ void wake(String reason) {
+ if (!_parked || _closing) return;
+ _parked = false;
+ _parkedRechecks = reason == _recheck ? _parkedRechecks + 1 : 0;
+ _reconnectTimer?.cancel();
+ _reconnectTimer = null;
+ _attempt = 0;
+ unawaited(connect());
+ }
+
/// Wait for this machine's handshake without queuing a request or changing
/// its timeout. Callers can then start independent RPCs with their own budgets.
Future<void> waitUntilReady({required Duration timeout}) {
@@ -338,6 +400,12 @@ class WsConn {
Future<void> connect() async {
if (_closing || _connecting) return;
+ // A dial asked for while parked (a forced redial) is the recheck's too.
+ if (_parked) {
+ _parked = false;
+ _reconnectTimer?.cancel();
+ _reconnectTimer = null;
+ }
_connecting = true;
_ready = false;
onStatus(
@@ -491,6 +559,7 @@ class WsConn {
} catch (_) {
return;
}
+ if (message['type'] == 'machine_select_error') _selectRefused(message);
if (codec != null) {
_inboundTail = _inboundTail
.then((_) => _onRelayFrame(codec, message))
@@ -525,6 +594,7 @@ class WsConn {
void _markReady() {
_observerHandshakeTimer?.cancel();
+ _parkedRechecks = 0;
_ready = true;
_refreshedToken = null;
_stableTimer?.cancel();
@@ -998,6 +1068,17 @@ class WsConn {
}
final code = channel.closeCode;
if (isLocal) {
+ if (channel.closeReason == 'MACHINE_OFFLINE') {
+ _park();
+ return;
+ }
+ if (finalSelectRefusals.contains(channel.closeReason)) {
+ // The relay could not select the machine, for good: told as a 4404 is, never redialled.
+ _closing = true;
+ onLocalFailure?.call(code ?? 1011, channel.closeReason!);
+ onStatus(ConnectionStatus.disconnected);
+ return;
+ }
if (code == 4404 || (observerShareId != null && code == 4403)) {
_closing = true;
// needsLink first: AppNotifier's onStatus handler reads machine.needsLink
@@ -1045,6 +1126,24 @@ class WsConn {
_scheduleReconnect();
}
+ /// The backend refused this socket's select. The app still hears it ([onEvent]); a refusal no
+ /// redial can change also ends the connection, which would otherwise sit open and unselected.
+ void _selectRefused(Map<String, dynamic> message) {
+ final payload = message['payload'];
+ if (payload is! Map || payload['machineId'] != machineId) return;
+ if (_closing || _ready || !finalSelectRefusals.contains(payload['error'])) {
+ return;
+ }
+ _closing = true;
+ _reconnectTimer?.cancel();
+ _reconnectTimer = null;
+ _disposePlugin();
+ onStatus(ConnectionStatus.disconnected);
+ final channel = _channel;
+ _channel = null;
+ unawaited(channel?.sink.close().catchError((Object _) {}));
+ }
+
Future<void> _refreshAndReconnect() async {
onStatus(ConnectionStatus.reconnecting);
try {
diff --git a/desktop/test/machine_sharing_ended_test.dart b/desktop/test/machine_sharing_ended_test.dart
index e80b3a50e..bd186795c 100644
--- a/desktop/test/machine_sharing_ended_test.dart
+++ b/desktop/test/machine_sharing_ended_test.dart
@@ -39,7 +39,7 @@ class _Discovery extends LocalCliDiscovery {
}
class _Connection extends WsConn {
- _Connection(String machineId)
+ _Connection(String machineId, {this.onWake})
: super(
wsBaseUrl: 'ws://fixture.invalid',
autonomousEnv: 'test',
@@ -49,6 +49,9 @@ class _Connection extends WsConn {
onEvent: (_) {},
onStatus: (_) {},
);
+ final void Function(String machineId)? onWake;
+ @override
+ void wake(String reason) => onWake?.call(machineId);
@override
Future<void> waitUntilReady({required Duration timeout}) async {}
@override
@@ -61,13 +64,13 @@ class _Connection extends WsConn {
/// Signed in, with this computer's daemon and its machine connections faked.
class _App extends AppNotifier {
- _App(_Api api)
+ _App(_Api api, {void Function(String machineId)? onWake})
: super(
config: AppConfig.dev,
authSession: AuthSession(),
configStore: null,
localCliDiscovery: _Discovery(),
- connectionForTest: _Connection.new,
+ connectionForTest: (id) => _Connection(id, onWake: onWake),
) {
this.api = api;
}
@@ -188,6 +191,86 @@ void main() {
expect(app.lastErrorRetryable, isTrue);
});
+ // Prod, 2026-10-28: a laptop removed from the account in September was still selected every few
+ // seconds. Refused as not on the account, an own machine is read again from the account's list.
+ for (final (how, refuse) in <(String, Future<void> Function(AppNotifier))>[
+ (
+ 'the backend’s refusal',
+ (app) => app.handleEventForTest('gone', {
+ 'type': 'machine_select_error',
+ 'payload': {'machineId': 'gone', 'error': 'NOT_YOUR_MACHINE'},
+ }),
+ ),
+ (
+ 'the daemon relay’s close',
+ (app) async => app.localFailureForTest('gone', 1011, 'NOT_YOUR_MACHINE'),
+ ),
+ ]) {
+ test('an own machine refused by $how leaves the list', () async {
+ const gone = Machine(
+ machineId: 'gone',
+ authMode: MachineAuthMode.remote,
+ name: 'M1 Laptop',
+ );
+ const kept = Machine(
+ machineId: 'kept',
+ authMode: MachineAuthMode.remote,
+ name: 'Studio',
+ );
+ final api = _Api()..listed = const [kept];
+ app = _App(api)..status = AppStatus.authenticated;
+ app.machines = [gone, kept];
+ for (final machine in app.machines) {
+ app.machineStates[machine.machineId] = MachineState(machine)
+ ..nodeOnline = true
+ ..connectionStatus = ConnectionStatus.connected;
+ }
+ await refuse(app);
+ for (var i = 0; i < 20; i++) {
+ await Future<void>.delayed(Duration.zero);
+ }
+ final listed = app.machines.map((m) => m.machineId);
+ expect(listed, contains('kept'));
+ expect(listed, isNot(contains('gone')));
+ expect(app.stateOf('gone'), isNull);
+ expect(app.lastErrorRetryable, isFalse);
+ });
+ }
+
+ // A relayed machine that was off waits parked (WsConn.isParked) until it is reported back: the
+ // machine list's refresh saying it is online is that word.
+ test('a machine the list reports online wakes its parked connection', () async {
+ const studio = Machine(
+ machineId: 'studio',
+ authMode: MachineAuthMode.remote,
+ name: 'Studio',
+ status: 'online',
+ );
+ final api = _Api()..listed = const [studio];
+ final woken = <String>[];
+ app = _App(api, onWake: woken.add)..status = AppStatus.authenticated;
+ await app.refreshMachines();
+ expect(woken, contains('studio'));
+ });
+
+ // The list's word alone, for a row that already reads online: no node-status change runs, so only the list's own
+ // wake can reach the parked connection (review of #2027).
+ test('a row already online that the list reports online still wakes its parked connection', () async {
+ const studio = Machine(
+ machineId: 'studio',
+ authMode: MachineAuthMode.remote,
+ name: 'Studio',
+ status: 'online',
+ );
+ final api = _Api()..listed = const [studio];
+ final woken = <String>[];
+ app = _App(api, onWake: woken.add)..status = AppStatus.authenticated;
+ app.machines = [studio];
+ app.machineStates['studio'] = MachineState(studio)..nodeOnline = true;
+ await app.refreshMachines();
+ expect(woken, contains('studio'));
+ });
+
test('every refusal code reads as a sentence', () {
for (final code in [
'NOT_YOUR_MACHINE',
diff --git a/desktop/test/ws_conn_test.dart b/desktop/test/ws_conn_test.dart
index 1fd282a22..cb06e421d 100644
--- a/desktop/test/ws_conn_test.dart
+++ b/desktop/test/ws_conn_test.dart
@@ -17,6 +17,12 @@ class FakeHub {
final HttpServer server;
final bool rejectOldToken;
final int? closeCodeOnSelect;
+
+ /// The reason [closeCodeOnSelect] closes with: what the daemon's relay says, as it says it.
+ String closeReasonOnSelect = 'rejected';
+
+ /// Answers every select with this `machine_select_error` code instead, as the backend does.
+ String? selectErrorOnSelect;
final List<Map<String, dynamic>> frames = [];
final List<String> protocols = [];
final List<String?> environments = [];
@@ -56,7 +62,19 @@ class FakeHub {
if (frame['type'] == 'machine_select') {
final machineId = (frame['payload'] as Map)['machineId'] as String;
if (hub.closeCodeOnSelect != null) {
- ws.close(hub.closeCodeOnSelect!, 'rejected');
+ ws.close(hub.closeCodeOnSelect!, hub.closeReasonOnSelect);
+ return;
+ }
+ if (hub.selectErrorOnSelect != null) {
+ ws.add(
+ jsonEncode({
+ 'type': 'machine_select_error',
+ 'payload': {
+ 'machineId': machineId,
+ 'error': hub.selectErrorOnSelect,
+ },
+ }),
+ );
return;
}
if (hub.rejectOldToken && protocol == 'tok-old') {
@@ -509,6 +527,87 @@ void main() {
expect(conn!.isClosed, isTrue);
});
+ // Prod, 2026-10-28: a laptop removed from the account in September was still selected every few
+ // seconds. Through this computer's daemon the refusal arrives as the relay's close, with the
+ // backend's code for its reason; straight from the backend (the web viewer), as the frame.
+ for (final (path, local) in [('daemon relay', true), ('backend', false)]) {
+ test(
+ 'a machine refused as not on the account ($path) is not dialled again',
+ () async {
+ hub = await FakeHub.start(closeCodeOnSelect: local ? 1011 : null);
+ hub.closeReasonOnSelect = 'NOT_YOUR_MACHINE';
+ if (!local) hub.selectErrorOnSelect = 'NOT_YOUR_MACHINE';
+ final failures = <String>[];
+ final statuses = <ConnectionStatus>[];
+ conn = WsConn(
+ wsBaseUrl: 'ws://127.0.0.1:${hub.port}',
+ autonomousEnv: 'prod',
+ machineId: 'm1',
+ accessTokenProvider: (_, _) async => 'sso-token',
+ onAuthFailure: (_) {},
+ onLocalFailure: (code, reason) => failures.add(reason),
+ onEvent: (_) {},
+ onStatus: statuses.add,
+ transportKind: local
+ ? WsTransportKind.localPlaintext
+ : WsTransportKind.cloudE2ee,
+ allowLocalTcpFixture: true,
+ localWsUri: local
+ ? Uri.parse('ws://127.0.0.1:${hub.port}/api/local-ws')
+ : null,
+ );
+ await conn!.connect();
+ // Past the first two steps of the backoff (0.5–1 s, then 1–2 s).
+ await Future<void>.delayed(const Duration(milliseconds: 3200));
+ expect(hub.protocols, hasLength(1));
+ expect(conn!.isClosed, isTrue);
+ expect(statuses.last, ConnectionStatus.disconnected);
+ if (local) expect(failures, ['NOT_YOUR_MACHINE']);
+ },
+ );
+ }
+
+ // Prod, 2026-10-28: relays living 0.4 s made 2,304 of 2,308 backend socket opens in 15 minutes.
+ // A relayed machine that is off was redialled on the 15–30 s backoff for as long as the app ran,
+ // each dial refused at once: about 2.7 a minute for every machine that was off. It now waits for
+ // word that the machine is back, and rechecks on its own only every 1, 2, 4, 8, then 15 minutes.
+ test('a relayed machine that is off waits to be woken instead of redialling', () async {
+ // The reason is what counts: a current daemon closes with 1013, an older one with 1011 (a
+ // dart:io server cannot send 1013, a reserved code).
+ hub = await FakeHub.start(closeCodeOnSelect: 1011);
+ hub.closeReasonOnSelect = 'MACHINE_OFFLINE';
+ conn = WsConn(
+ wsBaseUrl: 'wss://unused.example',
+ autonomousEnv: 'prod',
+ machineId: 'm1',
+ accessTokenProvider: (_, _) async => 'sso-token',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: (_) {},
+ transportKind: WsTransportKind.localPlaintext,
+ allowLocalTcpFixture: true,
+ localWsUri: Uri.parse('ws://127.0.0.1:${hub.port}/api/local-ws'),
+ );
+ await conn!.connect();
+ await Future<void>.delayed(const Duration(milliseconds: 3200));
+ expect(hub.protocols, hasLength(1));
+ expect(conn!.isParked, isTrue);
+ expect(conn!.isClosed, isFalse);
+
+ conn!.wake('machine online');
+ await Future<void>.delayed(const Duration(milliseconds: 250));
+ expect(hub.protocols, hasLength(2));
+ });
+
+ // The first recheck is the relay's own retry-after (30 s): a machine back online that a wake reached while the relay
+ // still answered from its refusal parks again, and must not wait a minute more (review of #2027).
+ test('a parked connection rechecks after 30 s, then 1, 2, 4, 8 and every 15 minutes', () {
+ expect(
+ [for (var n = 0; n < 7; n++) WsConn.parkedRecheckAfter(n).inSeconds],
+ [30, 60, 120, 240, 480, 900, 900],
+ );
+ });
+
test(
'a fixed reconnect delay retries at that pace instead of backing off',
() async {
diff --git a/mobile/lib/state/app_state.dart b/mobile/lib/state/app_state.dart
index 6fda1f981..010212c71 100644
--- a/mobile/lib/state/app_state.dart
+++ b/mobile/lib/state/app_state.dart
@@ -10334,9 +10334,20 @@ class AppNotifier extends ChangeNotifier {
_wakeMachinesBackOnline(payload['statuses']);
return;
case 'machine_select_error':
- _lastError =
- 'Machine selection failed: ${payload['error'] ?? 'unknown error'}';
- _lastErrorRetryable = true;
+ if (payload['error'] == 'NOT_YOUR_MACHINE') {
+ // Removed from the account, or no longer shared with it: no redial brings it back, and
+ // its socket has stopped (WsConn._selectRefused). The account's list is read again so the
+ // machine leaves the list, its tabs and the cache it came from. Prod, 2026-10-28: a laptop
+ // removed in September was still selected every ~7 s.
+ _lastError =
+ '${machine.machine.displayName} is no longer on your account.';
+ _lastErrorRetryable = false;
+ unawaited(refreshMachines());
+ } else {
+ _lastError =
+ 'Machine selection failed: ${payload['error'] ?? 'unknown error'}';
+ _lastErrorRetryable = true;
+ }
machine.connectionStatus = ConnectionStatus.disconnected;
break;
case 'agent_synced':
diff --git a/mobile/lib/ws/ws_conn.dart b/mobile/lib/ws/ws_conn.dart
index d93f1c5f3..5f00c5777 100644
--- a/mobile/lib/ws/ws_conn.dart
+++ b/mobile/lib/ws/ws_conn.dart
@@ -506,6 +506,46 @@ class WsConn {
unawaited(channel.sink.close().catchError((Object _) {}));
}
+ /// Refusals no redial can change: the machine left the account or was never shared with it, it
+ /// belongs to another environment, or its subscription stops it. Every code the relay's
+ /// `machine_select` answers with besides these may pass (`backend/src/lib/webWs.ts`).
+ static const _finalSelectRefusals = {
+ 'NOT_YOUR_MACHINE',
+ 'MACHINE_ENV_MISMATCH',
+ 'MACHINE_PAYMENT_PENDING',
+ 'MACHINE_SUBSCRIPTION_REQUIRED',
+ };
+
+ /// The relay refused this connection's select. The app still hears the refusal ([onEvent]); the
+ /// connection stops here for good when no redial can change it, or else waits out its backoff.
+ ///
+ /// ⚠️ **Why (prod, 2026-10-28).** Nothing here read the refusal, so [_armSelectWatchdog] took the
+ /// silence after it for a relay gone quiet and redialled six seconds later, for as long as the app
+ /// was open: a laptop removed from the account in September was still selected every ~7 s, about
+ /// 20 dials in ten minutes from one phone, each a fresh credential, key and TLS dial.
+ void _selectRefused(Map<String, dynamic> message) {
+ final payload = message['payload'];
+ if (payload is! Map || payload['machineId'] != machineId) return;
+ if (_closing || _ready) return;
+ _selectWatchdog?.cancel();
+ _selectWatchdog = null;
+ final code = '${payload['error'] ?? ''}';
+ appLog.info('ws', 'machine_select refused for $machineId: $code');
+ if (!_finalSelectRefusals.contains(code)) {
+ _abandonDial(_channel);
+ _scheduleReconnect();
+ return;
+ }
+ _closing = true;
+ _reconnectTimer?.cancel();
+ _reconnectTimer = null;
+ _disposePlugin();
+ onStatus(ConnectionStatus.disconnected);
+ final channel = _channel;
+ _channel = null;
+ unawaited(channel?.sink.close().catchError((Object _) {}));
+ }
+
void _signOut(String message) {
_closing = true;
onStatus(ConnectionStatus.disconnected);
@@ -623,6 +663,7 @@ class WsConn {
} catch (_) {
return;
}
+ if (message['type'] == 'machine_select_error') _selectRefused(message);
if (codec != null) {
_inboundTail = _inboundTail
.then((_) {
diff --git a/mobile/test/app_state_machine_events_test.dart b/mobile/test/app_state_machine_events_test.dart
index 01897d43e..47d171747 100644
--- a/mobile/test/app_state_machine_events_test.dart
+++ b/mobile/test/app_state_machine_events_test.dart
@@ -705,6 +705,29 @@ void main() {
);
});
+ test('a machine refused as not on the account is read again and leaves the list', () async {
+ // Prod, 2026-10-28: a laptop removed in September was still selected every ~7 s.
+ final rig = await signedInWith({
+ 'm': ['a'],
+ 'gone': ['b'],
+ });
+ addTearDown(rig.app.dispose);
+ final fetches = rig.api.machineFetches;
+ rig.api.onMachines = () async => [remoteMachine('m')];
+
+ await push(rig, 'gone', 'machine_select_error', {
+ 'machineId': 'gone',
+ 'error': 'NOT_YOUR_MACHINE',
+ });
+ await settle();
+
+ expect(rig.api.machineFetches, fetches + 1);
+ expect(rig.app.stateOf('gone'), isNull);
+ expect(rig.app.machines.map((m) => m.machineId), ['m']);
+ expect(rig.app.lastError, 'Machine gone is no longer on your account.');
+ expect(rig.app.lastErrorRetryable, isFalse);
+ });
+
test('pushes for a machine the phone does not have are ignored', () async {
final rig = await signedInWith({
'm': ['a'],
diff --git a/mobile/test/ws/ws_conn_refused_test.dart b/mobile/test/ws/ws_conn_refused_test.dart
new file mode 100644
index 000000000..1cb494aa7
--- /dev/null
+++ b/mobile/test/ws/ws_conn_refused_test.dart
@@ -0,0 +1,91 @@
+import 'dart:async';
+import 'dart:convert';
+
+import 'package:fake_async/fake_async.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+import 'package:harness_mobile/core/models.dart' show ConnectionStatus;
+import 'package:harness_mobile/ws/ws_conn.dart';
+
+import 'memory_web_socket.dart';
+
+/// A machine the relay refuses — removed from the account (prod, 2026-10-28: a laptop deleted in
+/// September, still selected every ~7 s), no longer shared, another environment, unpaid — is not
+/// dialled again. The select watchdog read the silence after the refusal as a relay gone quiet and
+/// redialled it after six seconds, for as long as the app was open.
+void main() {
+ void refused(
+ String error,
+ void Function(
+ FakeAsync clock,
+ WsConn conn,
+ List<MemoryWebSocket> relay,
+ List<ConnectionStatus> statuses,
+ )
+ body,
+ ) {
+ fakeAsync((clock) {
+ final relay = <MemoryWebSocket>[];
+ final statuses = <ConnectionStatus>[];
+ final conn = WsConn(
+ wsBaseUrl: 'ws://fixture.invalid',
+ connectChannel: (_, {protocols}) {
+ final (client, server) = MemoryWebSocket.pair();
+ server.stream.listen((raw) {
+ final frame = jsonDecode(raw as String) as Map<String, dynamic>;
+ if (frame['type'] != 'machine_select') return;
+ server.sink.add(
+ jsonEncode({
+ 'type': 'machine_select_error',
+ 'payload': {'machineId': 'm', 'error': error},
+ }),
+ );
+ });
+ relay.add(server);
+ client.accept();
+ return client;
+ },
+ autonomousEnv: 'test',
+ machineId: 'm',
+ accessTokenProvider: (_, _) async => 'token',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: statuses.add,
+ );
+ unawaited(conn.connect());
+ clock.flushMicrotasks();
+ body(clock, conn, relay, statuses);
+ unawaited(conn.close());
+ clock.flushMicrotasks();
+ });
+ }
+
+ for (final error in [
+ 'NOT_YOUR_MACHINE',
+ 'MACHINE_ENV_MISMATCH',
+ 'MACHINE_PAYMENT_PENDING',
+ 'MACHINE_SUBSCRIPTION_REQUIRED',
+ ]) {
+ test('$error is not redialled', () {
+ refused(error, (clock, conn, relay, statuses) {
+ clock.elapse(const Duration(minutes: 10));
+ expect(relay, hasLength(1));
+ expect(conn.isClosed, isTrue);
+ expect(statuses.last, ConnectionStatus.disconnected);
+ });
+ });
+ }
+
+ test('a refusal it may outlive (a machine that failed to start) waits out the backoff', () {
+ refused('MACHINE_START_FAILED', (clock, conn, relay, statuses) {
+ // Not the six-second watchdog: the ordinary 1, 2, 4 … 30 s backoff, from the refusal on.
+ clock.elapse(const Duration(milliseconds: 999));
+ expect(relay, hasLength(1));
+ clock.elapse(const Duration(milliseconds: 1));
+ expect(relay, hasLength(2));
+ clock.elapse(const Duration(seconds: 2));
+ expect(relay, hasLength(3));
+ expect(conn.isClosed, isFalse);
+ });
+ });
+}