Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(mobile): a machine whose relay session died is dialled again

Dev jacob6df5ab6e4ab7
diff --git a/mobile/lib/phone/agent_model_sections.dart b/mobile/lib/phone/agent_model_sections.dart
index ad9a1a01b..1be170017 100644
--- a/mobile/lib/phone/agent_model_sections.dart
+++ b/mobile/lib/phone/agent_model_sections.dart
@@ -23,9 +23,19 @@ import 'package:harness_mobile/core/models.dart';
/// not be one costs an agent answering on a model nobody asked for.
const Set<String> kModelSheetEngines = {'claude', 'codex', 'opencode'};
+/// Whether the model sheet is offered at all, on any engine.
+///
+/// Off by decision (owner, 2026-09-22): switching works end to end, the feature
+/// is simply not being shown yet. This is the ONE gate — flipping it to true is
+/// the whole of putting the row back, and everything behind it (the sheet, its
+/// sections, its tests) is kept working in the meantime rather than deleted and
+/// rewritten later.
+const bool kModelSheetEnabled = false;
+
/// Whether [engine] gets the sheet at all. Unknown or absent is NO — a menu
/// offers nothing it cannot back.
bool modelSheetSupports(String? engine) =>
+ kModelSheetEnabled &&
kModelSheetEngines.contains(engine?.trim().toLowerCase());
/// The sections worth a heading: the own grid always (so its empty-state
diff --git a/mobile/lib/state/app_state.dart b/mobile/lib/state/app_state.dart
index 9c722f282..b72a6296a 100644
--- a/mobile/lib/state/app_state.dart
+++ b/mobile/lib/state/app_state.dart
@@ -32,6 +32,7 @@ import '../core/models.dart';
import '../core/project_folder.dart';
import '../core/project_history.dart';
import '../core/retry.dart';
+import '../logging/app_log.dart';
import '../logging/startup_trace.dart';
import '../settings/config_store.dart';
import '../stats/harness_stats.dart';
@@ -2940,6 +2941,15 @@ class AppNotifier extends ChangeNotifier {
notifyListeners();
}
+ /// One tick of the agent-list safety net ([_syncAgentsIfChanged]), which the
+ /// app itself only reaches through a 60-second timer.
+ @visibleForTesting
+ Future<void> syncAgentsForTest(String machineId) async {
+ final machine = machineStates[machineId];
+ if (machine == null) return;
+ await _syncAgentsIfChanged(machine);
+ }
+
@visibleForTesting
void connectionStatusForTest(String machineId, ConnectionStatus status) =>
_onConnectionStatus(machineId, status);
@@ -3143,6 +3153,7 @@ class AppNotifier extends ChangeNotifier {
void _stopAgentSyncTimer(String machineId) {
_agentSyncTimers.remove(machineId)?.cancel();
+ _agentSyncTimeouts.remove(machineId);
}
void _stopAllAgentSyncTimers() {
@@ -3150,34 +3161,77 @@ class AppNotifier extends ChangeNotifier {
timer.cancel();
}
_agentSyncTimers.clear();
+ _agentSyncTimeouts.clear();
}
+ /// How many [agentSyncInterval] ticks in a row may time out before the machine
+ /// is treated as gone and redialled ([_recoverStaleSession]).
+ ///
+ /// Two, not one: a single missed tick is a busy machine or a slow relay, and a
+ /// forced redial costs every terminal on it a resync. Two in a row is a minute
+ /// of a machine not answering the cheapest request there is, which nothing
+ /// healthy does.
+ @visibleForTesting
+ static const agentSyncStaleTicks = 2;
+
+ /// Consecutive timed-out [_syncAgentsIfChanged] ticks, per machine. Cleared by
+ /// any answer, by any other failure, and by the timer stopping.
+ final Map<String, int> _agentSyncTimeouts = {};
+
/// Silent safety-net reconciliation, ticked every [agentSyncInterval] while a machine is connected.
/// Only writes/notifies if the fetched list actually differs from what's already shown — a steady
/// state where push events (agent_synced et al.) have kept everything in sync produces zero visible
/// effect. Deliberately does not touch agentLoadStatus/agentsLoadError/notifyListeners on failure:
- /// a real connectivity problem is already surfaced by the push path and the existing offline
- /// detection in _performMachineDataLoad, and a quiet background tick should not fight either.
+ /// a real connectivity problem is already surfaced by the push path, and a quiet background tick
+ /// should not fight it.
+ ///
+ /// ⚠️ **Except a timeout, which this is the only thing left watching for.** A
+ /// stale relay session (see [_recoverStaleSession]) leaves the transport
+ /// "connected", the machine `nodeOnline`, its list `loaded` — so no screen is
+ /// wrong, no push arrives to correct it, and nothing calls
+ /// [_performMachineDataLoad], which is where the offline detection this used
+ /// to defer to actually lives. Swallowed here, that state was permanent:
+ /// measured at 16 minutes and still going, a phone showing a machine's agent
+ /// list from before its Harness restarted, with every agent made since
+ /// invisible — and, with the list feeding `deskGroups`, a desk tab whose
+ /// harnesses had all silently vanished off the phone.
Future<void> _syncAgentsIfChanged(MachineState machine) async {
if (machine.connectionStatus != ConnectionStatus.connected) return;
if (machine.agentsLoadInFlight != null) {
return; // a real (foreground) load already owns this tick
}
- final connection = _conn(machine.machine.machineId);
+ final machineId = machine.machine.machineId;
+ final revision = _authRevision;
+ final connection = _conn(machineId);
try {
final response = await connection.request(
'agents_list',
payload: kAgentsListPayload,
timeout: const Duration(seconds: 10),
);
+ _agentSyncTimeouts.remove(machineId);
+ if (!_machineWorkCurrent(machine, revision)) return;
final agents = (response['agents'] as List<dynamic>? ?? [])
.map((item) => Agent.fromJson(item as Map<String, dynamic>))
.toList();
if (agentsEqual(machine.agents, agents)) return;
_replaceAgents(machine, agents);
notifyListeners();
+ } on WsRequestTimeout {
+ if (!_machineWorkCurrent(machine, revision)) return;
+ final missed = (_agentSyncTimeouts[machineId] ?? 0) + 1;
+ _agentSyncTimeouts[machineId] = missed;
+ if (missed < agentSyncStaleTicks) return;
+ _agentSyncTimeouts.remove(machineId);
+ appLog.warn(
+ 'ws',
+ 'agents_list timed out $missed× on $machineId — redialling',
+ );
+ _recoverStaleSession(machine, connection);
} catch (_) {
- // Silent by design — see doc comment above.
+ // Silent by design — see doc comment above. A socket that dropped
+ // mid-request is already redialling, so the count starts over.
+ _agentSyncTimeouts.remove(machineId);
}
}
@@ -3926,25 +3980,11 @@ class AppNotifier extends ChangeNotifier {
} catch (error) {
if (!_machineWorkCurrent(machine, revision)) return;
machine.agentsRefreshing = false;
- // A request timing out while the local relay session still nominally reports "connected" means
- // the remote node itself has stopped answering — exactly what a REST-status flip to offline
- // means elsewhere, so route it through _applyNodeStatus (not just `nodeOnline = false`) so the
- // pending agent gets captured for auto-reattach, same as any other offline detection path.
if (error is WsRequestTimeout) {
machine.agentsLoadError = machine.isLocalMachine
? 'Harness is offline — run harness login'
: 'Harness is offline — run harness start on that machine';
- if (machine.nodeOnline != false) {
- unawaited(_applyNodeStatus(machine, false));
- }
- if (!machine.isLocalMachine) {
- // The relay's cached upstream session can go stale at the E2EE-session layer without the
- // underlying transport ever closing — most commonly the relayed machine's own Harness
- // process restarting, which drops its in-memory session state but doesn't touch the socket.
- // Nothing else would ever notice, so force a fresh dial rather than let every future retry
- // keep timing out against the same dead session.
- unawaited(connection.forceReconnect());
- }
+ _recoverStaleSession(machine, connection);
} else if (connection.isClosed || machine.nodeOnline == false) {
machine.agentsLoadError = 'Could not load harnesses: $error';
} else {
@@ -3982,6 +4022,30 @@ class AppNotifier extends ChangeNotifier {
notifyListeners();
}
+ /// An RPC timed out on a machine whose transport still calls itself connected:
+ /// take the machine down and dial it again.
+ ///
+ /// A request timing out while the local relay session still nominally reports
+ /// "connected" means the node itself has stopped answering — exactly what a
+ /// REST-status flip to offline means elsewhere, so it is routed through
+ /// [_applyNodeStatus] (not just `nodeOnline = false`) so the pending agent gets
+ /// captured for auto-reattach, same as any other offline detection path.
+ ///
+ /// ⚠️ **And then a fresh dial, for a remote machine.** The relay's cached
+ /// upstream session can go stale at the E2EE-session layer without the
+ /// underlying transport ever closing — most commonly the relayed machine's own
+ /// Harness process restarting, which drops its in-memory session state but
+ /// doesn't touch the socket. No close event ever fires, so nothing else would
+ /// ever notice; without this every future request keeps timing out against the
+ /// same dead session, for as long as the app runs.
+ void _recoverStaleSession(MachineState machine, WsConn connection) {
+ if (machine.nodeOnline != false) {
+ unawaited(_applyNodeStatus(machine, false));
+ }
+ if (machine.isLocalMachine) return;
+ unawaited(connection.forceReconnect());
+ }
+
bool _machineWorkCurrent(MachineState machine, int revision) =>
_authWorkCurrent(revision) &&
identical(machineStates[machine.machine.machineId], machine);
@@ -5267,13 +5331,24 @@ class AppNotifier extends ChangeNotifier {
final pending = machine.pendingOfflineAgentId;
if (pending != null) {
unawaited(_recoverPendingAgent(machine, pending));
- } else if (panesFor(machineId).any(_paneNeedsAttach) &&
- _pool?[machineId]?.isReady == true) {
- // A tile restored from the saved layout has no pendingOfflineAgentId —
- // nothing of its was interrupted, it simply arrived before its machine
- // did. Without this it would sit on "Attaching…" forever on a machine
- // that has since come back, because every other route to _attachSession
- // runs off a load that nothing here would trigger.
+ } else if (_pool?[machineId]?.isReady == true &&
+ (wasOnline == false || panesFor(machineId).any(_paneNeedsAttach))) {
+ // ⚠️ **A machine that was OFF and is back owes a fresh list, whether or
+ // not anything here was waiting on it.** This arrives as a `node_status`
+ // push over a socket that never closed, so `_onConnectionStatus` — which
+ // is what reloads after every real reconnect — does not run, and the
+ // list this app holds is from before that machine's Harness restarted.
+ // Nothing corrects it: the pushes that would have go to a session the
+ // restart dropped. A phone therefore kept showing a machine's agents
+ // from minutes earlier, every agent made since invisible, and a desk tab
+ // holding those agents (`deskGroups`) read as an empty tab.
+ //
+ // A tile restored from the saved layout is the other half, and has no
+ // pendingOfflineAgentId — nothing of its was interrupted, it simply
+ // arrived before its machine did. Without this it would sit on
+ // "Attaching…" forever on a machine that has since come back, because
+ // every other route to _attachSession runs off a load that nothing here
+ // would trigger.
//
// ⚠️ **Only once the socket is actually up.** `/api/machines` reports a
// machine as running well before this app has finished dialling it, and
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index 7af9fb4f9..ec52d45a9 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -2,7 +2,7 @@ name: harness_mobile
description: "Harness viewer for iOS and Android."
publish_to: 'none'
-version: 1.0.0+30
+version: 1.0.0+31
environment:
sdk: ^3.13.0
diff --git a/mobile/test/phone_model_sheet_test.dart b/mobile/test/phone_model_sheet_test.dart
index 8924f0553..8ceb7f8f3 100644
--- a/mobile/test/phone_model_sheet_test.dart
+++ b/mobile/test/phone_model_sheet_test.dart
@@ -282,9 +282,21 @@ void main() {
);
});
+ test('the sheet is not offered while it is gated off', () {
+ // The row is hidden for now — see [kModelSheetEnabled]. Asserted rather
+ // than left implied so flipping the gate back on fails here first, in the
+ // one place that says what the rule underneath it is.
+ expect(kModelSheetEnabled, isFalse);
+ expect(modelSheetSupports('claude'), isFalse);
+ expect(modelSheetSupports('CODEX'), isFalse);
+ });
+
test('only the engines whose switching was driven get the sheet', () {
- expect(modelSheetSupports('claude'), isTrue);
- expect(modelSheetSupports('CODEX'), isTrue);
+ // The gate above sits in front of this rule; it is the rule the row goes
+ // back to, so it is kept honest while the row is away.
+ expect(kModelSheetEngines.contains('claude'), isTrue);
+ expect(kModelSheetEngines.contains('codex'), isTrue);
+ expect(kModelSheetEngines.contains('cursor'), isFalse);
expect(modelSheetSupports('cursor'), isFalse);
expect(modelSheetSupports(null), isFalse);
});
diff --git a/mobile/test/phone_stale_session_test.dart b/mobile/test/phone_stale_session_test.dart
new file mode 100644
index 000000000..c810c615a
--- /dev/null
+++ b/mobile/test/phone_stale_session_test.dart
@@ -0,0 +1,100 @@
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness_mobile/core/models.dart';
+import 'package:harness_mobile/state/app_state.dart';
+import 'package:harness_mobile/ws/ws_conn.dart';
+
+import 'agent_pager_fixture.dart';
+
+/// A machine whose relay session has gone stale: the transport still calls
+/// itself connected, and every request into it runs out its clock.
+///
+/// This is the shape of the real failure — a relayed machine's own Harness
+/// restarting drops its in-memory session without ever closing the socket, so
+/// nothing fires `onDone` and no screen goes wrong. Measured on a phone at 16
+/// minutes of a frozen agent list, with the desk tab holding those agents
+/// drawn as an empty tab.
+class _StaleConn extends PagerConn {
+ bool answering = false;
+ int asked = 0;
+ int redials = 0;
+
+ @override
+ Future<Map<String, dynamic>> request(
+ String type, {
+ Map<String, dynamic> payload = const {},
+ Duration timeout = const Duration(seconds: 20),
+ }) async {
+ if (type != 'agents_list') return {};
+ asked++;
+ if (answering) return {'agents': const []};
+ throw const WsRequestTimeout('agents_list');
+ }
+
+ @override
+ Future<void> forceReconnect() async => redials++;
+}
+
+void main() {
+ late _StaleConn conn;
+ late AppNotifier app;
+
+ setUp(() {
+ conn = _StaleConn();
+ app = pagerApp(conn, viewer: true);
+ });
+
+ tearDown(() => app.dispose());
+
+ test('one timed-out tick is not enough to redial', () async {
+ await app.syncAgentsForTest('m');
+
+ expect(conn.asked, 1);
+ expect(conn.redials, 0);
+ // Untouched: a single missed tick is a busy machine, and every screen
+ // should go on showing what it was showing.
+ expect(app.stateOf('m')!.nodeOnline, isTrue);
+ expect(app.stateOf('m')!.agents, hasLength(pagerAgentIds.length));
+ });
+
+ test('two in a row take the machine down and dial it again', () async {
+ for (var tick = 0; tick < AppNotifier.agentSyncStaleTicks; tick++) {
+ await app.syncAgentsForTest('m');
+ }
+
+ expect(conn.redials, 1);
+ expect(app.stateOf('m')!.nodeOnline, isFalse);
+ });
+
+ test('an answer in between starts the count over', () async {
+ await app.syncAgentsForTest('m');
+ conn.answering = true;
+ await app.syncAgentsForTest('m');
+ conn.answering = false;
+ await app.syncAgentsForTest('m');
+
+ // The machine answered, so the two timeouts either side of it are not two
+ // in a row and nothing is torn down.
+ expect(conn.redials, 0);
+ expect(app.stateOf('m')!.nodeOnline, isTrue);
+ });
+
+ test('a machine that keeps timing out is redialled again, not once', () async {
+ for (var tick = 0; tick < AppNotifier.agentSyncStaleTicks * 2; tick++) {
+ await app.syncAgentsForTest('m');
+ }
+
+ // The count resets after each redial, so a machine that is still dead a
+ // minute later is dialled again rather than left on one attempt forever.
+ expect(conn.redials, 2);
+ });
+
+ test('a disconnected machine is not polled at all', () async {
+ app.stateOf('m')!.connectionStatus = ConnectionStatus.reconnecting;
+
+ await app.syncAgentsForTest('m');
+
+ // Its own redial is already in flight; a tick here would only race it.
+ expect(conn.asked, 0);
+ expect(conn.redials, 0);
+ });
+}