Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix(terminal): remote panes recover by themselves after their stream dies (#1378)

0xken15b364aede940e
diff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.ts
index 5af84afd5..739fd8e16 100644
--- a/cli/src/lib/terminalStreamManager.spec.ts
+++ b/cli/src/lib/terminalStreamManager.spec.ts
@@ -770,6 +770,41 @@ describe('TerminalStreamManager', () => {
&& frame.payload.reason === 'heartbeat timeout')).toBe(true)
})
+ // A host that lost its backend link closes its remote streams without telling anyone; the client
+ // only found out after 15 s of silence and three ignored resyncs, then gave up (2026-10-16). A
+ // client that asks is now told at its next heartbeat.
+ describe('a heartbeat or resync for a stream this connection does not hold', () => {
+ const errors = () => sent.filter((frame) => frame.type === 'terminal_error')
+
+ it('is answered with TERMINAL_STREAM_NOT_FOUND when the client asks', async () => {
+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1', reportMissing: true })
+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-2', attempt: 1, reportMissing: true })
+ expect(errors().map((frame) => [frame.connId, frame.payload.code, frame.payload.streamId])).toEqual([
+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-1'],
+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-2'],
+ ])
+ })
+
+ it('stays silent for a client that does not ask, as every older client', async () => {
+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1' })
+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-1', attempt: 1 })
+ expect(errors()).toEqual([])
+ })
+
+ it('counts a stream held by another connection as missing for this one, and leaves it alone', async () => {
+ await manager.handleFrame('web-1', 'terminal_open', {
+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,
+ })
+ const streamId = sent.findLast((frame) => frame.type === 'terminal_ready')?.payload.streamId as string
+ await manager.handleFrame('web-2', 'terminal_alive', { streamId, reportMissing: true })
+ expect(errors().at(-1)).toMatchObject({
+ connId: 'web-2',
+ payload: { code: 'TERMINAL_STREAM_NOT_FOUND', streamId },
+ })
+ expect(stream.closed).toBe(false)
+ })
+ })
+
// Node's clock keeps running while a Mac sleeps, so a lid closed for forty minutes reached the next
// sweep as forty minutes without `terminal_alive` and closed every stream at the wake — measured
// 2026-09-28 as `[terminal-stream] closed` on every local terminal the moment the lid opened.
diff --git a/cli/src/lib/terminalStreamManager.ts b/cli/src/lib/terminalStreamManager.ts
index 1686a99e9..924fd8091 100644
--- a/cli/src/lib/terminalStreamManager.ts
+++ b/cli/src/lib/terminalStreamManager.ts
@@ -639,10 +639,21 @@ export class TerminalStreamManager {
private alive(connId: string, payload: FramePayload): void {
const state = this.streamFor(connId, payload)
- if (!state) return
+ if (!state) { this.reportMissing(connId, payload); return }
state.expiresAt = this.now() + HEARTBEAT_TIMEOUT_MS
}
+ /**
+ * Tell a client that asked (`reportMissing`) that the stream it names is gone. A host that lost its
+ * backend link closes its remote streams without notice, and the client used to learn it only after
+ * 15 s of silence and three resyncs this manager ignored, then gave up on the pane (2026-10-16). Opt-in,
+ * so an older client keeps the silence its own recovery was built around.
+ */
+ private reportMissing(connId: string, payload: FramePayload): void {
+ if (payload.reportMissing !== true || typeof payload.streamId !== 'string') return
+ this.sendError(connId, 'TERMINAL_STREAM_NOT_FOUND', { streamId: payload.streamId, message: 'no live terminal stream' })
+ }
+
private ack(connId: string, payload: FramePayload): void {
const state = this.streamFor(connId, payload)
if (!state) return
@@ -1010,11 +1021,10 @@ export class TerminalStreamManager {
private async resyncRequested(connId: string, payload: FramePayload): Promise<void> {
const state = this.streamFor(connId, payload)
- if (state) {
- state.resyncs++
- this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })
- await this.sendKeyframe(state, 'resync')
- }
+ if (!state) { this.reportMissing(connId, payload); return }
+ state.resyncs++
+ this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })
+ await this.sendKeyframe(state, 'resync')
}
private async closeRequested(connId: string, payload: FramePayload): Promise<void> {
diff --git a/cli/src/messages/sharedTabs.integration.spec.ts b/cli/src/messages/sharedTabs.integration.spec.ts
index dd9702713..6db96aca6 100644
--- a/cli/src/messages/sharedTabs.integration.spec.ts
+++ b/cli/src/messages/sharedTabs.integration.spec.ts
@@ -279,7 +279,8 @@ describe('shared tabs across three accounts, every piece real but the backend',
})
expect(relay.unauthorized).toEqual([carol.id])
expect(carol.asks).toContainEqual({ force: true, failedToken: 'stale-carol' })
- expect(carol.heard).toContainEqual({ sharedTabId: A, ready: true })
+ // The ready notice lands a turn after the socket is admitted; asserting it at once failed CI under load (#1378).
+ await until(() => expect(carol.heard).toContainEqual({ sharedTabId: A, ready: true }))
})
it('3. Bob adds bob/api and Carol adds carol/web: both appear in every member\'s roster', async () => {
diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dart
index dfb0f7a9c..626d6b3f7 100644
--- a/desktop/lib/state/app_state.dart
+++ b/desktop/lib/state/app_state.dart
@@ -979,6 +979,8 @@ class AppNotifier extends ChangeNotifier {
);
final Map<String, Timer> _offlineRetryTimers = {};
+ final Map<TerminalPane, Timer> _paneRetryTimers = {};
+ final Map<TerminalPane, int> _paneRetryAttempts = {};
/// One reload per machine, owed after this computer's daemon was too slow to answer its list while
/// still on the socket — the error strip says "retrying", and this is the retry.
@@ -3888,6 +3890,18 @@ class AppNotifier extends ChangeNotifier {
}
static const offlineRetryInterval = Duration(seconds: 5);
+
+ /// Waits before each automatic reattach of a pane whose stream died while its machine stayed
+ /// reachable; the last one repeats. A pane used to get one reopen and then sit on Reconnect until
+ /// someone clicked it, in whatever tab it was (2026-10-16).
+ static const paneRetryDelays = <Duration>[
+ Duration(seconds: 1),
+ Duration(seconds: 2),
+ Duration(seconds: 4),
+ Duration(seconds: 8),
+ Duration(seconds: 16),
+ Duration(seconds: 30),
+ ];
static const localDaemonReconnectDelay = Duration(seconds: 1);
static const agentSyncInterval = Duration(seconds: 60);
@@ -8305,6 +8319,7 @@ class AppNotifier extends ChangeNotifier {
_slowReloadTimers.clear();
_offlinePollsInFlight.clear();
_offlineRecoveryInFlight.clear();
+ _cancelPaneRetries();
}
void _startAgentSyncTimer(String machineId) {
@@ -16267,7 +16282,7 @@ class AppNotifier extends ChangeNotifier {
resyncTimeout: const Duration(seconds: 15),
);
pane.session = terminal;
- terminal.addListener(notifyListeners);
+ terminal.addListener(_onSessionChanged);
notifyListeners();
// Wait for the pane's actual measured viewport before asking the daemon to open anything.
// Sending the 80x24 fallback here used to make the daemon spawn the remote TTY (and render its
@@ -16286,7 +16301,7 @@ class AppNotifier extends ChangeNotifier {
final terminal = pane.session;
pane.session = null;
if (terminal == null) return;
- terminal.removeListener(notifyListeners);
+ terminal.removeListener(_onSessionChanged);
try {
if (sendClose) await terminal.close();
} finally {
@@ -17854,6 +17869,79 @@ class AppNotifier extends ChangeNotifier {
// could only ever promise restoration to one of them.
unawaited(_reattachPane(pane, intent: paneIntent));
}
+ // A load is also when a pane that could not be attached before (agents not listed yet) can be.
+ _schedulePaneRetries();
+ }
+
+ /// Every session change repaints, and may mean a pane's stream just died.
+ void _onSessionChanged() {
+ notifyListeners();
+ _schedulePaneRetries();
+ }
+
+ /// Keep one timer for each pane whose stream died while its machine stays reachable, in whichever
+ /// tab it is, and none for any other. Sessions notify on state changes, not on output, so the walk
+ /// is cheap; an app with every terminal live holds no timer.
+ void _schedulePaneRetries() {
+ if (_disposed) return;
+ for (final pane in allPanes) {
+ if (!_paneRetryWanted(pane)) {
+ _paneRetryTimers.remove(pane)?.cancel();
+ if (pane.session?.status == TerminalSessionStatus.controlling) {
+ _paneRetryAttempts.remove(pane);
+ }
+ continue;
+ }
+ if (_paneRetryTimers.containsKey(pane)) continue;
+ final attempts = _paneRetryAttempts[pane] ?? 0;
+ final delay = paneRetryDelays[min(attempts, paneRetryDelays.length - 1)];
+ _paneRetryTimers[pane] = Timer(delay, () {
+ _paneRetryTimers.remove(pane);
+ if (!_paneRetryWanted(pane)) return;
+ _paneRetryAttempts[pane] = attempts + 1;
+ appLog.info(
+ 'terminal',
+ 'reattach machine=${pane.machineId} agent=${pane.agentId} attempt=${attempts + 1}',
+ );
+ final session = pane.session!;
+ // A sweep reopen is never a gesture. It must not take the terminal on the strength of an
+ // earlier click whose open failed, and only its first attempt may redial the machine's link:
+ // a forced redial knocks every other pane on that machine over too.
+ session.takeover = false;
+ unawaited(session.reopen(redial: attempts == 0));
+ });
+ }
+ _paneRetryTimers.removeWhere((pane, timer) {
+ if (allPanes.contains(pane)) return false;
+ timer.cancel();
+ return true;
+ });
+ _paneRetryAttempts.removeWhere((pane, _) => !allPanes.contains(pane));
+ }
+
+ /// A dead stream on a machine that is up and can open terminals politely. Offline machines are left
+ /// to the offline/online path, and `takenOver` panes to a person. A dead watcher is included, unlike
+ /// in [_paneNeedsAttach]: the reopen is polite, so it comes back as a watcher and takes nothing.
+ bool _paneRetryWanted(TerminalPane pane) {
+ final session = pane.session;
+ final machine = machineStates[pane.machineId];
+ return pane.agentId != null &&
+ session != null &&
+ !session.closedHere &&
+ (session.status == TerminalSessionStatus.error ||
+ session.status == TerminalSessionStatus.closed) &&
+ machine != null &&
+ machine.nodeOnline == true &&
+ machine.terminalNoTakeoverAvailable &&
+ _canAttachPane(pane);
+ }
+
+ void _cancelPaneRetries() {
+ for (final timer in _paneRetryTimers.values) {
+ timer.cancel();
+ }
+ _paneRetryTimers.clear();
+ _paneRetryAttempts.clear();
}
/// Whether this tile is showing something that is not a working terminal.
@@ -19135,7 +19223,7 @@ class AppNotifier extends ChangeNotifier {
)..session = session;
panes.add(pane);
focusedPaneId = pane.id;
- session.addListener(notifyListeners);
+ session.addListener(_onSessionChanged);
return pane;
}
@@ -19249,7 +19337,7 @@ class AppNotifier extends ChangeNotifier {
_stopAllAgentSyncTimers();
_clearAllTurnActivity();
for (final pane in allPanes) {
- pane.session?.removeListener(notifyListeners);
+ pane.session?.removeListener(_onSessionChanged);
pane.session?.dispose();
}
for (final swarm in swarms) {
diff --git a/desktop/lib/terminal/terminal_session.dart b/desktop/lib/terminal/terminal_session.dart
index 64b5ff925..b9bca2af8 100644
--- a/desktop/lib/terminal/terminal_session.dart
+++ b/desktop/lib/terminal/terminal_session.dart
@@ -8,6 +8,7 @@ import 'package:xterm/xterm.dart';
import '../core/crash_log.dart';
import '../core/sleep_aware.dart';
+import '../logging/app_log.dart';
import 'terminal_binary.dart';
import 'terminal_input.dart';
import 'terminal_viewport.dart';
@@ -236,6 +237,10 @@ class TerminalSession extends ChangeNotifier {
String? errorCode;
String? errorMessage;
+ /// This app ended the stream itself ([close]). A dead stream is reattached automatically
+ /// (`AppNotifier` pane retries); one this app closed on purpose is not. Cleared by the next open.
+ bool closedHere = false;
+
/// Who took this terminal, while [status] is [TerminalSessionStatus.takenOver]
/// and the daemon said (`terminal_closed.takenBy`); null from an older daemon
/// or a taker that did not introduce itself — "another app", then.
@@ -344,7 +349,9 @@ class TerminalSession extends ChangeNotifier {
/// Input resumes only after the replacement stream's first keyframe.
/// [force] is a person asking: it arms one takeover and reopens a stream this
/// session is merely watching, which is what the band's "Take control" does.
- Future<void> reopen({bool force = false}) async {
+ /// [redial] false keeps an unanswered open from forcing a fresh dial of the machine's link
+ /// ([onOpenStalled]): an automatic retry must not knock the machine's other panes over.
+ Future<void> reopen({bool force = false, bool redial = true}) async {
if (force) takeover = true;
if (_disposed ||
status == TerminalSessionStatus.opening ||
@@ -358,6 +365,7 @@ class TerminalSession extends ChangeNotifier {
waitForViewportSize: false,
resetRecovery: true,
preserveTerminal: true,
+ allowRedial: redial,
);
}
@@ -369,8 +377,10 @@ class TerminalSession extends ChangeNotifier {
required bool waitForViewportSize,
required bool resetRecovery,
required bool preserveTerminal,
+ bool allowRedial = true,
}) async {
if (_disposed) return;
+ closedHere = false;
final generation = ++_generation;
_cancelTimers();
_abortActiveUpload();
@@ -401,7 +411,7 @@ class TerminalSession extends ChangeNotifier {
_resyncAttempts = 0;
if (resetRecovery) {
_autoReopenAttempts = 0;
- _openStallRecovered = false;
+ _openStallRecovered = !allowRedial;
}
cols = _clampCols(initialCols);
rows = _clampRows(initialRows);
@@ -707,6 +717,13 @@ class TerminalSession extends ChangeNotifier {
}
return true;
}
+ // The daemon no longer holds this stream (it closed it while its link to the backend was
+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now
+ // rather than waiting out 15 s of silence and three resyncs (2026-10-16).
+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {
+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));
+ return true;
+ }
// A polite open, refused: another client is driving this terminal and
// this open asked not to take it from them. Not a failure, and it must
// not read as one — `error` is what the notifier's reattach sweep
@@ -1489,7 +1506,11 @@ class TerminalSession extends ChangeNotifier {
// the same gap — instead of declaring TERMINAL_STREAM_TIMEOUT and throwing away typed input.
if (afterSleep) _lastStreamActivityAt = _now();
final generation = _generation;
- final sent = await send('terminal_alive', {'streamId': streamId});
+ final sent = await send('terminal_alive', {
+ 'streamId': streamId,
+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).
+ 'reportMissing': true,
+ });
if (!sent && _isCurrent(generation)) {
transportLost('Terminal heartbeat was not sent');
return;
@@ -1548,6 +1569,7 @@ class TerminalSession extends ChangeNotifier {
'streamId': currentStream,
'attempt': _resyncAttempts,
'reason': errorCode,
+ 'reportMissing': true,
});
if (!_isCurrent(generation) ||
streamId != currentStream ||
@@ -1584,8 +1606,9 @@ class TerminalSession extends ChangeNotifier {
return;
}
_autoReopenAttempts++;
- debugPrint(
- '[terminal-session] recovery_reopen agent=$agentId stream=$previousStream',
+ appLog.info(
+ 'terminal',
+ 'reopen machine=$machineId agent=$agentId stream=$previousStream reason=$reason',
);
if (previousStream != null) {
unawaited(send('terminal_close', {'streamId': previousStream}));
@@ -1600,6 +1623,7 @@ class TerminalSession extends ChangeNotifier {
}
Future<void> close() async {
+ closedHere = true;
_generation++;
final closingStream = streamId;
_cancelTimers();
@@ -1626,6 +1650,10 @@ class TerminalSession extends ChangeNotifier {
status == TerminalSessionStatus.takenOver) {
return;
}
+ appLog.warn(
+ 'terminal',
+ 'transport lost machine=$machineId agent=$agentId message=$message',
+ );
_generation++;
_cancelTimers();
_inputBytes.clear();
@@ -1640,6 +1668,12 @@ class TerminalSession extends ChangeNotifier {
void _fail(String code, String? message) {
if (_disposed) return;
+ // Why a pane shows Reconnect was only ever in debugPrint, invisible on a person's machine (2026-10-16).
+ appLog.warn(
+ 'terminal',
+ 'failed machine=$machineId agent=$agentId stream=$streamId code=$code'
+ '${message == null ? '' : ' message=$message'}',
+ );
_generation++;
_cancelTimers();
_inputBytes.clear();
diff --git a/desktop/test/harness_attachments_test.dart b/desktop/test/harness_attachments_test.dart
index c5873be4f..e0ce6c5d0 100644
--- a/desktop/test/harness_attachments_test.dart
+++ b/desktop/test/harness_attachments_test.dart
@@ -55,7 +55,7 @@ class _RecordingSession extends TerminalSession {
/// "Take control": the daemon answers by handing this window the lease.
@override
- Future<void> reopen({bool force = false}) async {
+ Future<void> reopen({bool force = false, bool redial = true}) async {
calls.add('reopen:$force');
watching = false;
notifyListeners();
diff --git a/desktop/test/pane_retry_test.dart b/desktop/test/pane_retry_test.dart
new file mode 100644
index 000000000..fc1690f9f
--- /dev/null
+++ b/desktop/test/pane_retry_test.dart
@@ -0,0 +1,267 @@
+import 'dart:typed_data';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/logging/app_log.dart';
+import 'package:harness/state/app_state.dart';
+import 'package:harness/terminal/terminal_binary.dart';
+import 'package:harness/terminal/terminal_session.dart';
+import 'package:harness/ws/ws_conn.dart';
+
+import 'swarm_state_test.dart' show createApp;
+
+/// A current daemon with two agents that answers every request and never answers an open, so a
+/// reattached session stays `opening` until the test breaks it again.
+class _Connection extends WsConn {
+ _Connection()
+ : super(
+ wsBaseUrl: '',
+ autonomousEnv: 'test',
+ machineId: 'm',
+ accessTokenProvider: (_, _) async => '',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: (_) {},
+ );
+ final opened = <String>[];
+ final opens = <Map<String, dynamic>>[];
+ @override
+ Future<void> waitUntilReady({required Duration timeout}) async {}
+ @override
+ Future<Map<String, dynamic>> request(
+ String type, {
+ Map<String, dynamic> payload = const {},
+ Duration timeout = const Duration(seconds: 20),
+ }) async => type == 'agents_list'
+ ? {
+ 'agents': [
+ for (final id in ['a0', 'a1'])
+ {
+ 'id': id,
+ 'name': id,
+ 'engine': 'codex',
+ 'terminal': {
+ 'runtimes': [
+ {'backend': 'tmux', 'paneId': id == 'a0' ? '%1' : '%2'},
+ ],
+ },
+ },
+ ],
+ }
+ : {
+ 'protocolVersion': 3,
+ 'backend': 'tmux',
+ 'available': true,
+ 'features': {'noTakeover': true},
+ };
+ @override
+ Future<bool> sendTerminalFrame(
+ String type,
+ Map<String, dynamic> payload,
+ ) async {
+ if (type == 'terminal_open') {
+ opened.add(payload['agentId'] as String);
+ opens.add(payload);
+ }
+ return true;
+ }
+}
+
+class _RecordingLog implements AppLog {
+ final lines = <String>[];
+ @override
+ void record(
+ AppLogLevel level,
+ String category,
+ String message, {
+ Object? error,
+ StackTrace? stackTrace,
+ }) => lines.add('$category $message');
+}
+
+void main() {
+ /// Two tabs, `a0` in the first (background) and `a1` in the second (in front), on a machine
+ /// that has come back once, so its agents and terminal capability are loaded.
+ Future<({AppNotifier app, _Connection connection, TerminalSession background})>
+ rig(WidgetTester tester) async {
+ final connection = _Connection();
+ // Disposed at the end of each test body: it disposes the sessions too, whose open watchdogs
+ // and this sweep's timers must not outlive the body.
+ final app = createApp(connectionForTest: (_) => connection);
+ app.machineStates['m']!.nodeOnline = true;
+ TerminalSession terminal(String id) => TerminalSession(
+ machineId: 'm',
+ agentId: id,
+ agentName: id,
+ engineId: 'codex',
+ send: connection.sendTerminalFrame,
+ sendBinary: (_) async => true,
+ // The fake daemon never answers an open; keep the open watchdog out of these timelines.
+ resyncTimeout: const Duration(hours: 1),
+ )..status = TerminalSessionStatus.controlling;
+ final background = terminal('a0');
+ app.adoptSessionForTest(background);
+ app.newSwarm(name: 'Second');
+ app.adoptSessionForTest(terminal('a1'));
+ await app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': false},
+ });
+ await app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': true},
+ });
+ for (var i = 0; i < 30; i++) {
+ await tester.pump(const Duration(milliseconds: 100));
+ }
+ connection.opened.clear();
+ connection.opens.clear();
+ return (app: app, connection: connection, background: background);
+ }
+
+ testWidgets(
+ 'a pane whose stream died on a reachable machine is reattached without a click, behind another tab',
+ (tester) async {
+ final log = _RecordingLog();
+ final previous = appLog;
+ appLog = log;
+ addTearDown(() => appLog = previous);
+ final r = await rig(tester);
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 900));
+ expect(r.connection.opened, isEmpty);
+ await tester.pump(const Duration(milliseconds: 200));
+
+ expect(r.connection.opened, ['a0']);
+ expect(log.lines, contains('terminal reattach machine=m agent=a0 attempt=1'));
+ r.app.dispose();
+ },
+ );
+
+ testWidgets('each further attempt waits longer: 1 s, then 2 s', (tester) async {
+ final r = await rig(tester);
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+ expect(r.connection.opened, ['a0']);
+
+ r.background.transportLost('gone again');
+ await tester.pump(const Duration(milliseconds: 1100));
+ expect(r.connection.opened, ['a0'], reason: 'the second wait is 2 s');
+ await tester.pump(const Duration(seconds: 1));
+ expect(r.connection.opened, ['a0', 'a0']);
+ r.app.dispose();
+ });
+
+ testWidgets(
+ 'a sweep reopen never takes the terminal, whatever the last open asked for',
+ (tester) async {
+ final r = await rig(tester);
+ expect(r.background.takeover, isTrue, reason: 'a person opened it');
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+
+ expect(r.connection.opens.single['takeover'], isFalse);
+ r.app.dispose();
+ },
+ );
+
+ testWidgets('a pane someone else took is never reattached', (tester) async {
+ final r = await rig(tester);
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+ await r.app.handleEventForTest('m', {
+ 'type': 'terminal_error',
+ 'payload': {
+ 'requestId': r.connection.opens.single['requestId'],
+ 'code': 'CONTROL_LEASE_HELD',
+ },
+ });
+ expect(r.background.status, TerminalSessionStatus.takenOver);
+
+ await tester.pump(const Duration(seconds: 40));
+
+ expect(r.connection.opens, hasLength(1));
+ r.app.dispose();
+ });
+
+ testWidgets('a dead watcher is reattached, as a watcher', (tester) async {
+ final r = await rig(tester);
+ r.background
+ ..watching = true
+ ..takeover = false;
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+
+ expect(r.connection.opens.single['takeover'], isFalse);
+ r.app.dispose();
+ });
+
+ testWidgets('a pane that is live again keeps no retry going', (tester) async {
+ final r = await rig(tester);
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+ await r.app.handleEventForTest('m', {
+ 'type': 'terminal_ready',
+ 'payload': {
+ 'requestId': r.connection.opens.single['requestId'],
+ 'protocolVersion': 3,
+ 'streamId': '00112233-4455-6677-8899-aabbccddeeff',
+ 'agentId': 'a0',
+ },
+ });
+ await r.background.handleBinary(
+ TerminalBinaryFrame(
+ kind: TerminalBinaryKind.keyframe,
+ streamId: '00112233-4455-6677-8899-aabbccddeeff',
+ seq: 0,
+ bytes: Uint8List.fromList('prompt> '.codeUnits),
+ compressed: false,
+ cols: 80,
+ rows: 24,
+ ),
+ );
+ expect(r.background.status, TerminalSessionStatus.controlling);
+
+ await tester.pump(const Duration(seconds: 60));
+
+ expect(r.connection.opens, hasLength(1));
+ r.app.dispose();
+ });
+
+ testWidgets('a stream this app closed itself is left closed', (tester) async {
+ final r = await rig(tester);
+
+ await r.background.close();
+ await tester.pump(const Duration(seconds: 3));
+
+ expect(r.connection.opened, isEmpty);
+ r.app.dispose();
+ });
+
+ testWidgets('nothing is retried while the machine is offline', (tester) async {
+ final r = await rig(tester);
+
+ await r.app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': false},
+ });
+ // The first retry would come after 1 s. Stay under the offline poll's 5 s tick, which reads the
+ // live machine list (refused under test).
+ await tester.pump(const Duration(seconds: 4));
+
+ expect(r.connection.opened, isEmpty);
+
+ // Back online, which stops the offline poll's timer before the test ends.
+ await r.app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': true},
+ });
+ for (var i = 0; i < 30; i++) {
+ await tester.pump(const Duration(milliseconds: 100));
+ }
+ r.app.dispose();
+ });
+}
diff --git a/desktop/test/terminal_session_test.dart b/desktop/test/terminal_session_test.dart
index 9b16ab5e5..f6c1285ba 100644
--- a/desktop/test/terminal_session_test.dart
+++ b/desktop/test/terminal_session_test.dart
@@ -3,6 +3,7 @@ import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
+import 'package:harness/logging/app_log.dart';
import 'package:harness/terminal/terminal_session.dart';
import 'package:harness/terminal/terminal_binary.dart';
import 'package:flutter_test/flutter_test.dart';
@@ -1390,6 +1391,120 @@ void main() {
expect(session.errorCode, 'TERMINAL_RESYNC_TIMEOUT');
});
+ testWidgets(
+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',
+ (tester) async {
+ session.dispose();
+ session = TerminalSession(
+ machineId: 'machine-1',
+ agentId: 'agent-1',
+ agentName: 'backend-api',
+ engineId: 'codex',
+ resyncTimeout: const Duration(seconds: 30),
+ send: (type, payload) async {
+ sent.add((type: type, payload: Map<String, dynamic>.from(payload)));
+ return true;
+ },
+ sendBinary: (frame) async => true,
+ );
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('screen'), keyframe: true, cols: 80, rows: 24),
+ );
+ await tester.pump(const Duration(seconds: 5));
+ expect(
+ sent.lastWhere((frame) => frame.type == 'terminal_alive').payload,
+ {'streamId': streamId, 'reportMissing': true},
+ );
+
+ await session.handleBinary(output(2, utf8.encode('gap')));
+ await tester.pump();
+ expect(
+ sent
+ .firstWhere((frame) => frame.type == 'terminal_resync')
+ .payload['reportMissing'],
+ isTrue,
+ );
+ await session.close();
+ },
+ );
+
+ test('a reopen that may not redial leaves the machine link alone', () async {
+ var redials = 0;
+ TerminalSession stalled() => TerminalSession(
+ machineId: 'machine-1',
+ agentId: 'agent-1',
+ agentName: 'backend-api',
+ engineId: 'codex',
+ send: (type, payload) async => type != 'terminal_open',
+ sendBinary: (frame) async => true,
+ onOpenStalled: () async => redials++,
+ );
+ final quiet = stalled();
+ addTearDown(quiet.dispose);
+ await quiet.reopen(redial: false);
+ expect(redials, 0);
+ expect(quiet.status, TerminalSessionStatus.error);
+
+ final loud = stalled();
+ final reopening = loud.reopen();
+ await pumpEventQueue();
+ expect(redials, 1);
+ loud.dispose();
+ await reopening;
+ });
+
+ test('a stream the daemon no longer has is reopened in place at once', () async {
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('screen'), keyframe: true, cols: 100, rows: 30),
+ );
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ 'message': 'no live terminal stream',
+ });
+ await pumpEventQueue();
+
+ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(2));
+ expect(session.status, TerminalSessionStatus.opening);
+ expect(session.errorCode, isNot('TERMINAL_STREAM_NOT_FOUND'));
+ });
+
+ test('a missing-stream report about another stream is ignored', () async {
+ await ready();
+ await session.handleFrame('terminal_error', {
+ 'streamId': 'ffffffff-ffff-ffff-ffff-ffffffffffff',
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+ await pumpEventQueue();
+
+ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(1));
+ expect(session.status, isNot(TerminalSessionStatus.error));
+ });
+
+ test('a failure is written to the file log with its code', () async {
+ final log = _RecordingLog();
+ final previous = appLog;
+ appLog = log;
+ addTearDown(() => appLog = previous);
+ await ready();
+
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_SNAPSHOT_FAILED',
+ 'message': 'tmux snapshot failed',
+ });
+
+ expect(
+ log.lines,
+ contains(
+ 'terminal failed machine=machine-1 agent=agent-1 stream=$streamId '
+ 'code=TERMINAL_SNAPSHOT_FAILED message=tmux snapshot failed',
+ ),
+ );
+ });
+
group('composer', () {
/// The `message` frames this session put on the wire.
List<Map<String, dynamic>> messages() => [
@@ -1892,3 +2007,15 @@ void main() {
);
});
}
+
+class _RecordingLog implements AppLog {
+ final lines = <String>[];
+ @override
+ void record(
+ AppLogLevel level,
+ String category,
+ String message, {
+ Object? error,
+ StackTrace? stackTrace,
+ }) => lines.add('$category $message');
+}
diff --git a/docs/plans/2026-10-16-terminal-auto-recovery-implementation.md b/docs/plans/2026-10-16-terminal-auto-recovery-implementation.md
new file mode 100644
index 000000000..6d524b948
--- /dev/null
+++ b/docs/plans/2026-10-16-terminal-auto-recovery-implementation.md
@@ -0,0 +1,797 @@
+# Terminal Auto-Recovery Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Remote terminal panes recover by themselves after their stream dies while the machine stays reachable, and the desktop's file log says why a pane failed.
+
+**Architecture:** Clients that opt in with `reportMissing: true` get an explicit `TERMINAL_STREAM_NOT_FOUND` from the host when their heartbeat or resync names a stream the host no longer holds, and reopen at once. The desktop `AppNotifier` reattaches any `error`/`closed` pane on a reachable machine on a per-pane backoff, whichever tab it is in. `TerminalSession` failures and recoveries go to `appLog`.
+
+**Tech Stack:** TypeScript + vitest (CLI daemon); Dart/Flutter + flutter_test, fake_async (desktop, mobile).
+
+**Spec:** `docs/plans/2026-10-16-terminal-auto-recovery.md`
+
+## Global Constraints
+
+- No backend change. No new frame type, and no change to the E2EE type sets in `cli/src/lib/e2ee/core.ts`.
+- Opt-in only: the host answers only frames that carry `reportMissing: true`. Clients without the flag get exactly today's silence.
+- Answer shape: `terminal_error { streamId, code: 'TERMINAL_STREAM_NOT_FOUND', message: 'no live terminal stream' }`.
+- Desktop retry backoff: 1, 2, 4, 8, 16, then every 30 s. Reset when the pane reaches `controlling`. No retry while the machine is offline. Never retry `takenOver` or watching panes. The reattach is a polite open (`AttachIntent.automatic`).
+- The desktop sweep's timers exist only while some pane needs a retry. An idle app runs none.
+- Mobile gets the session changes only (reportMissing and reopen on not-found), and no sweep.
+- Logs carry machine, agent, stream, code and message only: never terminal content or paths.
+- Do not commit unless the owner asks. Comments explain why and name the incident (2026-10-16).
+
+## Review Focus
+
+1. **Retry storm against a machine that refuses every open.** The backoff must cap at 30 s per pane. Many broken panes must not multiply the timers beyond one per pane. Task 3 asserts the 1 s → 2 s spacing, and the cap follows from the same table.
+2. **The sweep fighting another client.** A `takenOver` pane, or a session that is `watching`, must never be reopened automatically (`_paneNeedsAttach` already excludes them). Task 3 relies on `_paneNeedsAttach`, and the reviewer should confirm no new path bypasses it.
+3. **Timers left alive after sign-out or dispose.** `testWidgets` fails on a pending timer, so Task 3's tests prove dispose. Sign-out goes through `_stopAllOfflineRetries`, which Task 3 extends.
+4. **An old client meeting the new host, or the reverse.** The host must stay silent without the flag (Task 1 test). A new client must still work against an old host: no reply arrives, and the sweep still recovers the pane.
+5. **A NOT_FOUND for another stream, or one arriving while a reopen is in flight.** It must be ignored. Task 2 and Task 4 test the other-stream case. The in-flight case is covered by the existing `errorStream != streamId` filter, because `streamId` is null while opening.
+
+---
+
+### Task 1: Host reports a missing stream to clients that ask (CLI)
+
+**Files:**
+- Modify: `cli/src/lib/terminalStreamManager.ts` (`alive()` ~line 640, `resyncRequested()` ~line 1011, add `reportMissing()` beside them)
+- Test: `cli/src/lib/terminalStreamManager.spec.ts`
+
+**Interfaces:**
+- Produces: the host answers `terminal_error` with `code: 'TERMINAL_STREAM_NOT_FOUND'` and the named `streamId`, but only when the incoming `terminal_alive`/`terminal_resync` payload has `reportMissing: true`. Tasks 2 and 4 consume this.
+
+- [ ] **Step 1: Write the failing tests**
+
+Add inside `describe('TerminalStreamManager', …)` in `cli/src/lib/terminalStreamManager.spec.ts`, after the test "takes over the first controller and expires the replacement lease after heartbeat timeout":
+
+```ts
+ // A host that lost its backend link closes its remote streams without telling anyone; the client
+ // only found out after 15 s of silence and three ignored resyncs, then gave up (2026-10-16). A
+ // client that asks is now told at its next heartbeat.
+ describe('a heartbeat or resync for a stream this connection does not hold', () => {
+ const errors = () => sent.filter((frame) => frame.type === 'terminal_error')
+
+ it('is answered with TERMINAL_STREAM_NOT_FOUND when the client asks', async () => {
+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1', reportMissing: true })
+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-2', attempt: 1, reportMissing: true })
+ expect(errors().map((frame) => [frame.connId, frame.payload.code, frame.payload.streamId])).toEqual([
+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-1'],
+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-2'],
+ ])
+ })
+
+ it('stays silent for a client that does not ask, as every older client', async () => {
+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1' })
+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-1', attempt: 1 })
+ expect(errors()).toEqual([])
+ })
+
+ it('counts a stream held by another connection as missing for this one, and leaves it alone', async () => {
+ await manager.handleFrame('web-1', 'terminal_open', {
+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,
+ })
+ const streamId = sent.findLast((frame) => frame.type === 'terminal_ready')?.payload.streamId as string
+ await manager.handleFrame('web-2', 'terminal_alive', { streamId, reportMissing: true })
+ expect(errors().at(-1)).toMatchObject({
+ connId: 'web-2',
+ payload: { code: 'TERMINAL_STREAM_NOT_FOUND', streamId },
+ })
+ expect(stream.closed).toBe(false)
+ })
+ })
+```
+
+- [ ] **Step 2: Run the tests to verify they fail**
+
+Run: `cd cli && npx vitest run src/lib/terminalStreamManager.spec.ts -t "does not hold"`
+Expected: FAIL. The first and third tests find no `terminal_error`: `expected [] to deeply equal [...]`, and `toMatchObject` against `undefined`. The second test passes; it pins today's behaviour.
+
+- [ ] **Step 3: Implement**
+
+In `cli/src/lib/terminalStreamManager.ts`, replace `alive()`:
+
+```ts
+ private alive(connId: string, payload: FramePayload): void {
+ const state = this.streamFor(connId, payload)
+ if (!state) { this.reportMissing(connId, payload); return }
+ state.expiresAt = this.now() + HEARTBEAT_TIMEOUT_MS
+ }
+
+ /**
+ * Tell a client that asked (`reportMissing`) that the stream it names is gone. A host that lost its
+ * backend link closes its remote streams without notice, and the client used to learn it only after
+ * 15 s of silence and three resyncs this manager ignored, then gave up on the pane (2026-10-16). Opt-in,
+ * so an older client keeps the silence its own recovery was built around.
+ */
+ private reportMissing(connId: string, payload: FramePayload): void {
+ if (payload.reportMissing !== true || typeof payload.streamId !== 'string') return
+ this.sendError(connId, 'TERMINAL_STREAM_NOT_FOUND', { streamId: payload.streamId, message: 'no live terminal stream' })
+ }
+```
+
+and replace `resyncRequested()`:
+
+```ts
+ private async resyncRequested(connId: string, payload: FramePayload): Promise<void> {
+ const state = this.streamFor(connId, payload)
+ if (!state) { this.reportMissing(connId, payload); return }
+ state.resyncs++
+ this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })
+ await this.sendKeyframe(state, 'resync')
+ }
+```
+
+- [ ] **Step 4: Run the tests to verify they pass**
+
+Run: `cd cli && npx vitest run src/lib/terminalStreamManager.spec.ts`
+Expected: PASS, the whole file (new tests plus every existing one).
+
+- [ ] **Step 5: Typecheck**
+
+Run: `cd cli && npm run typecheck`
+Expected: no errors.
+
+---
+
+### Task 2: Desktop session asks, reopens on the answer, and logs failures
+
+**Files:**
+- Modify: `desktop/lib/terminal/terminal_session.dart`. That means the imports, the `terminal_error` branch (~line 659–735), `_sendHeartbeat` (~1492), `_sendResyncAttempt` (~1547), `_recoverByReopen` (~1569), `transportLost` and `_fail` (~1612–1650).
+- Test: `desktop/test/terminal_session_test.dart`
+
+**Interfaces:**
+- Consumes: Task 1's `TERMINAL_STREAM_NOT_FOUND` answer.
+- Produces: `terminal_alive` and `terminal_resync` payloads carry `'reportMissing': true`. `appLog` lines in category `terminal` start with `failed `, `transport lost `, or `reopen `. Task 3 adds `reattach `.
+
+- [ ] **Step 1: Write the failing tests**
+
+In `desktop/test/terminal_session_test.dart`, add `import 'package:harness/logging/app_log.dart';` to the imports. Add this class after `main()`'s closing brace:
+
+```dart
+class _RecordingLog implements AppLog {
+ final lines = <String>[];
+ @override
+ void record(
+ AppLogLevel level,
+ String category,
+ String message, {
+ Object? error,
+ StackTrace? stackTrace,
+ }) => lines.add('$category $message');
+}
+```
+
+Add these tests inside `main()`, after the test "resync retries three times, reopens once, then fails closed":
+
+```dart
+ testWidgets(
+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',
+ (tester) async {
+ session.dispose();
+ session = TerminalSession(
+ machineId: 'machine-1',
+ agentId: 'agent-1',
+ agentName: 'backend-api',
+ engineId: 'codex',
+ resyncTimeout: const Duration(seconds: 30),
+ send: (type, payload) async {
+ sent.add((type: type, payload: Map<String, dynamic>.from(payload)));
+ return true;
+ },
+ sendBinary: (frame) async => true,
+ );
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('screen'), keyframe: true, cols: 80, rows: 24),
+ );
+ await tester.pump(const Duration(seconds: 5));
+ expect(
+ sent.lastWhere((frame) => frame.type == 'terminal_alive').payload,
+ {'streamId': streamId, 'reportMissing': true},
+ );
+
+ await session.handleBinary(output(2, utf8.encode('gap')));
+ await tester.pump();
+ expect(
+ sent.firstWhere((frame) => frame.type == 'terminal_resync').payload['reportMissing'],
+ isTrue,
+ );
+ await session.close();
+ },
+ );
+
+ test('a stream the daemon no longer has is reopened in place at once', () async {
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('screen'), keyframe: true, cols: 100, rows: 30),
+ );
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ 'message': 'no live terminal stream',
+ });
+ await pumpEventQueue();
+
+ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(2));
+ expect(session.status, TerminalSessionStatus.opening);
+ expect(session.errorCode, isNot('TERMINAL_STREAM_NOT_FOUND'));
+ });
+
+ test('a missing-stream report about another stream is ignored', () async {
+ await ready();
+ await session.handleFrame('terminal_error', {
+ 'streamId': 'ffffffff-ffff-ffff-ffff-ffffffffffff',
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+ await pumpEventQueue();
+
+ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(1));
+ expect(session.status, isNot(TerminalSessionStatus.error));
+ });
+
+ test('a failure is written to the file log with its code', () async {
+ final log = _RecordingLog();
+ final previous = appLog;
+ appLog = log;
+ addTearDown(() => appLog = previous);
+ await ready();
+
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_SNAPSHOT_FAILED',
+ 'message': 'tmux snapshot failed',
+ });
+
+ expect(
+ log.lines,
+ contains(
+ 'terminal failed machine=machine-1 agent=agent-1 stream=$streamId '
+ 'code=TERMINAL_SNAPSHOT_FAILED message=tmux snapshot failed',
+ ),
+ );
+ });
+```
+
+- [ ] **Step 2: Run the tests to verify they fail**
+
+Run: `cd desktop && flutter test test/terminal_session_test.dart --plain-name "daemon"`, then `--plain-name "missing-stream"` and `--plain-name "file log"`.
+Expected:
+- The "heartbeats and resyncs ask…" test fails: the payload is `{streamId: …}` without `reportMissing`.
+- "reopened in place at once" fails: one `terminal_open`, status `error`.
+- "file log" fails: `log.lines` is empty.
+- "another stream" passes, because it pins the existing filter.
+
+- [ ] **Step 3: Implement**
+
+In `desktop/lib/terminal/terminal_session.dart`:
+
+1. Add after `import '../core/sleep_aware.dart';`:
+
+```dart
+import '../logging/app_log.dart';
+```
+
+2. In `case 'terminal_error':`, insert directly before the `if (errorCode == 'CONTROL_LEASE_HELD' && !_openAskedTakeover) {` block:
+
+```dart
+ // The daemon no longer holds this stream (it closed it while its link to the backend was
+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now
+ // rather than waiting out 15 s of silence and three resyncs (2026-10-16).
+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {
+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));
+ return true;
+ }
+```
+
+3. In `_sendHeartbeat`, replace `final sent = await send('terminal_alive', {'streamId': streamId});` with:
+
+```dart
+ final sent = await send('terminal_alive', {
+ 'streamId': streamId,
+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).
+ 'reportMissing': true,
+ });
+```
+
+4. In `_sendResyncAttempt`, add `'reportMissing': true,` to the `terminal_resync` payload, after `'reason': errorCode,`.
+
+5. In `_recoverByReopen`, replace the `debugPrint('[terminal-session] recovery_reopen agent=$agentId stream=$previousStream');` call with:
+
+```dart
+ appLog.info(
+ 'terminal',
+ 'reopen machine=$machineId agent=$agentId stream=$previousStream reason=$reason',
+ );
+```
+
+6. In `transportLost`, insert directly after the early `return;` guard block (before `_generation++;`):
+
+```dart
+ appLog.warn(
+ 'terminal',
+ 'transport lost machine=$machineId agent=$agentId message=$message',
+ );
+```
+
+7. In `_fail`, insert directly after `if (_disposed) return;`:
+
+```dart
+ // Why a pane shows Reconnect was only ever in debugPrint, invisible on a person's machine (2026-10-16).
+ appLog.warn(
+ 'terminal',
+ 'failed machine=$machineId agent=$agentId stream=$streamId code=$code'
+ '${message == null ? '' : ' message=$message'}',
+ );
+```
+
+- [ ] **Step 4: Run the tests to verify they pass**
+
+Run: `cd desktop && flutter test test/terminal_session_test.dart`
+Expected: PASS, the whole file. An existing test that asserted the exact `terminal_alive` payload would need `'reportMissing': true` added. That would be a plan deviation: record it as a ruling.
+
+- [ ] **Step 5: Analyze**
+
+Run: `cd desktop && flutter analyze lib/terminal/terminal_session.dart test/terminal_session_test.dart`
+Expected: No issues found.
+
+---
+
+### Task 3: Desktop reattaches broken panes on a reachable machine, with backoff
+
+**Files:**
+- Modify: `desktop/lib/state/app_state.dart`:
+ - near `offlineRetryInterval` (~line 3486): add `paneRetryDelays`;
+ - near `_offlineRetryTimers` (~969): add the two maps;
+ - `_attachSession` (~15618) and `_detachSession` (~15637): swap the listener;
+ - end of `_attachPendingPanes` (~17183): call the scheduler;
+ - `_stopAllOfflineRetries` (~7830): cancel the retries;
+ - `adoptSessionForTest` (~18441): swap the listener;
+ - add new methods next to `_paneNeedsAttach` (~17193).
+- Create: `desktop/test/pane_retry_test.dart`
+
+**Interfaces:**
+- Consumes: `_paneNeedsAttach`, `_canAttachPane`, `_reattachPane(pane, intent: AttachIntent.automatic)`, `MachineState.nodeOnline`, `MachineState.terminalNoTakeoverAvailable` (all existing). Task 2's `appLog` category `terminal`.
+- Produces: `static const List<Duration> paneRetryDelays` on `AppNotifier`, and the log line `terminal reattach machine=<id> agent=<id> attempt=<n>`.
+
+- [ ] **Step 1: Write the failing tests**
+
+Create `desktop/test/pane_retry_test.dart`:
+
+```dart
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness/logging/app_log.dart';
+import 'package:harness/state/app_state.dart';
+import 'package:harness/terminal/terminal_session.dart';
+import 'package:harness/ws/ws_conn.dart';
+
+import 'swarm_state_test.dart' show createApp;
+
+/// A current daemon with two agents that answers every request and never answers an open, so a
+/// reattached session stays `opening` until the test breaks it again.
+class _Connection extends WsConn {
+ _Connection()
+ : super(
+ wsBaseUrl: '',
+ autonomousEnv: 'test',
+ machineId: 'm',
+ accessTokenProvider: (_, _) async => '',
+ onAuthFailure: (_) {},
+ onEvent: (_) {},
+ onStatus: (_) {},
+ );
+ final opened = <String>[];
+ @override
+ Future<void> waitUntilReady({required Duration timeout}) async {}
+ @override
+ Future<Map<String, dynamic>> request(
+ String type, {
+ Map<String, dynamic> payload = const {},
+ Duration timeout = const Duration(seconds: 20),
+ }) async => type == 'agents_list'
+ ? {
+ 'agents': [
+ for (final id in ['a0', 'a1'])
+ {
+ 'id': id,
+ 'name': id,
+ 'engine': 'codex',
+ 'terminal': {
+ 'runtimes': [
+ {'backend': 'tmux', 'paneId': id == 'a0' ? '%1' : '%2'},
+ ],
+ },
+ },
+ ],
+ }
+ : {
+ 'protocolVersion': 3,
+ 'backend': 'tmux',
+ 'available': true,
+ 'features': {'noTakeover': true},
+ };
+ @override
+ Future<bool> sendTerminalFrame(
+ String type,
+ Map<String, dynamic> payload,
+ ) async {
+ if (type == 'terminal_open') opened.add(payload['agentId'] as String);
+ return true;
+ }
+}
+
+class _RecordingLog implements AppLog {
+ final lines = <String>[];
+ @override
+ void record(
+ AppLogLevel level,
+ String category,
+ String message, {
+ Object? error,
+ StackTrace? stackTrace,
+ }) => lines.add('$category $message');
+}
+
+void main() {
+ /// Two tabs, `a0` in the first (background) and `a1` in the second (in front), on a machine
+ /// that has come back once, so its agents and terminal capability are loaded.
+ Future<({AppNotifier app, _Connection connection, TerminalSession background})>
+ rig(WidgetTester tester) async {
+ final connection = _Connection();
+ final app = createApp(connectionForTest: (_) => connection);
+ addTearDown(app.dispose);
+ app.machineStates['m']!.nodeOnline = true;
+ TerminalSession terminal(String id) => TerminalSession(
+ machineId: 'm',
+ agentId: id,
+ agentName: id,
+ engineId: 'codex',
+ send: connection.sendTerminalFrame,
+ sendBinary: (_) async => true,
+ )..status = TerminalSessionStatus.controlling;
+ final background = terminal('a0');
+ app.adoptSessionForTest(background);
+ app.newSwarm(name: 'Second');
+ app.adoptSessionForTest(terminal('a1'));
+ await app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': false},
+ });
+ await app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': true},
+ });
+ for (var i = 0; i < 30; i++) {
+ await tester.pump(const Duration(milliseconds: 100));
+ }
+ connection.opened.clear();
+ return (app: app, connection: connection, background: background);
+ }
+
+ testWidgets(
+ 'a pane whose stream died on a reachable machine is reattached without a click, behind another tab',
+ (tester) async {
+ final log = _RecordingLog();
+ final previous = appLog;
+ appLog = log;
+ addTearDown(() => appLog = previous);
+ final r = await rig(tester);
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 900));
+ expect(r.connection.opened, isEmpty);
+ await tester.pump(const Duration(milliseconds: 200));
+
+ expect(r.connection.opened, ['a0']);
+ expect(log.lines, contains('terminal reattach machine=m agent=a0 attempt=1'));
+ for (final pane in r.app.allPanes) {
+ await pane.session?.close();
+ }
+ },
+ );
+
+ testWidgets('each further attempt waits longer: 1 s, then 2 s', (tester) async {
+ final r = await rig(tester);
+
+ r.background.transportLost('gone');
+ await tester.pump(const Duration(milliseconds: 1100));
+ expect(r.connection.opened, ['a0']);
+
+ r.background.transportLost('gone again');
+ await tester.pump(const Duration(milliseconds: 1100));
+ expect(r.connection.opened, ['a0'], reason: 'the second wait is 2 s');
+ await tester.pump(const Duration(seconds: 1));
+ expect(r.connection.opened, ['a0', 'a0']);
+ for (final pane in r.app.allPanes) {
+ await pane.session?.close();
+ }
+ });
+
+ testWidgets('nothing is retried while the machine is offline', (tester) async {
+ final r = await rig(tester);
+
+ await r.app.handleEventForTest('m', {
+ 'type': 'node_status',
+ 'payload': {'online': false},
+ });
+ await tester.pump(const Duration(seconds: 40));
+
+ expect(r.connection.opened, isEmpty);
+ for (final pane in r.app.allPanes) {
+ await pane.session?.close();
+ }
+ });
+}
+```
+
+`AppNotifier` is assumed to be exported from `package:harness/state/app_state.dart`; `swarm_reconnect_test.dart` uses `createApp` from the same file. If `createApp`'s return type lives elsewhere, import what `swarm_state_test.dart` imports.
+
+- [ ] **Step 2: Run the tests to verify they fail**
+
+Run: `cd desktop && flutter test test/pane_retry_test.dart`
+Expected: the first two tests FAIL with `expected ['a0'] … actual []`, because nothing reattaches the background pane. The offline test passes, since it pins the boundary.
+
+- [ ] **Step 3: Implement**
+
+In `desktop/lib/state/app_state.dart`:
+
+1. After `static const offlineRetryInterval = Duration(seconds: 5);` add:
+
+```dart
+ /// Waits before each automatic reattach of a pane whose stream died while its machine stayed
+ /// reachable; the last one repeats. A pane used to get one reopen and then sit on Reconnect until
+ /// someone clicked it, in whatever tab it was (2026-10-16).
+ static const paneRetryDelays = <Duration>[
+ Duration(seconds: 1),
+ Duration(seconds: 2),
+ Duration(seconds: 4),
+ Duration(seconds: 8),
+ Duration(seconds: 16),
+ Duration(seconds: 30),
+ ];
+```
+
+2. After `final Map<String, Timer> _offlineRetryTimers = {};` add:
+
+```dart
+ final Map<TerminalPane, Timer> _paneRetryTimers = {};
+ final Map<TerminalPane, int> _paneRetryAttempts = {};
+```
+
+3. In `_attachSession`, replace `terminal.addListener(notifyListeners);` with `terminal.addListener(_onSessionChanged);`. In `_detachSession`, replace `terminal.removeListener(notifyListeners);` with `terminal.removeListener(_onSessionChanged);`. In `adoptSessionForTest`, replace `session.addListener(notifyListeners);` with `session.addListener(_onSessionChanged);`.
+
+4. At the end of `_attachPendingPanes` (after its `for` loop), add:
+
+```dart
+ // A load is also when a pane that could not be attached before (agents not listed yet) can be.
+ _schedulePaneRetries();
+```
+
+5. Directly before `bool _paneNeedsAttach(TerminalPane pane) {`, add:
+
+```dart
+ /// Every session change repaints, and may mean a pane's stream just died.
+ void _onSessionChanged() {
+ notifyListeners();
+ _schedulePaneRetries();
+ }
+
+ /// Keep one timer for each pane whose stream died while its machine stays reachable, in whichever
+ /// tab it is, and none for any other. Sessions notify on state changes, not on output, so the walk
+ /// is cheap; an app with every terminal live holds no timer.
+ void _schedulePaneRetries() {
+ if (_disposed) return;
+ for (final pane in allPanes) {
+ if (!_paneRetryWanted(pane)) {
+ _paneRetryTimers.remove(pane)?.cancel();
+ if (pane.session?.status == TerminalSessionStatus.controlling) {
+ _paneRetryAttempts.remove(pane);
+ }
+ continue;
+ }
+ if (_paneRetryTimers.containsKey(pane)) continue;
+ final attempts = _paneRetryAttempts[pane] ?? 0;
+ final delay = paneRetryDelays[min(attempts, paneRetryDelays.length - 1)];
+ _paneRetryTimers[pane] = Timer(delay, () {
+ _paneRetryTimers.remove(pane);
+ if (!_paneRetryWanted(pane)) return;
+ _paneRetryAttempts[pane] = attempts + 1;
+ appLog.info(
+ 'terminal',
+ 'reattach machine=${pane.machineId} agent=${pane.agentId} attempt=${attempts + 1}',
+ );
+ unawaited(_reattachPane(pane, intent: AttachIntent.automatic));
+ });
+ }
+ _paneRetryTimers.removeWhere((pane, timer) {
+ if (allPanes.contains(pane)) return false;
+ timer.cancel();
+ return true;
+ });
+ _paneRetryAttempts.removeWhere((pane, _) => !allPanes.contains(pane));
+ }
+
+ /// A dead stream on a machine that is up and can open terminals politely. Offline machines are left
+ /// to the offline/online path, and `takenOver`/watching panes to a person (see [_paneNeedsAttach]).
+ bool _paneRetryWanted(TerminalPane pane) {
+ final machine = machineStates[pane.machineId];
+ return pane.session != null &&
+ _paneNeedsAttach(pane) &&
+ machine != null &&
+ machine.nodeOnline == true &&
+ machine.terminalNoTakeoverAvailable &&
+ _canAttachPane(pane);
+ }
+
+ void _cancelPaneRetries() {
+ for (final timer in _paneRetryTimers.values) {
+ timer.cancel();
+ }
+ _paneRetryTimers.clear();
+ _paneRetryAttempts.clear();
+ }
+```
+
+6. At the end of `_stopAllOfflineRetries()`, add `_cancelPaneRetries();`. It is called from sign-out (~line 5882) and from `dispose()` (~18549).
+
+- [ ] **Step 4: Run the tests to verify they pass**
+
+Run: `cd desktop && flutter test test/pane_retry_test.dart test/swarm_reconnect_test.dart test/retake_taken_over_panes_test.dart test/attach_intent_test.dart`
+Expected: PASS, all four files. The last three guard the existing reattach and takeover rules.
+
+- [ ] **Step 5: Analyze and run the bounded desktop suite**
+
+Run: `cd desktop && flutter analyze lib/state/app_state.dart test/pane_retry_test.dart && make -C .. desktop-test`
+Expected: no analyzer issues. The suite matches the documented baseline: any failure must also fail on `origin/main`, checked by name.
+
+---
+
+### Task 4: Mobile session asks and reopens on the answer
+
+**Files:**
+- Modify: `mobile/lib/terminal/terminal_session.dart`: the `terminal_error` branch (~line 968–1052), `terminal_alive` send (~2098), `terminal_resync` payload (~2175).
+- Test: `mobile/test/terminal_session_protocol_test.dart`
+
+**Interfaces:**
+- Consumes: Task 1's answer. Mirrors Task 2's payload flag and reopen. Mobile gets no logging and no sweep (spec part 3).
+
+- [ ] **Step 1: Write the failing tests**
+
+Add inside `main()` of `mobile/test/terminal_session_protocol_test.dart`, after the test "a second answer to the same open leaves one heartbeat, and none after":
+
+```dart
+ test('heartbeats and resyncs ask the daemon to report a stream it no longer has', () {
+ fakeAsync((async) {
+ final s = session(disposeAtEnd: false, resync: const Duration(seconds: 30));
+ s.open();
+ async.flushMicrotasks();
+ final streamId = _streamId();
+ s.handleFrame('terminal_ready', {
+ 'requestId': wire.last('terminal_open')['requestId'],
+ 'agentId': 'a',
+ 'protocolVersion': TerminalSession.protocolVersion,
+ 'streamId': streamId,
+ });
+ s.handleBinary(_keyframe(streamId));
+ async.flushMicrotasks();
+
+ async.elapse(const Duration(seconds: 5));
+ expect(wire.last('terminal_alive'), {'streamId': streamId, 'reportMissing': true});
+
+ s.handleBinary(_output(streamId, 2, 'gap'));
+ async.flushMicrotasks();
+ expect(wire.last('terminal_resync')['reportMissing'], isTrue);
+ s.dispose();
+ });
+ });
+
+ test('a stream the daemon no longer has is reopened in place at once', () async {
+ final s = session();
+ final opening = s.open();
+ final streamId = await ready(s);
+ await opening;
+ await s.handleBinary(_keyframe(streamId));
+
+ await s.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+ await until(() => wire.of('terminal_open').length == 2);
+
+ expect(s.status, TerminalSessionStatus.opening);
+ });
+
+ test('a missing-stream report about another stream is ignored', () async {
+ final s = session();
+ final opening = s.open();
+ await ready(s);
+ await opening;
+
+ await s.handleFrame('terminal_error', {
+ 'streamId': _streamId(),
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+
+ expect(wire.of('terminal_open'), hasLength(1));
+ expect(s.status, isNot(TerminalSessionStatus.error));
+ });
+```
+
+Check `_output`'s exact signature at `test/terminal_session_protocol_test.dart:1943` (`_output(String streamId, int seq, String? text, {…})`), and the shape of `ready()` and `open()` that the neighbouring tests use. Match them rather than this plan if they differ, and record a ruling.
+
+- [ ] **Step 2: Run the tests to verify they fail**
+
+Run: `cd mobile && flutter test test/terminal_session_protocol_test.dart --plain-name "daemon"`, then `--plain-name "reopened in place at once"`.
+Expected:
+- The heartbeat test fails: `reportMissing` is missing.
+- "reopened in place" fails: `never got there`, because the session `_fail`s instead.
+- The other-stream test passes.
+
+- [ ] **Step 3: Implement**
+
+In `mobile/lib/terminal/terminal_session.dart`:
+
+1. In `case 'terminal_error':`, insert directly before the final `_fail(errorCode, payload['message']?.toString());`:
+
+```dart
+ // The daemon no longer holds this stream (it closed it while its link to the backend was
+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now
+ // rather than waiting out the heartbeat silence and three resyncs (2026-10-16).
+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {
+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));
+ return true;
+ }
+```
+
+2. Replace `final sent = await send('terminal_alive', {'streamId': streamId});` with:
+
+```dart
+ final sent = await send('terminal_alive', {
+ 'streamId': streamId,
+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).
+ 'reportMissing': true,
+ });
+```
+
+3. Add `'reportMissing': true,` to the `terminal_resync` payload, after `'reason': errorCode,`.
+
+- [ ] **Step 4: Run the tests to verify they pass**
+
+Run: `cd mobile && flutter test test/terminal_session_protocol_test.dart test/app_state_terminals_test.dart`
+Expected: PASS, both files. `app_state_terminals_test.dart` guards the pager's "not reopened by a guess" rule.
+
+- [ ] **Step 5: Analyze**
+
+Run: `cd mobile && flutter analyze lib/terminal/terminal_session.dart test/terminal_session_protocol_test.dart`
+Expected: No issues found.
+
+---
+
+### Task 5: End to end on the local rig
+
+**Files:** none changed. Results go into the PR description.
+
+**Interfaces:**
+- Consumes: Tasks 1–3 built and installed: the dev CLI on this Mac and on both docker boxes, and the desktop app from this branch.
+
+- [ ] **Step 1: Baseline on `origin/main` builds**
+
+With the released CLI and desktop, open three panes on `machine-remote-1` across two tabs, each showing steady output (`yes | head -c 100M | pv -qL 20k`, or an agent working). For each scenario below, record whether each pane comes back without a click, and how long it takes:
+1. `docker exec machine-remote-1 sh -lc 'harness stop && harness start'`
+2. Stop the local backend for 20 s, then start it again.
+3. On this Mac, drop only the viewer's relay sockets: `pkill -f harnessd-gateway`. The master restarts the gateway process.
+
+- [ ] **Step 2: Same scenarios on this branch's builds**
+
+Install the dev CLI on the Mac and both boxes (`docker cp` recipe) and run the desktop from this branch. Repeat the three scenarios.
+Expected: every pane returns to `controlling` with no click, within a few seconds of the machine being reachable again. `~/.harness/logs/app-*.log` shows `terminal failed …` or `terminal transport lost …`, then `terminal reattach …` or `terminal reopen …` for each pane.
+
+- [ ] **Step 3: Record**
+
+Put the before and after table (scenario × pane × recovered? × seconds) in the PR description, with the commands used.
+
+---
+
+## After the tasks (the owner's call)
+
+- The changes span CLI, desktop and mobile. Run `npm run test:core` only if `src/core` wiring changed (it should not). Record the local runs in the PR, since PR CI runs only the Linux unit checks.
+- Release order does not matter (see the spec's compatibility table): the CLI self-updates, and desktop and mobile ship with their next releases.
diff --git a/docs/plans/2026-10-16-terminal-auto-recovery.md b/docs/plans/2026-10-16-terminal-auto-recovery.md
new file mode 100644
index 000000000..555797808
--- /dev/null
+++ b/docs/plans/2026-10-16-terminal-auto-recovery.md
@@ -0,0 +1,150 @@
+# Remote terminals recover by themselves: find dead streams fast, keep retrying, log why
+
+Requested 2026-10-16. Phase 2 of the "remote terminal keeps showing Reconnect while the network is
+fine" investigation; Phase 1 (relay rate budget) shipped in #1365 and #1367
+([spec](2026-10-16-terminal-relay-rate-budget.md)). This spec also carries Phase 3 (failure reasons in
+the desktop's file log), because Phase 2 cannot be verified on someone else's machine without it.
+
+Scope: the CLI's terminal stream manager (host side), `TerminalSession` and `AppNotifier` in the desktop
+(the web build shares this code) and in mobile (its own copies). No backend change, no new frame type,
+no E2EE type-set change.
+
+## Who hits it
+
+People with many harnesses across several remote machines and tabs. Every pane is a separate chance to
+fall into one of the gaps below, so the person with the most panes sees Reconnect most. A person with a
+few panes on direct p2p rarely does.
+
+## What happens today
+
+When a host loses its backend link (a backend deploy, which happened about every four hours on
+2026-10-15/16, a network blip, sleep), the host closes every remote terminal stream without telling
+anyone (`backendSocket.ts` `linkDown`, `notify=false`). The backend immediately publishes
+`node_status {online:false}` (`role.detach()`). The desktop marks the machine's panes offline and,
+when the machine comes back, reloads its agents and reattaches every pane in every tab (polling up to
+~10 s for the agents to be ready). That path mostly works. The 2026-10-15 14:07 desktop log shows
+panes back four seconds after the drop.
+
+Panes are left on Reconnect by three gaps:
+
+- **(a) The host comes back after the recovery window.** Adapter backoff reaches 16–30 s, or the CLI is
+ restarting for an update. A pane whose reattach missed its window stays `error` until the machine's
+ next status change, which may never come.
+- **(b) A stream dies while the machine stays online, as far as the desktop can tell.** Examples: the
+ viewer's relay socket is replaced, a p2p demotion races a close, or the host's core restarts while
+ its link stays up. The desktop notices only after 15 s of silence (`TERMINAL_STREAM_TIMEOUT`), then
+ sends three resyncs 4 s apart. The host ignores a resync, ack or alive for a stream it does not have
+ (`streamFor` returns null and the handlers return quietly), so the desktop reopens once, and if that
+ fails it calls `_fail`. After `_fail`, nothing retries: the reattach sweep runs only on machine
+ events or a tab switch.
+- **(c) A polite reopen meets the old stream's control lease** (`CONTROL_LEASE_HELD`), and the pane
+ becomes `takenOver`, which is never retried automatically. Out of scope here (see below).
+
+## Change
+
+### 1. Host: report a missing stream to clients that ask (CLI)
+
+In `cli/src/lib/terminalStreamManager.ts`, `alive()` and `resyncRequested()` answer a frame for a
+stream this connection does not own, but only when the frame carries `reportMissing: true`:
+
+```
+terminal_error { streamId, code: 'TERMINAL_STREAM_NOT_FOUND', message: 'no live terminal stream' }
+```
+
+`TERMINAL_STREAM_NOT_FOUND` is the code the chunked-upload path already sends for the same condition.
+Clients that do not send the flag (every client shipped before this change) get exactly today's
+silence, so an older desktop keeps its one-reopen behaviour instead of failing sooner. The answer
+travels like any other terminal frame over relay or p2p, so it needs no new frame type or E2EE type.
+
+### 2. Clients: ask, and reopen on the answer (desktop + mobile `TerminalSession`)
+
+- `terminal_alive` and `terminal_resync` payloads add `reportMissing: true`.
+- A `terminal_error` with `TERMINAL_STREAM_NOT_FOUND` for the current stream reopens the stream in
+ place (`_recoverByReopen`, keeping the rendered screen) instead of `_fail`.
+
+With a new CLI, a dead stream is found by the next heartbeat (≤ 5 s) instead of after
+15 s of silence plus 12 s of resync. With an old CLI nothing changes here; the sweep in 3 still
+recovers the pane.
+
+### 3. Desktop: keep reattaching broken panes while the machine is online (desktop `AppNotifier`; web shares it)
+
+Mobile is excluded on purpose. Its pager has a deliberate rule that a page a person has already looked
+at is not reopened by a guess (`app_state_terminals_test.dart`, "a page a person has looked at is not
+reopened by a guess"), and it already redials a dropped p2p channel itself. Mobile gets parts 1 and 2
+only.
+
+A pane whose session is `error` or `closed` (`_paneNeedsAttach`) on a machine that is online and
+connected is reattached by a sweep, whichever tab it is in:
+
+- One timer per notifier, armed only while at least one such pane exists and cancelled when none
+ remain, so an idle app runs no timer.
+- Each pane retries with backoff: 1, 2, 4, 8, 16, then every 30 s. The backoff resets when the pane
+ reaches `controlling`. It stops while the machine is offline, because the existing offline/online
+ path already owns that case.
+- Reattach goes through the existing `_attachPendingPanes(machine, intent: AttachIntent.automatic)`.
+ That keeps today's rules: it is a polite open (a watcher where someone else is driving), it
+ respects `terminalNoTakeoverAvailable` and `_canAttachPane`, and `takenOver` panes are never
+ touched.
+- The per-session limit of one automatic reopen stays. The sweep is what makes "one reopen, then
+ `error`" stop being a dead end.
+
+No UI change. The header keeps showing Reconnect while a pane is `error`, and clicking it still
+retries immediately. With the sweep, that state now lasts seconds instead of until someone clicks.
+
+### 4. Desktop: write terminal failures and recoveries to the file log (Phase 3)
+
+Today `TerminalSession`'s diagnostics go to `debugPrint` only, so `~/.harness/logs/app-*.log` cannot say
+why a pane showed Reconnect. Log, through `appLog` (category `terminal`, one line each):
+
+- `_fail`: machine, agent, stream, code, message;
+- `transportLost`: machine, agent, message;
+- `_recoverByReopen`: attempt and reason; `recovery_failed`;
+- each sweep reattach: machine, agent, attempt number.
+
+No terminal content and no paths are logged. Per-frame events (each resync attempt, each ack) stay out
+of the file. Mobile is out of scope for this part.
+
+## Compatibility
+
+| Client | CLI on the host | Result |
+|---|---|---|
+| new | new | dead stream found within 5 s, reopened; sweep covers anything left |
+| new | old | today's 15 s + 12 s detection; sweep then recovers the pane |
+| old | new | unchanged: no `reportMissing`, so the host stays silent |
+| old | old | unchanged |
+
+The host and the clients ship independently: the CLI by self-update, desktop and mobile by release.
+
+## Tests
+
+- CLI (`terminalStreamManager` spec): `terminal_alive` and `terminal_resync` for an unknown stream with
+ `reportMissing: true` produce one `TERMINAL_STREAM_NOT_FOUND` error naming that stream. Without the
+ flag they produce nothing. A stream owned by another connection counts as missing for this one.
+- Desktop and mobile `TerminalSession`: alive and resync payloads carry `reportMissing: true`;
+ `TERMINAL_STREAM_NOT_FOUND` for the current stream reopens and keeps the terminal, and for another
+ stream it is ignored.
+- Desktop `AppNotifier`, with fake time:
+ - an `error` pane on an online machine is reattached on the backoff schedule, in a background tab too;
+ - the timer is cancelled once no pane needs it;
+ - no sweep runs while the machine is offline;
+ - `takenOver` panes are never reattached.
+- Desktop log: `_fail` and a sweep reattach each write one `terminal` line to the file log sink.
+- End to end on the local rig (this Mac plus the two docker remote boxes): open several
+ remote panes across two tabs, then run each of these, and every pane must return to `controlling` with no
+ click:
+ 1. restart the remote box's daemon;
+ 2. drop the remote box's backend link (stop the local backend for ~20 s);
+ 3. kill only the viewer's relay socket.
+
+ Record times to recovery before and after.
+
+Validation: the CLI suites that cover `terminalStreamManager`, plus `test:core` if any `src/core`
+wiring changes; the desktop affected tests (`terminal_session_test.dart`, the new pane-retry test);
+the mobile `terminal_session_protocol_test.dart`. Record them in the PR, since PR CI runs only Linux unit checks.
+
+## Out of scope
+
+- Gap (c): letting a pane's own reopen reclaim its old stream's lease.
+- Keeping p2p-carried streams alive across a backend link drop. Both ends tear p2p down with the relay
+ socket today, and separating their lifetimes is an architectural change.
+- Reducing frame counts (client ack cadence, coarser relay flushes).
diff --git a/mobile/lib/terminal/terminal_session.dart b/mobile/lib/terminal/terminal_session.dart
index 52b2204b9..ef434f7b4 100644
--- a/mobile/lib/terminal/terminal_session.dart
+++ b/mobile/lib/terminal/terminal_session.dart
@@ -1049,6 +1049,14 @@ class TerminalSession extends ChangeNotifier {
// TERMINAL_PASTE_FAILED already is, since it may mean the pty is in an unknown state.
// `_fail` itself resolves any in-flight upload (see `_abortActiveUpload`), so there is
// nothing extra to do here for TERMINAL_PASTE_IMAGE_FAILED/TERMINAL_PASTE_FILE_FAILED.
+ //
+ // The daemon no longer holds this stream (it closed it while its link to the backend was
+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now
+ // rather than waiting out the heartbeat silence and three resyncs (2026-10-16).
+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {
+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));
+ return true;
+ }
_fail(errorCode, payload['message']?.toString());
return true;
case 'terminal_transport_error':
@@ -2095,7 +2103,11 @@ class TerminalSession extends ChangeNotifier {
Future<void> _sendHeartbeat() async {
if (!acceptsInput) return;
final generation = _generation;
- final sent = await send('terminal_alive', {'streamId': streamId});
+ final sent = await send('terminal_alive', {
+ 'streamId': streamId,
+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).
+ 'reportMissing': true,
+ });
if (!sent && _isCurrent(generation)) {
transportLost('Terminal heartbeat was not sent');
return;
@@ -2176,6 +2188,7 @@ class TerminalSession extends ChangeNotifier {
'streamId': currentStream,
'attempt': _resyncAttempts,
'reason': errorCode,
+ 'reportMissing': true,
});
if (!_isCurrent(generation) ||
streamId != currentStream ||
diff --git a/mobile/test/terminal_session_protocol_test.dart b/mobile/test/terminal_session_protocol_test.dart
index 66d052f66..8f1cd4d83 100644
--- a/mobile/test/terminal_session_protocol_test.dart
+++ b/mobile/test/terminal_session_protocol_test.dart
@@ -377,6 +377,71 @@ void main() {
},
);
+ test(
+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',
+ () {
+ fakeAsync((async) {
+ final s = session(
+ disposeAtEnd: false,
+ resync: const Duration(seconds: 30),
+ );
+ s.open();
+ async.flushMicrotasks();
+ final streamId = _streamId();
+ s.handleFrame('terminal_ready', {
+ 'requestId': wire.last('terminal_open')['requestId'],
+ 'agentId': 'a',
+ 'protocolVersion': TerminalSession.protocolVersion,
+ 'streamId': streamId,
+ });
+ s.handleBinary(_keyframe(streamId));
+ async.flushMicrotasks();
+
+ async.elapse(const Duration(seconds: 5));
+ expect(wire.last('terminal_alive'), {
+ 'streamId': streamId,
+ 'reportMissing': true,
+ });
+
+ s.handleBinary(_output(streamId, 2, 'gap'));
+ async.flushMicrotasks();
+ expect(wire.last('terminal_resync')['reportMissing'], isTrue);
+ s.dispose();
+ });
+ },
+ );
+
+ test('a stream the daemon no longer has is reopened in place at once', () async {
+ final s = session();
+ final opening = s.open();
+ final streamId = await ready(s);
+ await opening;
+ await s.handleBinary(_keyframe(streamId));
+
+ await s.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+ await until(() => wire.of('terminal_open').length == 2);
+
+ expect(s.status, TerminalSessionStatus.opening);
+ });
+
+ test('a missing-stream report about another stream is ignored', () async {
+ final s = session();
+ final opening = s.open();
+ await ready(s);
+ await opening;
+
+ await s.handleFrame('terminal_error', {
+ 'streamId': _streamId(),
+ 'code': 'TERMINAL_STREAM_NOT_FOUND',
+ });
+
+ expect(wire.of('terminal_open'), hasLength(1));
+ expect(s.status, isNot(TerminalSessionStatus.error));
+ });
+
test(
'heartbeats keep a live stream alive, and a failed one is a lost stream',
() {
@@ -658,6 +723,7 @@ void main() {
'streamId': streamId,
'attempt': 1,
'reason': 'TERMINAL_KEYFRAME_TIMEOUT',
+ 'reportMissing': true,
});
await s.handleBinary(_keyframe(streamId, seq: 5));