The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
fix(terminal): remote panes recover by themselves after their stream dies (#1378)
0xken15b364aede940e
diff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.tsindex 5af84afd5..739fd8e16 100644--- a/cli/src/lib/terminalStreamManager.spec.ts+++ b/cli/src/lib/terminalStreamManager.spec.ts@@ -770,6 +770,41 @@ describe('TerminalStreamManager', () => {&& frame.payload.reason === 'heartbeat timeout')).toBe(true)})+ // A host that lost its backend link closes its remote streams without telling anyone; the client+ // only found out after 15 s of silence and three ignored resyncs, then gave up (2026-10-16). A+ // client that asks is now told at its next heartbeat.+ describe('a heartbeat or resync for a stream this connection does not hold', () => {+ const errors = () => sent.filter((frame) => frame.type === 'terminal_error')++ it('is answered with TERMINAL_STREAM_NOT_FOUND when the client asks', async () => {+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1', reportMissing: true })+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-2', attempt: 1, reportMissing: true })+ expect(errors().map((frame) => [frame.connId, frame.payload.code, frame.payload.streamId])).toEqual([+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-1'],+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-2'],+ ])+ })++ it('stays silent for a client that does not ask, as every older client', async () => {+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1' })+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-1', attempt: 1 })+ expect(errors()).toEqual([])+ })++ it('counts a stream held by another connection as missing for this one, and leaves it alone', async () => {+ await manager.handleFrame('web-1', 'terminal_open', {+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,+ })+ const streamId = sent.findLast((frame) => frame.type === 'terminal_ready')?.payload.streamId as string+ await manager.handleFrame('web-2', 'terminal_alive', { streamId, reportMissing: true })+ expect(errors().at(-1)).toMatchObject({+ connId: 'web-2',+ payload: { code: 'TERMINAL_STREAM_NOT_FOUND', streamId },+ })+ expect(stream.closed).toBe(false)+ })+ })+// Node's clock keeps running while a Mac sleeps, so a lid closed for forty minutes reached the next// sweep as forty minutes without `terminal_alive` and closed every stream at the wake — measured// 2026-09-28 as `[terminal-stream] closed` on every local terminal the moment the lid opened.diff --git a/cli/src/lib/terminalStreamManager.ts b/cli/src/lib/terminalStreamManager.tsindex 1686a99e9..924fd8091 100644--- a/cli/src/lib/terminalStreamManager.ts+++ b/cli/src/lib/terminalStreamManager.ts@@ -639,10 +639,21 @@ export class TerminalStreamManager {private alive(connId: string, payload: FramePayload): void {const state = this.streamFor(connId, payload)- if (!state) return+ if (!state) { this.reportMissing(connId, payload); return }state.expiresAt = this.now() + HEARTBEAT_TIMEOUT_MS}+ /**+ * Tell a client that asked (`reportMissing`) that the stream it names is gone. A host that lost its+ * backend link closes its remote streams without notice, and the client used to learn it only after+ * 15 s of silence and three resyncs this manager ignored, then gave up on the pane (2026-10-16). Opt-in,+ * so an older client keeps the silence its own recovery was built around.+ */+ private reportMissing(connId: string, payload: FramePayload): void {+ if (payload.reportMissing !== true || typeof payload.streamId !== 'string') return+ this.sendError(connId, 'TERMINAL_STREAM_NOT_FOUND', { streamId: payload.streamId, message: 'no live terminal stream' })+ }+private ack(connId: string, payload: FramePayload): void {const state = this.streamFor(connId, payload)if (!state) return@@ -1010,11 +1021,10 @@ export class TerminalStreamManager {private async resyncRequested(connId: string, payload: FramePayload): Promise<void> {const state = this.streamFor(connId, payload)- if (state) {- state.resyncs++- this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })- await this.sendKeyframe(state, 'resync')- }+ if (!state) { this.reportMissing(connId, payload); return }+ state.resyncs+++ this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })+ await this.sendKeyframe(state, 'resync')}private async closeRequested(connId: string, payload: FramePayload): Promise<void> {diff --git a/cli/src/messages/sharedTabs.integration.spec.ts b/cli/src/messages/sharedTabs.integration.spec.tsindex dd9702713..6db96aca6 100644--- a/cli/src/messages/sharedTabs.integration.spec.ts+++ b/cli/src/messages/sharedTabs.integration.spec.ts@@ -279,7 +279,8 @@ describe('shared tabs across three accounts, every piece real but the backend',})expect(relay.unauthorized).toEqual([carol.id])expect(carol.asks).toContainEqual({ force: true, failedToken: 'stale-carol' })- expect(carol.heard).toContainEqual({ sharedTabId: A, ready: true })+ // The ready notice lands a turn after the socket is admitted; asserting it at once failed CI under load (#1378).+ await until(() => expect(carol.heard).toContainEqual({ sharedTabId: A, ready: true }))})it('3. Bob adds bob/api and Carol adds carol/web: both appear in every member\'s roster', async () => {diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dartindex dfb0f7a9c..626d6b3f7 100644--- a/desktop/lib/state/app_state.dart+++ b/desktop/lib/state/app_state.dart@@ -979,6 +979,8 @@ class AppNotifier extends ChangeNotifier {);final Map<String, Timer> _offlineRetryTimers = {};+ final Map<TerminalPane, Timer> _paneRetryTimers = {};+ final Map<TerminalPane, int> _paneRetryAttempts = {};/// One reload per machine, owed after this computer's daemon was too slow to answer its list while/// still on the socket — the error strip says "retrying", and this is the retry.@@ -3888,6 +3890,18 @@ class AppNotifier extends ChangeNotifier {}static const offlineRetryInterval = Duration(seconds: 5);++ /// Waits before each automatic reattach of a pane whose stream died while its machine stayed+ /// reachable; the last one repeats. A pane used to get one reopen and then sit on Reconnect until+ /// someone clicked it, in whatever tab it was (2026-10-16).+ static const paneRetryDelays = <Duration>[+ Duration(seconds: 1),+ Duration(seconds: 2),+ Duration(seconds: 4),+ Duration(seconds: 8),+ Duration(seconds: 16),+ Duration(seconds: 30),+ ];static const localDaemonReconnectDelay = Duration(seconds: 1);static const agentSyncInterval = Duration(seconds: 60);@@ -8305,6 +8319,7 @@ class AppNotifier extends ChangeNotifier {_slowReloadTimers.clear();_offlinePollsInFlight.clear();_offlineRecoveryInFlight.clear();+ _cancelPaneRetries();}void _startAgentSyncTimer(String machineId) {@@ -16267,7 +16282,7 @@ class AppNotifier extends ChangeNotifier {resyncTimeout: const Duration(seconds: 15),);pane.session = terminal;- terminal.addListener(notifyListeners);+ terminal.addListener(_onSessionChanged);notifyListeners();// Wait for the pane's actual measured viewport before asking the daemon to open anything.// Sending the 80x24 fallback here used to make the daemon spawn the remote TTY (and render its@@ -16286,7 +16301,7 @@ class AppNotifier extends ChangeNotifier {final terminal = pane.session;pane.session = null;if (terminal == null) return;- terminal.removeListener(notifyListeners);+ terminal.removeListener(_onSessionChanged);try {if (sendClose) await terminal.close();} finally {@@ -17854,6 +17869,79 @@ class AppNotifier extends ChangeNotifier {// could only ever promise restoration to one of them.unawaited(_reattachPane(pane, intent: paneIntent));}+ // A load is also when a pane that could not be attached before (agents not listed yet) can be.+ _schedulePaneRetries();+ }++ /// Every session change repaints, and may mean a pane's stream just died.+ void _onSessionChanged() {+ notifyListeners();+ _schedulePaneRetries();+ }++ /// Keep one timer for each pane whose stream died while its machine stays reachable, in whichever+ /// tab it is, and none for any other. Sessions notify on state changes, not on output, so the walk+ /// is cheap; an app with every terminal live holds no timer.+ void _schedulePaneRetries() {+ if (_disposed) return;+ for (final pane in allPanes) {+ if (!_paneRetryWanted(pane)) {+ _paneRetryTimers.remove(pane)?.cancel();+ if (pane.session?.status == TerminalSessionStatus.controlling) {+ _paneRetryAttempts.remove(pane);+ }+ continue;+ }+ if (_paneRetryTimers.containsKey(pane)) continue;+ final attempts = _paneRetryAttempts[pane] ?? 0;+ final delay = paneRetryDelays[min(attempts, paneRetryDelays.length - 1)];+ _paneRetryTimers[pane] = Timer(delay, () {+ _paneRetryTimers.remove(pane);+ if (!_paneRetryWanted(pane)) return;+ _paneRetryAttempts[pane] = attempts + 1;+ appLog.info(+ 'terminal',+ 'reattach machine=${pane.machineId} agent=${pane.agentId} attempt=${attempts + 1}',+ );+ final session = pane.session!;+ // A sweep reopen is never a gesture. It must not take the terminal on the strength of an+ // earlier click whose open failed, and only its first attempt may redial the machine's link:+ // a forced redial knocks every other pane on that machine over too.+ session.takeover = false;+ unawaited(session.reopen(redial: attempts == 0));+ });+ }+ _paneRetryTimers.removeWhere((pane, timer) {+ if (allPanes.contains(pane)) return false;+ timer.cancel();+ return true;+ });+ _paneRetryAttempts.removeWhere((pane, _) => !allPanes.contains(pane));+ }++ /// A dead stream on a machine that is up and can open terminals politely. Offline machines are left+ /// to the offline/online path, and `takenOver` panes to a person. A dead watcher is included, unlike+ /// in [_paneNeedsAttach]: the reopen is polite, so it comes back as a watcher and takes nothing.+ bool _paneRetryWanted(TerminalPane pane) {+ final session = pane.session;+ final machine = machineStates[pane.machineId];+ return pane.agentId != null &&+ session != null &&+ !session.closedHere &&+ (session.status == TerminalSessionStatus.error ||+ session.status == TerminalSessionStatus.closed) &&+ machine != null &&+ machine.nodeOnline == true &&+ machine.terminalNoTakeoverAvailable &&+ _canAttachPane(pane);+ }++ void _cancelPaneRetries() {+ for (final timer in _paneRetryTimers.values) {+ timer.cancel();+ }+ _paneRetryTimers.clear();+ _paneRetryAttempts.clear();}/// Whether this tile is showing something that is not a working terminal.@@ -19135,7 +19223,7 @@ class AppNotifier extends ChangeNotifier {)..session = session;panes.add(pane);focusedPaneId = pane.id;- session.addListener(notifyListeners);+ session.addListener(_onSessionChanged);return pane;}@@ -19249,7 +19337,7 @@ class AppNotifier extends ChangeNotifier {_stopAllAgentSyncTimers();_clearAllTurnActivity();for (final pane in allPanes) {- pane.session?.removeListener(notifyListeners);+ pane.session?.removeListener(_onSessionChanged);pane.session?.dispose();}for (final swarm in swarms) {diff --git a/desktop/lib/terminal/terminal_session.dart b/desktop/lib/terminal/terminal_session.dartindex 64b5ff925..b9bca2af8 100644--- a/desktop/lib/terminal/terminal_session.dart+++ b/desktop/lib/terminal/terminal_session.dart@@ -8,6 +8,7 @@ import 'package:xterm/xterm.dart';import '../core/crash_log.dart';import '../core/sleep_aware.dart';+import '../logging/app_log.dart';import 'terminal_binary.dart';import 'terminal_input.dart';import 'terminal_viewport.dart';@@ -236,6 +237,10 @@ class TerminalSession extends ChangeNotifier {String? errorCode;String? errorMessage;+ /// This app ended the stream itself ([close]). A dead stream is reattached automatically+ /// (`AppNotifier` pane retries); one this app closed on purpose is not. Cleared by the next open.+ bool closedHere = false;+/// Who took this terminal, while [status] is [TerminalSessionStatus.takenOver]/// and the daemon said (`terminal_closed.takenBy`); null from an older daemon/// or a taker that did not introduce itself — "another app", then.@@ -344,7 +349,9 @@ class TerminalSession extends ChangeNotifier {/// Input resumes only after the replacement stream's first keyframe./// [force] is a person asking: it arms one takeover and reopens a stream this/// session is merely watching, which is what the band's "Take control" does.- Future<void> reopen({bool force = false}) async {+ /// [redial] false keeps an unanswered open from forcing a fresh dial of the machine's link+ /// ([onOpenStalled]): an automatic retry must not knock the machine's other panes over.+ Future<void> reopen({bool force = false, bool redial = true}) async {if (force) takeover = true;if (_disposed ||status == TerminalSessionStatus.opening ||@@ -358,6 +365,7 @@ class TerminalSession extends ChangeNotifier {waitForViewportSize: false,resetRecovery: true,preserveTerminal: true,+ allowRedial: redial,);}@@ -369,8 +377,10 @@ class TerminalSession extends ChangeNotifier {required bool waitForViewportSize,required bool resetRecovery,required bool preserveTerminal,+ bool allowRedial = true,}) async {if (_disposed) return;+ closedHere = false;final generation = ++_generation;_cancelTimers();_abortActiveUpload();@@ -401,7 +411,7 @@ class TerminalSession extends ChangeNotifier {_resyncAttempts = 0;if (resetRecovery) {_autoReopenAttempts = 0;- _openStallRecovered = false;+ _openStallRecovered = !allowRedial;}cols = _clampCols(initialCols);rows = _clampRows(initialRows);@@ -707,6 +717,13 @@ class TerminalSession extends ChangeNotifier {}return true;}+ // The daemon no longer holds this stream (it closed it while its link to the backend was+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now+ // rather than waiting out 15 s of silence and three resyncs (2026-10-16).+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));+ return true;+ }// A polite open, refused: another client is driving this terminal and// this open asked not to take it from them. Not a failure, and it must// not read as one — `error` is what the notifier's reattach sweep@@ -1489,7 +1506,11 @@ class TerminalSession extends ChangeNotifier {// the same gap — instead of declaring TERMINAL_STREAM_TIMEOUT and throwing away typed input.if (afterSleep) _lastStreamActivityAt = _now();final generation = _generation;- final sent = await send('terminal_alive', {'streamId': streamId});+ final sent = await send('terminal_alive', {+ 'streamId': streamId,+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).+ 'reportMissing': true,+ });if (!sent && _isCurrent(generation)) {transportLost('Terminal heartbeat was not sent');return;@@ -1548,6 +1569,7 @@ class TerminalSession extends ChangeNotifier {'streamId': currentStream,'attempt': _resyncAttempts,'reason': errorCode,+ 'reportMissing': true,});if (!_isCurrent(generation) ||streamId != currentStream ||@@ -1584,8 +1606,9 @@ class TerminalSession extends ChangeNotifier {return;}_autoReopenAttempts++;- debugPrint(- '[terminal-session] recovery_reopen agent=$agentId stream=$previousStream',+ appLog.info(+ 'terminal',+ 'reopen machine=$machineId agent=$agentId stream=$previousStream reason=$reason',);if (previousStream != null) {unawaited(send('terminal_close', {'streamId': previousStream}));@@ -1600,6 +1623,7 @@ class TerminalSession extends ChangeNotifier {}Future<void> close() async {+ closedHere = true;_generation++;final closingStream = streamId;_cancelTimers();@@ -1626,6 +1650,10 @@ class TerminalSession extends ChangeNotifier {status == TerminalSessionStatus.takenOver) {return;}+ appLog.warn(+ 'terminal',+ 'transport lost machine=$machineId agent=$agentId message=$message',+ );_generation++;_cancelTimers();_inputBytes.clear();@@ -1640,6 +1668,12 @@ class TerminalSession extends ChangeNotifier {void _fail(String code, String? message) {if (_disposed) return;+ // Why a pane shows Reconnect was only ever in debugPrint, invisible on a person's machine (2026-10-16).+ appLog.warn(+ 'terminal',+ 'failed machine=$machineId agent=$agentId stream=$streamId code=$code'+ '${message == null ? '' : ' message=$message'}',+ );_generation++;_cancelTimers();_inputBytes.clear();diff --git a/desktop/test/harness_attachments_test.dart b/desktop/test/harness_attachments_test.dartindex c5873be4f..e0ce6c5d0 100644--- a/desktop/test/harness_attachments_test.dart+++ b/desktop/test/harness_attachments_test.dart@@ -55,7 +55,7 @@ class _RecordingSession extends TerminalSession {/// "Take control": the daemon answers by handing this window the lease.@override- Future<void> reopen({bool force = false}) async {+ Future<void> reopen({bool force = false, bool redial = true}) async {calls.add('reopen:$force');watching = false;notifyListeners();diff --git a/desktop/test/pane_retry_test.dart b/desktop/test/pane_retry_test.dartnew file mode 100644index 000000000..fc1690f9f--- /dev/null+++ b/desktop/test/pane_retry_test.dart@@ -0,0 +1,267 @@+import 'dart:typed_data';++import 'package:flutter_test/flutter_test.dart';+import 'package:harness/logging/app_log.dart';+import 'package:harness/state/app_state.dart';+import 'package:harness/terminal/terminal_binary.dart';+import 'package:harness/terminal/terminal_session.dart';+import 'package:harness/ws/ws_conn.dart';++import 'swarm_state_test.dart' show createApp;++/// A current daemon with two agents that answers every request and never answers an open, so a+/// reattached session stays `opening` until the test breaks it again.+class _Connection extends WsConn {+ _Connection()+ : super(+ wsBaseUrl: '',+ autonomousEnv: 'test',+ machineId: 'm',+ accessTokenProvider: (_, _) async => '',+ onAuthFailure: (_) {},+ onEvent: (_) {},+ onStatus: (_) {},+ );+ final opened = <String>[];+ final opens = <Map<String, dynamic>>[];+ @override+ Future<void> waitUntilReady({required Duration timeout}) async {}+ @override+ Future<Map<String, dynamic>> request(+ String type, {+ Map<String, dynamic> payload = const {},+ Duration timeout = const Duration(seconds: 20),+ }) async => type == 'agents_list'+ ? {+ 'agents': [+ for (final id in ['a0', 'a1'])+ {+ 'id': id,+ 'name': id,+ 'engine': 'codex',+ 'terminal': {+ 'runtimes': [+ {'backend': 'tmux', 'paneId': id == 'a0' ? '%1' : '%2'},+ ],+ },+ },+ ],+ }+ : {+ 'protocolVersion': 3,+ 'backend': 'tmux',+ 'available': true,+ 'features': {'noTakeover': true},+ };+ @override+ Future<bool> sendTerminalFrame(+ String type,+ Map<String, dynamic> payload,+ ) async {+ if (type == 'terminal_open') {+ opened.add(payload['agentId'] as String);+ opens.add(payload);+ }+ return true;+ }+}++class _RecordingLog implements AppLog {+ final lines = <String>[];+ @override+ void record(+ AppLogLevel level,+ String category,+ String message, {+ Object? error,+ StackTrace? stackTrace,+ }) => lines.add('$category $message');+}++void main() {+ /// Two tabs, `a0` in the first (background) and `a1` in the second (in front), on a machine+ /// that has come back once, so its agents and terminal capability are loaded.+ Future<({AppNotifier app, _Connection connection, TerminalSession background})>+ rig(WidgetTester tester) async {+ final connection = _Connection();+ // Disposed at the end of each test body: it disposes the sessions too, whose open watchdogs+ // and this sweep's timers must not outlive the body.+ final app = createApp(connectionForTest: (_) => connection);+ app.machineStates['m']!.nodeOnline = true;+ TerminalSession terminal(String id) => TerminalSession(+ machineId: 'm',+ agentId: id,+ agentName: id,+ engineId: 'codex',+ send: connection.sendTerminalFrame,+ sendBinary: (_) async => true,+ // The fake daemon never answers an open; keep the open watchdog out of these timelines.+ resyncTimeout: const Duration(hours: 1),+ )..status = TerminalSessionStatus.controlling;+ final background = terminal('a0');+ app.adoptSessionForTest(background);+ app.newSwarm(name: 'Second');+ app.adoptSessionForTest(terminal('a1'));+ await app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': false},+ });+ await app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': true},+ });+ for (var i = 0; i < 30; i++) {+ await tester.pump(const Duration(milliseconds: 100));+ }+ connection.opened.clear();+ connection.opens.clear();+ return (app: app, connection: connection, background: background);+ }++ testWidgets(+ 'a pane whose stream died on a reachable machine is reattached without a click, behind another tab',+ (tester) async {+ final log = _RecordingLog();+ final previous = appLog;+ appLog = log;+ addTearDown(() => appLog = previous);+ final r = await rig(tester);++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 900));+ expect(r.connection.opened, isEmpty);+ await tester.pump(const Duration(milliseconds: 200));++ expect(r.connection.opened, ['a0']);+ expect(log.lines, contains('terminal reattach machine=m agent=a0 attempt=1'));+ r.app.dispose();+ },+ );++ testWidgets('each further attempt waits longer: 1 s, then 2 s', (tester) async {+ final r = await rig(tester);++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));+ expect(r.connection.opened, ['a0']);++ r.background.transportLost('gone again');+ await tester.pump(const Duration(milliseconds: 1100));+ expect(r.connection.opened, ['a0'], reason: 'the second wait is 2 s');+ await tester.pump(const Duration(seconds: 1));+ expect(r.connection.opened, ['a0', 'a0']);+ r.app.dispose();+ });++ testWidgets(+ 'a sweep reopen never takes the terminal, whatever the last open asked for',+ (tester) async {+ final r = await rig(tester);+ expect(r.background.takeover, isTrue, reason: 'a person opened it');++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));++ expect(r.connection.opens.single['takeover'], isFalse);+ r.app.dispose();+ },+ );++ testWidgets('a pane someone else took is never reattached', (tester) async {+ final r = await rig(tester);+ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));+ await r.app.handleEventForTest('m', {+ 'type': 'terminal_error',+ 'payload': {+ 'requestId': r.connection.opens.single['requestId'],+ 'code': 'CONTROL_LEASE_HELD',+ },+ });+ expect(r.background.status, TerminalSessionStatus.takenOver);++ await tester.pump(const Duration(seconds: 40));++ expect(r.connection.opens, hasLength(1));+ r.app.dispose();+ });++ testWidgets('a dead watcher is reattached, as a watcher', (tester) async {+ final r = await rig(tester);+ r.background+ ..watching = true+ ..takeover = false;++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));++ expect(r.connection.opens.single['takeover'], isFalse);+ r.app.dispose();+ });++ testWidgets('a pane that is live again keeps no retry going', (tester) async {+ final r = await rig(tester);+ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));+ await r.app.handleEventForTest('m', {+ 'type': 'terminal_ready',+ 'payload': {+ 'requestId': r.connection.opens.single['requestId'],+ 'protocolVersion': 3,+ 'streamId': '00112233-4455-6677-8899-aabbccddeeff',+ 'agentId': 'a0',+ },+ });+ await r.background.handleBinary(+ TerminalBinaryFrame(+ kind: TerminalBinaryKind.keyframe,+ streamId: '00112233-4455-6677-8899-aabbccddeeff',+ seq: 0,+ bytes: Uint8List.fromList('prompt> '.codeUnits),+ compressed: false,+ cols: 80,+ rows: 24,+ ),+ );+ expect(r.background.status, TerminalSessionStatus.controlling);++ await tester.pump(const Duration(seconds: 60));++ expect(r.connection.opens, hasLength(1));+ r.app.dispose();+ });++ testWidgets('a stream this app closed itself is left closed', (tester) async {+ final r = await rig(tester);++ await r.background.close();+ await tester.pump(const Duration(seconds: 3));++ expect(r.connection.opened, isEmpty);+ r.app.dispose();+ });++ testWidgets('nothing is retried while the machine is offline', (tester) async {+ final r = await rig(tester);++ await r.app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': false},+ });+ // The first retry would come after 1 s. Stay under the offline poll's 5 s tick, which reads the+ // live machine list (refused under test).+ await tester.pump(const Duration(seconds: 4));++ expect(r.connection.opened, isEmpty);++ // Back online, which stops the offline poll's timer before the test ends.+ await r.app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': true},+ });+ for (var i = 0; i < 30; i++) {+ await tester.pump(const Duration(milliseconds: 100));+ }+ r.app.dispose();+ });+}diff --git a/desktop/test/terminal_session_test.dart b/desktop/test/terminal_session_test.dartindex 9b16ab5e5..f6c1285ba 100644--- a/desktop/test/terminal_session_test.dart+++ b/desktop/test/terminal_session_test.dart@@ -3,6 +3,7 @@ import 'dart:convert';import 'dart:io';import 'dart:typed_data';+import 'package:harness/logging/app_log.dart';import 'package:harness/terminal/terminal_session.dart';import 'package:harness/terminal/terminal_binary.dart';import 'package:flutter_test/flutter_test.dart';@@ -1390,6 +1391,120 @@ void main() {expect(session.errorCode, 'TERMINAL_RESYNC_TIMEOUT');});+ testWidgets(+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',+ (tester) async {+ session.dispose();+ session = TerminalSession(+ machineId: 'machine-1',+ agentId: 'agent-1',+ agentName: 'backend-api',+ engineId: 'codex',+ resyncTimeout: const Duration(seconds: 30),+ send: (type, payload) async {+ sent.add((type: type, payload: Map<String, dynamic>.from(payload)));+ return true;+ },+ sendBinary: (frame) async => true,+ );+ await ready();+ await session.handleBinary(+ output(0, utf8.encode('screen'), keyframe: true, cols: 80, rows: 24),+ );+ await tester.pump(const Duration(seconds: 5));+ expect(+ sent.lastWhere((frame) => frame.type == 'terminal_alive').payload,+ {'streamId': streamId, 'reportMissing': true},+ );++ await session.handleBinary(output(2, utf8.encode('gap')));+ await tester.pump();+ expect(+ sent+ .firstWhere((frame) => frame.type == 'terminal_resync')+ .payload['reportMissing'],+ isTrue,+ );+ await session.close();+ },+ );++ test('a reopen that may not redial leaves the machine link alone', () async {+ var redials = 0;+ TerminalSession stalled() => TerminalSession(+ machineId: 'machine-1',+ agentId: 'agent-1',+ agentName: 'backend-api',+ engineId: 'codex',+ send: (type, payload) async => type != 'terminal_open',+ sendBinary: (frame) async => true,+ onOpenStalled: () async => redials++,+ );+ final quiet = stalled();+ addTearDown(quiet.dispose);+ await quiet.reopen(redial: false);+ expect(redials, 0);+ expect(quiet.status, TerminalSessionStatus.error);++ final loud = stalled();+ final reopening = loud.reopen();+ await pumpEventQueue();+ expect(redials, 1);+ loud.dispose();+ await reopening;+ });++ test('a stream the daemon no longer has is reopened in place at once', () async {+ await ready();+ await session.handleBinary(+ output(0, utf8.encode('screen'), keyframe: true, cols: 100, rows: 30),+ );+ await session.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ 'message': 'no live terminal stream',+ });+ await pumpEventQueue();++ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(2));+ expect(session.status, TerminalSessionStatus.opening);+ expect(session.errorCode, isNot('TERMINAL_STREAM_NOT_FOUND'));+ });++ test('a missing-stream report about another stream is ignored', () async {+ await ready();+ await session.handleFrame('terminal_error', {+ 'streamId': 'ffffffff-ffff-ffff-ffff-ffffffffffff',+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });+ await pumpEventQueue();++ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(1));+ expect(session.status, isNot(TerminalSessionStatus.error));+ });++ test('a failure is written to the file log with its code', () async {+ final log = _RecordingLog();+ final previous = appLog;+ appLog = log;+ addTearDown(() => appLog = previous);+ await ready();++ await session.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_SNAPSHOT_FAILED',+ 'message': 'tmux snapshot failed',+ });++ expect(+ log.lines,+ contains(+ 'terminal failed machine=machine-1 agent=agent-1 stream=$streamId '+ 'code=TERMINAL_SNAPSHOT_FAILED message=tmux snapshot failed',+ ),+ );+ });+group('composer', () {/// The `message` frames this session put on the wire.List<Map<String, dynamic>> messages() => [@@ -1892,3 +2007,15 @@ void main() {);});}++class _RecordingLog implements AppLog {+ final lines = <String>[];+ @override+ void record(+ AppLogLevel level,+ String category,+ String message, {+ Object? error,+ StackTrace? stackTrace,+ }) => lines.add('$category $message');+}diff --git a/docs/plans/2026-10-16-terminal-auto-recovery-implementation.md b/docs/plans/2026-10-16-terminal-auto-recovery-implementation.mdnew file mode 100644index 000000000..6d524b948--- /dev/null+++ b/docs/plans/2026-10-16-terminal-auto-recovery-implementation.md@@ -0,0 +1,797 @@+# Terminal Auto-Recovery Implementation Plan++> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.++**Goal:** Remote terminal panes recover by themselves after their stream dies while the machine stays reachable, and the desktop's file log says why a pane failed.++**Architecture:** Clients that opt in with `reportMissing: true` get an explicit `TERMINAL_STREAM_NOT_FOUND` from the host when their heartbeat or resync names a stream the host no longer holds, and reopen at once. The desktop `AppNotifier` reattaches any `error`/`closed` pane on a reachable machine on a per-pane backoff, whichever tab it is in. `TerminalSession` failures and recoveries go to `appLog`.++**Tech Stack:** TypeScript + vitest (CLI daemon); Dart/Flutter + flutter_test, fake_async (desktop, mobile).++**Spec:** `docs/plans/2026-10-16-terminal-auto-recovery.md`++## Global Constraints++- No backend change. No new frame type, and no change to the E2EE type sets in `cli/src/lib/e2ee/core.ts`.+- Opt-in only: the host answers only frames that carry `reportMissing: true`. Clients without the flag get exactly today's silence.+- Answer shape: `terminal_error { streamId, code: 'TERMINAL_STREAM_NOT_FOUND', message: 'no live terminal stream' }`.+- Desktop retry backoff: 1, 2, 4, 8, 16, then every 30 s. Reset when the pane reaches `controlling`. No retry while the machine is offline. Never retry `takenOver` or watching panes. The reattach is a polite open (`AttachIntent.automatic`).+- The desktop sweep's timers exist only while some pane needs a retry. An idle app runs none.+- Mobile gets the session changes only (reportMissing and reopen on not-found), and no sweep.+- Logs carry machine, agent, stream, code and message only: never terminal content or paths.+- Do not commit unless the owner asks. Comments explain why and name the incident (2026-10-16).++## Review Focus++1. **Retry storm against a machine that refuses every open.** The backoff must cap at 30 s per pane. Many broken panes must not multiply the timers beyond one per pane. Task 3 asserts the 1 s → 2 s spacing, and the cap follows from the same table.+2. **The sweep fighting another client.** A `takenOver` pane, or a session that is `watching`, must never be reopened automatically (`_paneNeedsAttach` already excludes them). Task 3 relies on `_paneNeedsAttach`, and the reviewer should confirm no new path bypasses it.+3. **Timers left alive after sign-out or dispose.** `testWidgets` fails on a pending timer, so Task 3's tests prove dispose. Sign-out goes through `_stopAllOfflineRetries`, which Task 3 extends.+4. **An old client meeting the new host, or the reverse.** The host must stay silent without the flag (Task 1 test). A new client must still work against an old host: no reply arrives, and the sweep still recovers the pane.+5. **A NOT_FOUND for another stream, or one arriving while a reopen is in flight.** It must be ignored. Task 2 and Task 4 test the other-stream case. The in-flight case is covered by the existing `errorStream != streamId` filter, because `streamId` is null while opening.++---++### Task 1: Host reports a missing stream to clients that ask (CLI)++**Files:**+- Modify: `cli/src/lib/terminalStreamManager.ts` (`alive()` ~line 640, `resyncRequested()` ~line 1011, add `reportMissing()` beside them)+- Test: `cli/src/lib/terminalStreamManager.spec.ts`++**Interfaces:**+- Produces: the host answers `terminal_error` with `code: 'TERMINAL_STREAM_NOT_FOUND'` and the named `streamId`, but only when the incoming `terminal_alive`/`terminal_resync` payload has `reportMissing: true`. Tasks 2 and 4 consume this.++- [ ] **Step 1: Write the failing tests**++Add inside `describe('TerminalStreamManager', …)` in `cli/src/lib/terminalStreamManager.spec.ts`, after the test "takes over the first controller and expires the replacement lease after heartbeat timeout":++```ts+ // A host that lost its backend link closes its remote streams without telling anyone; the client+ // only found out after 15 s of silence and three ignored resyncs, then gave up (2026-10-16). A+ // client that asks is now told at its next heartbeat.+ describe('a heartbeat or resync for a stream this connection does not hold', () => {+ const errors = () => sent.filter((frame) => frame.type === 'terminal_error')++ it('is answered with TERMINAL_STREAM_NOT_FOUND when the client asks', async () => {+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1', reportMissing: true })+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-2', attempt: 1, reportMissing: true })+ expect(errors().map((frame) => [frame.connId, frame.payload.code, frame.payload.streamId])).toEqual([+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-1'],+ ['web-1', 'TERMINAL_STREAM_NOT_FOUND', 'gone-2'],+ ])+ })++ it('stays silent for a client that does not ask, as every older client', async () => {+ await manager.handleFrame('web-1', 'terminal_alive', { streamId: 'gone-1' })+ await manager.handleFrame('web-1', 'terminal_resync', { streamId: 'gone-1', attempt: 1 })+ expect(errors()).toEqual([])+ })++ it('counts a stream held by another connection as missing for this one, and leaves it alone', async () => {+ await manager.handleFrame('web-1', 'terminal_open', {+ requestId: 'open-1', protocolVersion: 3, agentId: 'agent-1', cols: 100, rows: 30,+ })+ const streamId = sent.findLast((frame) => frame.type === 'terminal_ready')?.payload.streamId as string+ await manager.handleFrame('web-2', 'terminal_alive', { streamId, reportMissing: true })+ expect(errors().at(-1)).toMatchObject({+ connId: 'web-2',+ payload: { code: 'TERMINAL_STREAM_NOT_FOUND', streamId },+ })+ expect(stream.closed).toBe(false)+ })+ })+```++- [ ] **Step 2: Run the tests to verify they fail**++Run: `cd cli && npx vitest run src/lib/terminalStreamManager.spec.ts -t "does not hold"`+Expected: FAIL. The first and third tests find no `terminal_error`: `expected [] to deeply equal [...]`, and `toMatchObject` against `undefined`. The second test passes; it pins today's behaviour.++- [ ] **Step 3: Implement**++In `cli/src/lib/terminalStreamManager.ts`, replace `alive()`:++```ts+ private alive(connId: string, payload: FramePayload): void {+ const state = this.streamFor(connId, payload)+ if (!state) { this.reportMissing(connId, payload); return }+ state.expiresAt = this.now() + HEARTBEAT_TIMEOUT_MS+ }++ /**+ * Tell a client that asked (`reportMissing`) that the stream it names is gone. A host that lost its+ * backend link closes its remote streams without notice, and the client used to learn it only after+ * 15 s of silence and three resyncs this manager ignored, then gave up on the pane (2026-10-16). Opt-in,+ * so an older client keeps the silence its own recovery was built around.+ */+ private reportMissing(connId: string, payload: FramePayload): void {+ if (payload.reportMissing !== true || typeof payload.streamId !== 'string') return+ this.sendError(connId, 'TERMINAL_STREAM_NOT_FOUND', { streamId: payload.streamId, message: 'no live terminal stream' })+ }+```++and replace `resyncRequested()`:++```ts+ private async resyncRequested(connId: string, payload: FramePayload): Promise<void> {+ const state = this.streamFor(connId, payload)+ if (!state) { this.reportMissing(connId, payload); return }+ state.resyncs+++ this.diagnostic(state, 'resync_requested', { attempt: payload.attempt })+ await this.sendKeyframe(state, 'resync')+ }+```++- [ ] **Step 4: Run the tests to verify they pass**++Run: `cd cli && npx vitest run src/lib/terminalStreamManager.spec.ts`+Expected: PASS, the whole file (new tests plus every existing one).++- [ ] **Step 5: Typecheck**++Run: `cd cli && npm run typecheck`+Expected: no errors.++---++### Task 2: Desktop session asks, reopens on the answer, and logs failures++**Files:**+- Modify: `desktop/lib/terminal/terminal_session.dart`. That means the imports, the `terminal_error` branch (~line 659–735), `_sendHeartbeat` (~1492), `_sendResyncAttempt` (~1547), `_recoverByReopen` (~1569), `transportLost` and `_fail` (~1612–1650).+- Test: `desktop/test/terminal_session_test.dart`++**Interfaces:**+- Consumes: Task 1's `TERMINAL_STREAM_NOT_FOUND` answer.+- Produces: `terminal_alive` and `terminal_resync` payloads carry `'reportMissing': true`. `appLog` lines in category `terminal` start with `failed `, `transport lost `, or `reopen `. Task 3 adds `reattach `.++- [ ] **Step 1: Write the failing tests**++In `desktop/test/terminal_session_test.dart`, add `import 'package:harness/logging/app_log.dart';` to the imports. Add this class after `main()`'s closing brace:++```dart+class _RecordingLog implements AppLog {+ final lines = <String>[];+ @override+ void record(+ AppLogLevel level,+ String category,+ String message, {+ Object? error,+ StackTrace? stackTrace,+ }) => lines.add('$category $message');+}+```++Add these tests inside `main()`, after the test "resync retries three times, reopens once, then fails closed":++```dart+ testWidgets(+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',+ (tester) async {+ session.dispose();+ session = TerminalSession(+ machineId: 'machine-1',+ agentId: 'agent-1',+ agentName: 'backend-api',+ engineId: 'codex',+ resyncTimeout: const Duration(seconds: 30),+ send: (type, payload) async {+ sent.add((type: type, payload: Map<String, dynamic>.from(payload)));+ return true;+ },+ sendBinary: (frame) async => true,+ );+ await ready();+ await session.handleBinary(+ output(0, utf8.encode('screen'), keyframe: true, cols: 80, rows: 24),+ );+ await tester.pump(const Duration(seconds: 5));+ expect(+ sent.lastWhere((frame) => frame.type == 'terminal_alive').payload,+ {'streamId': streamId, 'reportMissing': true},+ );++ await session.handleBinary(output(2, utf8.encode('gap')));+ await tester.pump();+ expect(+ sent.firstWhere((frame) => frame.type == 'terminal_resync').payload['reportMissing'],+ isTrue,+ );+ await session.close();+ },+ );++ test('a stream the daemon no longer has is reopened in place at once', () async {+ await ready();+ await session.handleBinary(+ output(0, utf8.encode('screen'), keyframe: true, cols: 100, rows: 30),+ );+ await session.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ 'message': 'no live terminal stream',+ });+ await pumpEventQueue();++ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(2));+ expect(session.status, TerminalSessionStatus.opening);+ expect(session.errorCode, isNot('TERMINAL_STREAM_NOT_FOUND'));+ });++ test('a missing-stream report about another stream is ignored', () async {+ await ready();+ await session.handleFrame('terminal_error', {+ 'streamId': 'ffffffff-ffff-ffff-ffff-ffffffffffff',+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });+ await pumpEventQueue();++ expect(sent.where((frame) => frame.type == 'terminal_open'), hasLength(1));+ expect(session.status, isNot(TerminalSessionStatus.error));+ });++ test('a failure is written to the file log with its code', () async {+ final log = _RecordingLog();+ final previous = appLog;+ appLog = log;+ addTearDown(() => appLog = previous);+ await ready();++ await session.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_SNAPSHOT_FAILED',+ 'message': 'tmux snapshot failed',+ });++ expect(+ log.lines,+ contains(+ 'terminal failed machine=machine-1 agent=agent-1 stream=$streamId '+ 'code=TERMINAL_SNAPSHOT_FAILED message=tmux snapshot failed',+ ),+ );+ });+```++- [ ] **Step 2: Run the tests to verify they fail**++Run: `cd desktop && flutter test test/terminal_session_test.dart --plain-name "daemon"`, then `--plain-name "missing-stream"` and `--plain-name "file log"`.+Expected:+- The "heartbeats and resyncs ask…" test fails: the payload is `{streamId: …}` without `reportMissing`.+- "reopened in place at once" fails: one `terminal_open`, status `error`.+- "file log" fails: `log.lines` is empty.+- "another stream" passes, because it pins the existing filter.++- [ ] **Step 3: Implement**++In `desktop/lib/terminal/terminal_session.dart`:++1. Add after `import '../core/sleep_aware.dart';`:++```dart+import '../logging/app_log.dart';+```++2. In `case 'terminal_error':`, insert directly before the `if (errorCode == 'CONTROL_LEASE_HELD' && !_openAskedTakeover) {` block:++```dart+ // The daemon no longer holds this stream (it closed it while its link to the backend was+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now+ // rather than waiting out 15 s of silence and three resyncs (2026-10-16).+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));+ return true;+ }+```++3. In `_sendHeartbeat`, replace `final sent = await send('terminal_alive', {'streamId': streamId});` with:++```dart+ final sent = await send('terminal_alive', {+ 'streamId': streamId,+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).+ 'reportMissing': true,+ });+```++4. In `_sendResyncAttempt`, add `'reportMissing': true,` to the `terminal_resync` payload, after `'reason': errorCode,`.++5. In `_recoverByReopen`, replace the `debugPrint('[terminal-session] recovery_reopen agent=$agentId stream=$previousStream');` call with:++```dart+ appLog.info(+ 'terminal',+ 'reopen machine=$machineId agent=$agentId stream=$previousStream reason=$reason',+ );+```++6. In `transportLost`, insert directly after the early `return;` guard block (before `_generation++;`):++```dart+ appLog.warn(+ 'terminal',+ 'transport lost machine=$machineId agent=$agentId message=$message',+ );+```++7. In `_fail`, insert directly after `if (_disposed) return;`:++```dart+ // Why a pane shows Reconnect was only ever in debugPrint, invisible on a person's machine (2026-10-16).+ appLog.warn(+ 'terminal',+ 'failed machine=$machineId agent=$agentId stream=$streamId code=$code'+ '${message == null ? '' : ' message=$message'}',+ );+```++- [ ] **Step 4: Run the tests to verify they pass**++Run: `cd desktop && flutter test test/terminal_session_test.dart`+Expected: PASS, the whole file. An existing test that asserted the exact `terminal_alive` payload would need `'reportMissing': true` added. That would be a plan deviation: record it as a ruling.++- [ ] **Step 5: Analyze**++Run: `cd desktop && flutter analyze lib/terminal/terminal_session.dart test/terminal_session_test.dart`+Expected: No issues found.++---++### Task 3: Desktop reattaches broken panes on a reachable machine, with backoff++**Files:**+- Modify: `desktop/lib/state/app_state.dart`:+ - near `offlineRetryInterval` (~line 3486): add `paneRetryDelays`;+ - near `_offlineRetryTimers` (~969): add the two maps;+ - `_attachSession` (~15618) and `_detachSession` (~15637): swap the listener;+ - end of `_attachPendingPanes` (~17183): call the scheduler;+ - `_stopAllOfflineRetries` (~7830): cancel the retries;+ - `adoptSessionForTest` (~18441): swap the listener;+ - add new methods next to `_paneNeedsAttach` (~17193).+- Create: `desktop/test/pane_retry_test.dart`++**Interfaces:**+- Consumes: `_paneNeedsAttach`, `_canAttachPane`, `_reattachPane(pane, intent: AttachIntent.automatic)`, `MachineState.nodeOnline`, `MachineState.terminalNoTakeoverAvailable` (all existing). Task 2's `appLog` category `terminal`.+- Produces: `static const List<Duration> paneRetryDelays` on `AppNotifier`, and the log line `terminal reattach machine=<id> agent=<id> attempt=<n>`.++- [ ] **Step 1: Write the failing tests**++Create `desktop/test/pane_retry_test.dart`:++```dart+import 'package:flutter_test/flutter_test.dart';+import 'package:harness/logging/app_log.dart';+import 'package:harness/state/app_state.dart';+import 'package:harness/terminal/terminal_session.dart';+import 'package:harness/ws/ws_conn.dart';++import 'swarm_state_test.dart' show createApp;++/// A current daemon with two agents that answers every request and never answers an open, so a+/// reattached session stays `opening` until the test breaks it again.+class _Connection extends WsConn {+ _Connection()+ : super(+ wsBaseUrl: '',+ autonomousEnv: 'test',+ machineId: 'm',+ accessTokenProvider: (_, _) async => '',+ onAuthFailure: (_) {},+ onEvent: (_) {},+ onStatus: (_) {},+ );+ final opened = <String>[];+ @override+ Future<void> waitUntilReady({required Duration timeout}) async {}+ @override+ Future<Map<String, dynamic>> request(+ String type, {+ Map<String, dynamic> payload = const {},+ Duration timeout = const Duration(seconds: 20),+ }) async => type == 'agents_list'+ ? {+ 'agents': [+ for (final id in ['a0', 'a1'])+ {+ 'id': id,+ 'name': id,+ 'engine': 'codex',+ 'terminal': {+ 'runtimes': [+ {'backend': 'tmux', 'paneId': id == 'a0' ? '%1' : '%2'},+ ],+ },+ },+ ],+ }+ : {+ 'protocolVersion': 3,+ 'backend': 'tmux',+ 'available': true,+ 'features': {'noTakeover': true},+ };+ @override+ Future<bool> sendTerminalFrame(+ String type,+ Map<String, dynamic> payload,+ ) async {+ if (type == 'terminal_open') opened.add(payload['agentId'] as String);+ return true;+ }+}++class _RecordingLog implements AppLog {+ final lines = <String>[];+ @override+ void record(+ AppLogLevel level,+ String category,+ String message, {+ Object? error,+ StackTrace? stackTrace,+ }) => lines.add('$category $message');+}++void main() {+ /// Two tabs, `a0` in the first (background) and `a1` in the second (in front), on a machine+ /// that has come back once, so its agents and terminal capability are loaded.+ Future<({AppNotifier app, _Connection connection, TerminalSession background})>+ rig(WidgetTester tester) async {+ final connection = _Connection();+ final app = createApp(connectionForTest: (_) => connection);+ addTearDown(app.dispose);+ app.machineStates['m']!.nodeOnline = true;+ TerminalSession terminal(String id) => TerminalSession(+ machineId: 'm',+ agentId: id,+ agentName: id,+ engineId: 'codex',+ send: connection.sendTerminalFrame,+ sendBinary: (_) async => true,+ )..status = TerminalSessionStatus.controlling;+ final background = terminal('a0');+ app.adoptSessionForTest(background);+ app.newSwarm(name: 'Second');+ app.adoptSessionForTest(terminal('a1'));+ await app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': false},+ });+ await app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': true},+ });+ for (var i = 0; i < 30; i++) {+ await tester.pump(const Duration(milliseconds: 100));+ }+ connection.opened.clear();+ return (app: app, connection: connection, background: background);+ }++ testWidgets(+ 'a pane whose stream died on a reachable machine is reattached without a click, behind another tab',+ (tester) async {+ final log = _RecordingLog();+ final previous = appLog;+ appLog = log;+ addTearDown(() => appLog = previous);+ final r = await rig(tester);++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 900));+ expect(r.connection.opened, isEmpty);+ await tester.pump(const Duration(milliseconds: 200));++ expect(r.connection.opened, ['a0']);+ expect(log.lines, contains('terminal reattach machine=m agent=a0 attempt=1'));+ for (final pane in r.app.allPanes) {+ await pane.session?.close();+ }+ },+ );++ testWidgets('each further attempt waits longer: 1 s, then 2 s', (tester) async {+ final r = await rig(tester);++ r.background.transportLost('gone');+ await tester.pump(const Duration(milliseconds: 1100));+ expect(r.connection.opened, ['a0']);++ r.background.transportLost('gone again');+ await tester.pump(const Duration(milliseconds: 1100));+ expect(r.connection.opened, ['a0'], reason: 'the second wait is 2 s');+ await tester.pump(const Duration(seconds: 1));+ expect(r.connection.opened, ['a0', 'a0']);+ for (final pane in r.app.allPanes) {+ await pane.session?.close();+ }+ });++ testWidgets('nothing is retried while the machine is offline', (tester) async {+ final r = await rig(tester);++ await r.app.handleEventForTest('m', {+ 'type': 'node_status',+ 'payload': {'online': false},+ });+ await tester.pump(const Duration(seconds: 40));++ expect(r.connection.opened, isEmpty);+ for (final pane in r.app.allPanes) {+ await pane.session?.close();+ }+ });+}+```++`AppNotifier` is assumed to be exported from `package:harness/state/app_state.dart`; `swarm_reconnect_test.dart` uses `createApp` from the same file. If `createApp`'s return type lives elsewhere, import what `swarm_state_test.dart` imports.++- [ ] **Step 2: Run the tests to verify they fail**++Run: `cd desktop && flutter test test/pane_retry_test.dart`+Expected: the first two tests FAIL with `expected ['a0'] … actual []`, because nothing reattaches the background pane. The offline test passes, since it pins the boundary.++- [ ] **Step 3: Implement**++In `desktop/lib/state/app_state.dart`:++1. After `static const offlineRetryInterval = Duration(seconds: 5);` add:++```dart+ /// Waits before each automatic reattach of a pane whose stream died while its machine stayed+ /// reachable; the last one repeats. A pane used to get one reopen and then sit on Reconnect until+ /// someone clicked it, in whatever tab it was (2026-10-16).+ static const paneRetryDelays = <Duration>[+ Duration(seconds: 1),+ Duration(seconds: 2),+ Duration(seconds: 4),+ Duration(seconds: 8),+ Duration(seconds: 16),+ Duration(seconds: 30),+ ];+```++2. After `final Map<String, Timer> _offlineRetryTimers = {};` add:++```dart+ final Map<TerminalPane, Timer> _paneRetryTimers = {};+ final Map<TerminalPane, int> _paneRetryAttempts = {};+```++3. In `_attachSession`, replace `terminal.addListener(notifyListeners);` with `terminal.addListener(_onSessionChanged);`. In `_detachSession`, replace `terminal.removeListener(notifyListeners);` with `terminal.removeListener(_onSessionChanged);`. In `adoptSessionForTest`, replace `session.addListener(notifyListeners);` with `session.addListener(_onSessionChanged);`.++4. At the end of `_attachPendingPanes` (after its `for` loop), add:++```dart+ // A load is also when a pane that could not be attached before (agents not listed yet) can be.+ _schedulePaneRetries();+```++5. Directly before `bool _paneNeedsAttach(TerminalPane pane) {`, add:++```dart+ /// Every session change repaints, and may mean a pane's stream just died.+ void _onSessionChanged() {+ notifyListeners();+ _schedulePaneRetries();+ }++ /// Keep one timer for each pane whose stream died while its machine stays reachable, in whichever+ /// tab it is, and none for any other. Sessions notify on state changes, not on output, so the walk+ /// is cheap; an app with every terminal live holds no timer.+ void _schedulePaneRetries() {+ if (_disposed) return;+ for (final pane in allPanes) {+ if (!_paneRetryWanted(pane)) {+ _paneRetryTimers.remove(pane)?.cancel();+ if (pane.session?.status == TerminalSessionStatus.controlling) {+ _paneRetryAttempts.remove(pane);+ }+ continue;+ }+ if (_paneRetryTimers.containsKey(pane)) continue;+ final attempts = _paneRetryAttempts[pane] ?? 0;+ final delay = paneRetryDelays[min(attempts, paneRetryDelays.length - 1)];+ _paneRetryTimers[pane] = Timer(delay, () {+ _paneRetryTimers.remove(pane);+ if (!_paneRetryWanted(pane)) return;+ _paneRetryAttempts[pane] = attempts + 1;+ appLog.info(+ 'terminal',+ 'reattach machine=${pane.machineId} agent=${pane.agentId} attempt=${attempts + 1}',+ );+ unawaited(_reattachPane(pane, intent: AttachIntent.automatic));+ });+ }+ _paneRetryTimers.removeWhere((pane, timer) {+ if (allPanes.contains(pane)) return false;+ timer.cancel();+ return true;+ });+ _paneRetryAttempts.removeWhere((pane, _) => !allPanes.contains(pane));+ }++ /// A dead stream on a machine that is up and can open terminals politely. Offline machines are left+ /// to the offline/online path, and `takenOver`/watching panes to a person (see [_paneNeedsAttach]).+ bool _paneRetryWanted(TerminalPane pane) {+ final machine = machineStates[pane.machineId];+ return pane.session != null &&+ _paneNeedsAttach(pane) &&+ machine != null &&+ machine.nodeOnline == true &&+ machine.terminalNoTakeoverAvailable &&+ _canAttachPane(pane);+ }++ void _cancelPaneRetries() {+ for (final timer in _paneRetryTimers.values) {+ timer.cancel();+ }+ _paneRetryTimers.clear();+ _paneRetryAttempts.clear();+ }+```++6. At the end of `_stopAllOfflineRetries()`, add `_cancelPaneRetries();`. It is called from sign-out (~line 5882) and from `dispose()` (~18549).++- [ ] **Step 4: Run the tests to verify they pass**++Run: `cd desktop && flutter test test/pane_retry_test.dart test/swarm_reconnect_test.dart test/retake_taken_over_panes_test.dart test/attach_intent_test.dart`+Expected: PASS, all four files. The last three guard the existing reattach and takeover rules.++- [ ] **Step 5: Analyze and run the bounded desktop suite**++Run: `cd desktop && flutter analyze lib/state/app_state.dart test/pane_retry_test.dart && make -C .. desktop-test`+Expected: no analyzer issues. The suite matches the documented baseline: any failure must also fail on `origin/main`, checked by name.++---++### Task 4: Mobile session asks and reopens on the answer++**Files:**+- Modify: `mobile/lib/terminal/terminal_session.dart`: the `terminal_error` branch (~line 968–1052), `terminal_alive` send (~2098), `terminal_resync` payload (~2175).+- Test: `mobile/test/terminal_session_protocol_test.dart`++**Interfaces:**+- Consumes: Task 1's answer. Mirrors Task 2's payload flag and reopen. Mobile gets no logging and no sweep (spec part 3).++- [ ] **Step 1: Write the failing tests**++Add inside `main()` of `mobile/test/terminal_session_protocol_test.dart`, after the test "a second answer to the same open leaves one heartbeat, and none after":++```dart+ test('heartbeats and resyncs ask the daemon to report a stream it no longer has', () {+ fakeAsync((async) {+ final s = session(disposeAtEnd: false, resync: const Duration(seconds: 30));+ s.open();+ async.flushMicrotasks();+ final streamId = _streamId();+ s.handleFrame('terminal_ready', {+ 'requestId': wire.last('terminal_open')['requestId'],+ 'agentId': 'a',+ 'protocolVersion': TerminalSession.protocolVersion,+ 'streamId': streamId,+ });+ s.handleBinary(_keyframe(streamId));+ async.flushMicrotasks();++ async.elapse(const Duration(seconds: 5));+ expect(wire.last('terminal_alive'), {'streamId': streamId, 'reportMissing': true});++ s.handleBinary(_output(streamId, 2, 'gap'));+ async.flushMicrotasks();+ expect(wire.last('terminal_resync')['reportMissing'], isTrue);+ s.dispose();+ });+ });++ test('a stream the daemon no longer has is reopened in place at once', () async {+ final s = session();+ final opening = s.open();+ final streamId = await ready(s);+ await opening;+ await s.handleBinary(_keyframe(streamId));++ await s.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });+ await until(() => wire.of('terminal_open').length == 2);++ expect(s.status, TerminalSessionStatus.opening);+ });++ test('a missing-stream report about another stream is ignored', () async {+ final s = session();+ final opening = s.open();+ await ready(s);+ await opening;++ await s.handleFrame('terminal_error', {+ 'streamId': _streamId(),+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });++ expect(wire.of('terminal_open'), hasLength(1));+ expect(s.status, isNot(TerminalSessionStatus.error));+ });+```++Check `_output`'s exact signature at `test/terminal_session_protocol_test.dart:1943` (`_output(String streamId, int seq, String? text, {…})`), and the shape of `ready()` and `open()` that the neighbouring tests use. Match them rather than this plan if they differ, and record a ruling.++- [ ] **Step 2: Run the tests to verify they fail**++Run: `cd mobile && flutter test test/terminal_session_protocol_test.dart --plain-name "daemon"`, then `--plain-name "reopened in place at once"`.+Expected:+- The heartbeat test fails: `reportMissing` is missing.+- "reopened in place" fails: `never got there`, because the session `_fail`s instead.+- The other-stream test passes.++- [ ] **Step 3: Implement**++In `mobile/lib/terminal/terminal_session.dart`:++1. In `case 'terminal_error':`, insert directly before the final `_fail(errorCode, payload['message']?.toString());`:++```dart+ // The daemon no longer holds this stream (it closed it while its link to the backend was+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now+ // rather than waiting out the heartbeat silence and three resyncs (2026-10-16).+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));+ return true;+ }+```++2. Replace `final sent = await send('terminal_alive', {'streamId': streamId});` with:++```dart+ final sent = await send('terminal_alive', {+ 'streamId': streamId,+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).+ 'reportMissing': true,+ });+```++3. Add `'reportMissing': true,` to the `terminal_resync` payload, after `'reason': errorCode,`.++- [ ] **Step 4: Run the tests to verify they pass**++Run: `cd mobile && flutter test test/terminal_session_protocol_test.dart test/app_state_terminals_test.dart`+Expected: PASS, both files. `app_state_terminals_test.dart` guards the pager's "not reopened by a guess" rule.++- [ ] **Step 5: Analyze**++Run: `cd mobile && flutter analyze lib/terminal/terminal_session.dart test/terminal_session_protocol_test.dart`+Expected: No issues found.++---++### Task 5: End to end on the local rig++**Files:** none changed. Results go into the PR description.++**Interfaces:**+- Consumes: Tasks 1–3 built and installed: the dev CLI on this Mac and on both docker boxes, and the desktop app from this branch.++- [ ] **Step 1: Baseline on `origin/main` builds**++With the released CLI and desktop, open three panes on `machine-remote-1` across two tabs, each showing steady output (`yes | head -c 100M | pv -qL 20k`, or an agent working). For each scenario below, record whether each pane comes back without a click, and how long it takes:+1. `docker exec machine-remote-1 sh -lc 'harness stop && harness start'`+2. Stop the local backend for 20 s, then start it again.+3. On this Mac, drop only the viewer's relay sockets: `pkill -f harnessd-gateway`. The master restarts the gateway process.++- [ ] **Step 2: Same scenarios on this branch's builds**++Install the dev CLI on the Mac and both boxes (`docker cp` recipe) and run the desktop from this branch. Repeat the three scenarios.+Expected: every pane returns to `controlling` with no click, within a few seconds of the machine being reachable again. `~/.harness/logs/app-*.log` shows `terminal failed …` or `terminal transport lost …`, then `terminal reattach …` or `terminal reopen …` for each pane.++- [ ] **Step 3: Record**++Put the before and after table (scenario × pane × recovered? × seconds) in the PR description, with the commands used.++---++## After the tasks (the owner's call)++- The changes span CLI, desktop and mobile. Run `npm run test:core` only if `src/core` wiring changed (it should not). Record the local runs in the PR, since PR CI runs only the Linux unit checks.+- Release order does not matter (see the spec's compatibility table): the CLI self-updates, and desktop and mobile ship with their next releases.diff --git a/docs/plans/2026-10-16-terminal-auto-recovery.md b/docs/plans/2026-10-16-terminal-auto-recovery.mdnew file mode 100644index 000000000..555797808--- /dev/null+++ b/docs/plans/2026-10-16-terminal-auto-recovery.md@@ -0,0 +1,150 @@+# Remote terminals recover by themselves: find dead streams fast, keep retrying, log why++Requested 2026-10-16. Phase 2 of the "remote terminal keeps showing Reconnect while the network is+fine" investigation; Phase 1 (relay rate budget) shipped in #1365 and #1367+([spec](2026-10-16-terminal-relay-rate-budget.md)). This spec also carries Phase 3 (failure reasons in+the desktop's file log), because Phase 2 cannot be verified on someone else's machine without it.++Scope: the CLI's terminal stream manager (host side), `TerminalSession` and `AppNotifier` in the desktop+(the web build shares this code) and in mobile (its own copies). No backend change, no new frame type,+no E2EE type-set change.++## Who hits it++People with many harnesses across several remote machines and tabs. Every pane is a separate chance to+fall into one of the gaps below, so the person with the most panes sees Reconnect most. A person with a+few panes on direct p2p rarely does.++## What happens today++When a host loses its backend link (a backend deploy, which happened about every four hours on+2026-10-15/16, a network blip, sleep), the host closes every remote terminal stream without telling+anyone (`backendSocket.ts` `linkDown`, `notify=false`). The backend immediately publishes+`node_status {online:false}` (`role.detach()`). The desktop marks the machine's panes offline and,+when the machine comes back, reloads its agents and reattaches every pane in every tab (polling up to+~10 s for the agents to be ready). That path mostly works. The 2026-10-15 14:07 desktop log shows+panes back four seconds after the drop.++Panes are left on Reconnect by three gaps:++- **(a) The host comes back after the recovery window.** Adapter backoff reaches 16–30 s, or the CLI is+ restarting for an update. A pane whose reattach missed its window stays `error` until the machine's+ next status change, which may never come.+- **(b) A stream dies while the machine stays online, as far as the desktop can tell.** Examples: the+ viewer's relay socket is replaced, a p2p demotion races a close, or the host's core restarts while+ its link stays up. The desktop notices only after 15 s of silence (`TERMINAL_STREAM_TIMEOUT`), then+ sends three resyncs 4 s apart. The host ignores a resync, ack or alive for a stream it does not have+ (`streamFor` returns null and the handlers return quietly), so the desktop reopens once, and if that+ fails it calls `_fail`. After `_fail`, nothing retries: the reattach sweep runs only on machine+ events or a tab switch.+- **(c) A polite reopen meets the old stream's control lease** (`CONTROL_LEASE_HELD`), and the pane+ becomes `takenOver`, which is never retried automatically. Out of scope here (see below).++## Change++### 1. Host: report a missing stream to clients that ask (CLI)++In `cli/src/lib/terminalStreamManager.ts`, `alive()` and `resyncRequested()` answer a frame for a+stream this connection does not own, but only when the frame carries `reportMissing: true`:++```+terminal_error { streamId, code: 'TERMINAL_STREAM_NOT_FOUND', message: 'no live terminal stream' }+```++`TERMINAL_STREAM_NOT_FOUND` is the code the chunked-upload path already sends for the same condition.+Clients that do not send the flag (every client shipped before this change) get exactly today's+silence, so an older desktop keeps its one-reopen behaviour instead of failing sooner. The answer+travels like any other terminal frame over relay or p2p, so it needs no new frame type or E2EE type.++### 2. Clients: ask, and reopen on the answer (desktop + mobile `TerminalSession`)++- `terminal_alive` and `terminal_resync` payloads add `reportMissing: true`.+- A `terminal_error` with `TERMINAL_STREAM_NOT_FOUND` for the current stream reopens the stream in+ place (`_recoverByReopen`, keeping the rendered screen) instead of `_fail`.++With a new CLI, a dead stream is found by the next heartbeat (≤ 5 s) instead of after+15 s of silence plus 12 s of resync. With an old CLI nothing changes here; the sweep in 3 still+recovers the pane.++### 3. Desktop: keep reattaching broken panes while the machine is online (desktop `AppNotifier`; web shares it)++Mobile is excluded on purpose. Its pager has a deliberate rule that a page a person has already looked+at is not reopened by a guess (`app_state_terminals_test.dart`, "a page a person has looked at is not+reopened by a guess"), and it already redials a dropped p2p channel itself. Mobile gets parts 1 and 2+only.++A pane whose session is `error` or `closed` (`_paneNeedsAttach`) on a machine that is online and+connected is reattached by a sweep, whichever tab it is in:++- One timer per notifier, armed only while at least one such pane exists and cancelled when none+ remain, so an idle app runs no timer.+- Each pane retries with backoff: 1, 2, 4, 8, 16, then every 30 s. The backoff resets when the pane+ reaches `controlling`. It stops while the machine is offline, because the existing offline/online+ path already owns that case.+- Reattach goes through the existing `_attachPendingPanes(machine, intent: AttachIntent.automatic)`.+ That keeps today's rules: it is a polite open (a watcher where someone else is driving), it+ respects `terminalNoTakeoverAvailable` and `_canAttachPane`, and `takenOver` panes are never+ touched.+- The per-session limit of one automatic reopen stays. The sweep is what makes "one reopen, then+ `error`" stop being a dead end.++No UI change. The header keeps showing Reconnect while a pane is `error`, and clicking it still+retries immediately. With the sweep, that state now lasts seconds instead of until someone clicks.++### 4. Desktop: write terminal failures and recoveries to the file log (Phase 3)++Today `TerminalSession`'s diagnostics go to `debugPrint` only, so `~/.harness/logs/app-*.log` cannot say+why a pane showed Reconnect. Log, through `appLog` (category `terminal`, one line each):++- `_fail`: machine, agent, stream, code, message;+- `transportLost`: machine, agent, message;+- `_recoverByReopen`: attempt and reason; `recovery_failed`;+- each sweep reattach: machine, agent, attempt number.++No terminal content and no paths are logged. Per-frame events (each resync attempt, each ack) stay out+of the file. Mobile is out of scope for this part.++## Compatibility++| Client | CLI on the host | Result |+|---|---|---|+| new | new | dead stream found within 5 s, reopened; sweep covers anything left |+| new | old | today's 15 s + 12 s detection; sweep then recovers the pane |+| old | new | unchanged: no `reportMissing`, so the host stays silent |+| old | old | unchanged |++The host and the clients ship independently: the CLI by self-update, desktop and mobile by release.++## Tests++- CLI (`terminalStreamManager` spec): `terminal_alive` and `terminal_resync` for an unknown stream with+ `reportMissing: true` produce one `TERMINAL_STREAM_NOT_FOUND` error naming that stream. Without the+ flag they produce nothing. A stream owned by another connection counts as missing for this one.+- Desktop and mobile `TerminalSession`: alive and resync payloads carry `reportMissing: true`;+ `TERMINAL_STREAM_NOT_FOUND` for the current stream reopens and keeps the terminal, and for another+ stream it is ignored.+- Desktop `AppNotifier`, with fake time:+ - an `error` pane on an online machine is reattached on the backoff schedule, in a background tab too;+ - the timer is cancelled once no pane needs it;+ - no sweep runs while the machine is offline;+ - `takenOver` panes are never reattached.+- Desktop log: `_fail` and a sweep reattach each write one `terminal` line to the file log sink.+- End to end on the local rig (this Mac plus the two docker remote boxes): open several+ remote panes across two tabs, then run each of these, and every pane must return to `controlling` with no+ click:+ 1. restart the remote box's daemon;+ 2. drop the remote box's backend link (stop the local backend for ~20 s);+ 3. kill only the viewer's relay socket.++ Record times to recovery before and after.++Validation: the CLI suites that cover `terminalStreamManager`, plus `test:core` if any `src/core`+wiring changes; the desktop affected tests (`terminal_session_test.dart`, the new pane-retry test);+the mobile `terminal_session_protocol_test.dart`. Record them in the PR, since PR CI runs only Linux unit checks.++## Out of scope++- Gap (c): letting a pane's own reopen reclaim its old stream's lease.+- Keeping p2p-carried streams alive across a backend link drop. Both ends tear p2p down with the relay+ socket today, and separating their lifetimes is an architectural change.+- Reducing frame counts (client ack cadence, coarser relay flushes).diff --git a/mobile/lib/terminal/terminal_session.dart b/mobile/lib/terminal/terminal_session.dartindex 52b2204b9..ef434f7b4 100644--- a/mobile/lib/terminal/terminal_session.dart+++ b/mobile/lib/terminal/terminal_session.dart@@ -1049,6 +1049,14 @@ class TerminalSession extends ChangeNotifier {// TERMINAL_PASTE_FAILED already is, since it may mean the pty is in an unknown state.// `_fail` itself resolves any in-flight upload (see `_abortActiveUpload`), so there is// nothing extra to do here for TERMINAL_PASTE_IMAGE_FAILED/TERMINAL_PASTE_FILE_FAILED.+ //+ // The daemon no longer holds this stream (it closed it while its link to the backend was+ // down, say) and said so because this session asked (`reportMissing`). Reopen in place now+ // rather than waiting out the heartbeat silence and three resyncs (2026-10-16).+ if (errorCode == 'TERMINAL_STREAM_NOT_FOUND') {+ unawaited(_recoverByReopen(reason: 'TERMINAL_STREAM_NOT_FOUND'));+ return true;+ }_fail(errorCode, payload['message']?.toString());return true;case 'terminal_transport_error':@@ -2095,7 +2103,11 @@ class TerminalSession extends ChangeNotifier {Future<void> _sendHeartbeat() async {if (!acceptsInput) return;final generation = _generation;- final sent = await send('terminal_alive', {'streamId': streamId});+ final sent = await send('terminal_alive', {+ 'streamId': streamId,+ // Ask the daemon to say so if it no longer holds this stream (CLI `reportMissing`).+ 'reportMissing': true,+ });if (!sent && _isCurrent(generation)) {transportLost('Terminal heartbeat was not sent');return;@@ -2176,6 +2188,7 @@ class TerminalSession extends ChangeNotifier {'streamId': currentStream,'attempt': _resyncAttempts,'reason': errorCode,+ 'reportMissing': true,});if (!_isCurrent(generation) ||streamId != currentStream ||diff --git a/mobile/test/terminal_session_protocol_test.dart b/mobile/test/terminal_session_protocol_test.dartindex 66d052f66..8f1cd4d83 100644--- a/mobile/test/terminal_session_protocol_test.dart+++ b/mobile/test/terminal_session_protocol_test.dart@@ -377,6 +377,71 @@ void main() {},);+ test(+ 'heartbeats and resyncs ask the daemon to report a stream it no longer has',+ () {+ fakeAsync((async) {+ final s = session(+ disposeAtEnd: false,+ resync: const Duration(seconds: 30),+ );+ s.open();+ async.flushMicrotasks();+ final streamId = _streamId();+ s.handleFrame('terminal_ready', {+ 'requestId': wire.last('terminal_open')['requestId'],+ 'agentId': 'a',+ 'protocolVersion': TerminalSession.protocolVersion,+ 'streamId': streamId,+ });+ s.handleBinary(_keyframe(streamId));+ async.flushMicrotasks();++ async.elapse(const Duration(seconds: 5));+ expect(wire.last('terminal_alive'), {+ 'streamId': streamId,+ 'reportMissing': true,+ });++ s.handleBinary(_output(streamId, 2, 'gap'));+ async.flushMicrotasks();+ expect(wire.last('terminal_resync')['reportMissing'], isTrue);+ s.dispose();+ });+ },+ );++ test('a stream the daemon no longer has is reopened in place at once', () async {+ final s = session();+ final opening = s.open();+ final streamId = await ready(s);+ await opening;+ await s.handleBinary(_keyframe(streamId));++ await s.handleFrame('terminal_error', {+ 'streamId': streamId,+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });+ await until(() => wire.of('terminal_open').length == 2);++ expect(s.status, TerminalSessionStatus.opening);+ });++ test('a missing-stream report about another stream is ignored', () async {+ final s = session();+ final opening = s.open();+ await ready(s);+ await opening;++ await s.handleFrame('terminal_error', {+ 'streamId': _streamId(),+ 'code': 'TERMINAL_STREAM_NOT_FOUND',+ });++ expect(wire.of('terminal_open'), hasLength(1));+ expect(s.status, isNot(TerminalSessionStatus.error));+ });+test('heartbeats keep a live stream alive, and a failed one is a lost stream',() {@@ -658,6 +723,7 @@ void main() {'streamId': streamId,'attempt': 1,'reason': 'TERMINAL_KEYFRAME_TIMEOUT',+ 'reportMissing': true,});await s.handleBinary(_keyframe(streamId, seq: 5));