Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

fix: realign terminal input sequence on TERMINAL_INPUT_INVALID to prevent session freezing

0xken1584a23e5f8d35
diff --git a/cli/src/lib/terminalStreamManager.spec.ts b/cli/src/lib/terminalStreamManager.spec.ts
index 4b7be05b1..0806b3752 100644
--- a/cli/src/lib/terminalStreamManager.spec.ts
+++ b/cli/src/lib/terminalStreamManager.spec.ts
@@ -157,6 +157,17 @@ describe('TerminalStreamManager', () => {
})
expect(stream.writes).toHaveLength(1)
expect(sent.at(-1)?.payload.code).toBe('TERMINAL_INPUT_INVALID')
+ // …and says which seq it is still waiting for, so a client that skipped a number can realign
+ // instead of having every later keystroke refused.
+ expect(sent.at(-1)?.payload).toMatchObject({ reason: 'seq', expectedSeq: 1, streamId })
+
+ // An oversized frame is refused the same way, with the counter untouched: the next accepted
+ // seq is still 1.
+ await manager.handleBinary('web-1', {
+ kind: TerminalBinaryKind.input, streamId, seq: 1, compressed: false, bytes: Buffer.alloc(64 * 1024 + 1, 0x61),
+ })
+ expect(stream.writes).toHaveLength(1)
+ expect(sent.at(-1)?.payload).toMatchObject({ code: 'TERMINAL_INPUT_INVALID', reason: 'size', expectedSeq: 1 })
const mouse = Buffer.from('\u001b[<0;12;8M')
await manager.handleBinary('web-1', {
diff --git a/cli/src/lib/terminalStreamManager.ts b/cli/src/lib/terminalStreamManager.ts
index 3a1f6112f..f438af183 100644
--- a/cli/src/lib/terminalStreamManager.ts
+++ b/cli/src/lib/terminalStreamManager.ts
@@ -261,7 +261,11 @@ export class TerminalStreamManager {
})
}
- private sendError(connId: string, code: string, options: { streamId?: string; requestId?: unknown; message?: string } = {}): void {
+ private sendError(
+ connId: string,
+ code: string,
+ options: { streamId?: string; requestId?: unknown; message?: string; reason?: 'seq' | 'size' | 'empty'; expectedSeq?: number } = {},
+ ): void {
this.deps.sendTarget(connId, 'terminal_error', {
protocolVersion: PROTOCOL_VERSION,
code,
@@ -450,7 +454,21 @@ export class TerminalStreamManager {
private async input(state: ActiveStream, inputSeq: number, bytes: Uint8Array): Promise<void> {
if (!Number.isSafeInteger(inputSeq) || inputSeq !== state.lastInputSeq + 1 || bytes.length === 0 || bytes.length > INPUT_MAX_BYTES) {
- this.sendError(state.connId, 'TERMINAL_INPUT_INVALID', { streamId: state.streamId })
+ // Nothing here reached the pty, and `lastInputSeq` is deliberately left where it was. Say what
+ // WOULD have been accepted: a client whose counter has drifted (a frame it dropped on its own
+ // side while resyncing, say) can realign on `expectedSeq` and carry on, instead of having every
+ // later keystroke refused too and treating a fine stream as a dead one.
+ const expectedSeq = state.lastInputSeq + 1
+ // A seq that is not a safe integer at all is an ordering fault too, as far as the client is concerned.
+ const reason = bytes.length === 0 ? 'empty' : bytes.length > INPUT_MAX_BYTES ? 'size' : 'seq'
+ this.sendError(state.connId, 'TERMINAL_INPUT_INVALID', {
+ streamId: state.streamId,
+ reason,
+ expectedSeq,
+ message: reason === 'seq'
+ ? `input seq ${inputSeq} arrived, ${expectedSeq} expected`
+ : reason === 'size' ? `input frame of ${bytes.length} bytes exceeds ${INPUT_MAX_BYTES}` : 'empty input frame',
+ })
return
}
state.lastInputSeq = inputSeq
diff --git a/desktop/lib/state/app_state.dart b/desktop/lib/state/app_state.dart
index 407778db4..96af2c202 100644
--- a/desktop/lib/state/app_state.dart
+++ b/desktop/lib/state/app_state.dart
@@ -2791,35 +2791,53 @@ class AppNotifier extends ChangeNotifier {
String? revision,
}) async {
final machine = machineStates[machineId];
- if (machine == null || machine.needsLink || machine.nodeOnline == false ||
+ if (machine == null ||
+ machine.needsLink ||
+ machine.nodeOnline == false ||
machine.connectionStatus != ConnectionStatus.connected) {
- throw const RemoteMediaException('This machine is disconnected. Reconnect and try opening the preview again.');
+ throw const RemoteMediaException(
+ 'This machine is disconnected. Reconnect and try opening the preview again.',
+ );
}
if (!machine.mediaPreviewAvailable) {
- throw const RemoteMediaException('Update the Harness CLI on this remote machine to open image and video previews.');
+ throw const RemoteMediaException(
+ 'Update the Harness CLI on this remote machine to open image and video previews.',
+ );
}
final connection = _conn(machineId);
if (!connection.isReady) {
- throw const RemoteMediaException('This machine is disconnected. Reconnect and try opening the preview again.');
+ throw const RemoteMediaException(
+ 'This machine is disconnected. Reconnect and try opening the preview again.',
+ );
}
try {
- return await connection.request('agent_read_file', payload: {
- 'agentId': agentId, 'path': target, 'media': true, 'offset': offset,
- 'revision': ?revision,
- }, timeout: const Duration(seconds: 15));
+ return await connection.request(
+ 'agent_read_file',
+ payload: {
+ 'agentId': agentId,
+ 'path': target,
+ 'media': true,
+ 'offset': offset,
+ 'revision': ?revision,
+ },
+ timeout: const Duration(seconds: 15),
+ );
} on WsRequestFailure catch (error) {
throw RemoteMediaException(switch (error.code) {
'MEDIA_NOT_FOUND' || 'NOT_FOUND' => 'This file is no longer available on the remote machine. It may have moved or been deleted.',
'MEDIA_TOO_LARGE' => 'Remote previews support files up to 512 MB. Use a smaller export or transfer this file separately.',
'MEDIA_CHANGED' => 'The file changed while downloading. Wait for it to finish generating and try again.',
'MEDIA_UNSUPPORTED' => 'This file is not a supported image or video.',
- 'MEDIA_INVALID_REQUEST' => 'Use a full path or a path inside this agent’s working folder.',
+ 'MEDIA_INVALID_REQUEST' =>
+ 'Use a full path or a path inside this agent’s working folder.',
'AGENT_NOT_FOUND' => 'This agent is no longer available. Reconnect to the agent and try again.',
'NOT_TEXT' || 'FILE_TOO_LARGE' => 'Update the Harness CLI on this remote machine to open media previews.',
_ => 'The remote machine could not read this file. Check that it is accessible and try again.',
});
} catch (_) {
- throw const RemoteMediaException('The media download was interrupted. Check the connection and try again.');
+ throw const RemoteMediaException(
+ 'The media download was interrupted. Check the connection and try again.',
+ );
}
}
diff --git a/desktop/lib/terminal/terminal_session.dart b/desktop/lib/terminal/terminal_session.dart
index c95ab6060..93d9339d3 100644
--- a/desktop/lib/terminal/terminal_session.dart
+++ b/desktop/lib/terminal/terminal_session.dart
@@ -129,6 +129,12 @@ class TerminalSession extends ChangeNotifier {
int _framesSinceAck = 0;
int _renderedSinceAckBytes = 0;
int _inputSeq = 0;
+
+ /// The last `expectedSeq` a TERMINAL_INPUT_INVALID realigned the counter to.
+ /// The daemon refuses every in-flight frame it cannot accept, each naming
+ /// the SAME expected seq; only the first of those may rewind the counter,
+ /// or frames accepted since would be re-sent under numbers already used.
+ int? _lastRealignedTo;
int _resizeSeq = 0;
bool _resyncRequested = false;
int _resyncAttempts = 0;
@@ -202,6 +208,7 @@ class TerminalSession extends ChangeNotifier {
_framesSinceAck = 0;
_renderedSinceAckBytes = 0;
_inputSeq = 0;
+ _lastRealignedTo = null;
_resizeSeq = 0;
_utf8Tail = const [];
_remoteCursorVisible = true;
@@ -414,6 +421,37 @@ class TerminalSession extends ChangeNotifier {
);
return true;
}
+ // A refused INPUT frame never reached tmux either: the stream is fine, only our counter
+ // disagrees with the daemon's (a frame dropped on this side while resyncing, say). A daemon
+ // that says what it expects gets realigned in place — the refused keystroke is gone either
+ // way, and the person retypes it — and an older one that does not gets a fresh stream. What
+ // it must NOT get is the frozen pane this used to be: every later keystroke was refused too,
+ // and a working terminal read as a dead one.
+ if (errorCode == 'TERMINAL_INPUT_INVALID') {
+ final expected = payload['expectedSeq'];
+ if (expected is int && expected >= 0) {
+ // A stale duplicate: the daemon refused several in-flight frames
+ // at once, each naming this same seq, and the counter has since
+ // moved on from it and been accepted. Rewinding again would put the
+ // next keystroke under a number the daemon has already taken.
+ // Safe because its expectation only ever grows and its errors
+ // arrive in order — a NEW gap always names a larger seq.
+ if (expected == _lastRealignedTo) return true;
+ _lastRealignedTo = expected;
+ _inputSeq = expected;
+ // Bytes still waiting in the coalescer are kept: they are numbered
+ // on the way out, after this, so they go under the right seq.
+ debugPrint(
+ 'TerminalSession: input seq realigned to $expected '
+ '(${payload['reason'] ?? payload['message'] ?? 'no reason given'})',
+ );
+ notifyListeners();
+ } else {
+ _inputBytes.clear();
+ unawaited(_recoverByReopen(reason: 'TERMINAL_INPUT_INVALID'));
+ }
+ return true;
+ }
// A genuine mid-transfer failure (the daemon couldn't write the clipboard/disk, or the
// pty write itself failed) IS treated like a real transport/protocol failure — same as
// TERMINAL_PASTE_FAILED already is, since it may mean the pty is in an unknown state.
@@ -905,17 +943,37 @@ class TerminalSession extends ChangeNotifier {
_lastInputFlushAt = DateTime.now();
final currentStreamId = streamId;
if (currentStreamId == null) return;
- final frame = TerminalBinaryFrame(
- kind: TerminalBinaryKind.input,
- streamId: currentStreamId,
- seq: _inputSeq++,
- bytes: Uint8List.fromList(bytes),
- compressed: false,
- );
final queued = _inputSendTail.then((_) async {
if (!acceptsInput || streamId != currentStreamId) return;
- final sent = await sendBinary(frame);
- if (!sent) transportLost('Terminal input was not sent');
+ // Numbered HERE, on the tail, not when the flush was asked for. A frame
+ // the guard above drops — the session went `resyncing` while an earlier
+ // send was still in flight — must not consume a seq: the daemon wants
+ // them contiguous and never re-syncs its counter, so one skipped number
+ // had the very next keystroke refused and the whole pane frozen. The
+ // tail runs one closure at a time, which is what keeps the numbers in
+ // order.
+ //
+ // Never larger than the daemon accepts, either — split rather than let
+ // one oversized frame be refused with the counter intact behind it.
+ for (
+ var offset = 0;
+ offset < bytes.length;
+ offset += kInputFrameMaxBytes
+ ) {
+ final end = min(offset + kInputFrameMaxBytes, bytes.length);
+ final frame = TerminalBinaryFrame(
+ kind: TerminalBinaryKind.input,
+ streamId: currentStreamId,
+ seq: _inputSeq++,
+ bytes: Uint8List.fromList(bytes.sublist(offset, end)),
+ compressed: false,
+ );
+ final sent = await sendBinary(frame);
+ if (!sent) {
+ transportLost('Terminal input was not sent');
+ return;
+ }
+ }
});
_inputSendTail = queued.catchError((_) {
transportLost('Terminal input was not sent');
@@ -938,6 +996,10 @@ class TerminalSession extends ChangeNotifier {
/// charged its full window to every single keystroke and to the start of every drag, which is
/// latency paid for a batch that mostly never materializes.
static const _inputCoalesceWindow = Duration(milliseconds: 4);
+
+ /// The daemon's ceiling on one input frame (`INPUT_MAX_BYTES` in
+ /// `cli/src/lib/terminalStreamManager.ts`). Larger is refused, not split.
+ static const kInputFrameMaxBytes = 64 * 1024;
static const _resizeCoalesceWindow = Duration(milliseconds: 50);
void resize(int width, int height) {
@@ -1107,16 +1169,20 @@ class TerminalSession extends ChangeNotifier {
});
}
- Future<void> _recoverByReopen() async {
+ Future<void> _recoverByReopen({
+ String reason = 'TERMINAL_RESYNC_TIMEOUT',
+ }) async {
final previousStream = streamId;
if (_autoReopenAttempts >= 1) {
debugPrint(
'[terminal-session] recovery_failed agent=$agentId '
- 'stream=$previousStream attempts=$_resyncAttempts',
+ 'stream=$previousStream reason=$reason attempts=$_resyncAttempts',
);
_fail(
- 'TERMINAL_RESYNC_TIMEOUT',
- 'Terminal did not recover after resync and reopen.',
+ reason,
+ reason == 'TERMINAL_RESYNC_TIMEOUT'
+ ? 'Terminal did not recover after resync and reopen.'
+ : 'Terminal did not recover after reopening the stream.',
);
return;
}
diff --git a/desktop/test/terminal_session_test.dart b/desktop/test/terminal_session_test.dart
index e77469e60..d0c937fed 100644
--- a/desktop/test/terminal_session_test.dart
+++ b/desktop/test/terminal_session_test.dart
@@ -1,3 +1,4 @@
+import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
@@ -669,6 +670,210 @@ void main() {
},
);
+ test('a keystroke dropped while resyncing does not leave a hole in the input seq', () async {
+ // The daemon counts input frames and never re-syncs that counter. A frame this side drops
+ // — the session went `resyncing` while an earlier send was still in flight — used to take
+ // its number with it, so the first keystroke after recovery arrived one ahead and was
+ // refused: TERMINAL_INPUT_INVALID, and a frozen pane over a perfectly good tmux.
+ final gate = Completer<void>();
+ final frames = <TerminalBinaryFrame>[];
+ final blocking = TerminalSession(
+ machineId: 'machine-1',
+ agentId: 'agent-1',
+ agentName: 'backend-api',
+ engineId: 'codex',
+ send: (type, payload) async {
+ sent.add((type: type, payload: Map<String, dynamic>.from(payload)));
+ return true;
+ },
+ sendBinary: (frame) async {
+ frames.add(frame);
+ if (frames.length == 1) await gate.future; // the first send stalls
+ return true;
+ },
+ );
+ addTearDown(blocking.dispose);
+ await blocking.open(initialCols: 100, initialRows: 30);
+ await blocking.handleFrame('terminal_ready', {
+ 'requestId': sent.single.payload['requestId'],
+ 'protocolVersion': 3,
+ 'streamId': streamId,
+ 'agentId': 'agent-1',
+ });
+ await blocking.handleBinary(
+ output(0, utf8.encode('prompt> '), keyframe: true, cols: 80, rows: 24),
+ );
+
+ blocking.terminal.onOutput?.call('a'); // seq 0, send stalls on the gate
+ await Future<void>.delayed(const Duration(milliseconds: 1));
+ blocking.terminal.onOutput?.call('b'); // queued behind it
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(frames, hasLength(1));
+
+ // The output stream skips a number: resync, same stream.
+ await blocking.handleBinary(output(5, utf8.encode('gap')));
+ expect(blocking.status, TerminalSessionStatus.resyncing);
+ gate.complete(); // 'a' goes out; 'b' reaches the tail while resyncing and is dropped
+ await Future<void>.delayed(const Duration(milliseconds: 1));
+
+ await blocking.handleBinary(
+ output(9, utf8.encode('fresh'), keyframe: true, cols: 80, rows: 24),
+ );
+ expect(blocking.status, TerminalSessionStatus.controlling);
+ blocking.terminal.onOutput?.call('c');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+
+ final inputs = frames.where((f) => f.kind == TerminalBinaryKind.input);
+ expect(inputs.map((f) => utf8.decode(f.bytes)), ['a', 'c']);
+ // Contiguous: the dropped 'b' did not spend a number.
+ expect(inputs.map((f) => f.seq), [0, 1]);
+ });
+
+ test(
+ 'TERMINAL_INPUT_INVALID with expectedSeq realigns in place, nothing frozen',
+ () async {
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('prompt> '), keyframe: true, cols: 80, rows: 24),
+ );
+ session.terminal.onOutput?.call('a');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ session.terminal.onOutput?.call('b');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(binarySent.map((f) => f.seq), [0, 1]);
+
+ // The daemon only ever saw seq 0 — say it is still waiting for 1.
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_INPUT_INVALID',
+ 'reason': 'seq',
+ 'expectedSeq': 1,
+ });
+ expect(session.status, TerminalSessionStatus.controlling);
+ expect(session.errorCode, isNull);
+ expect(sent.where((f) => f.type == 'terminal_close'), isEmpty);
+ expect(sent.where((f) => f.type == 'terminal_open'), hasLength(1));
+
+ session.terminal.onOutput?.call('c');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(binarySent.last.seq, 1);
+ expect(utf8.decode(binarySent.last.bytes), 'c');
+ },
+ );
+
+ test('a stale duplicate TERMINAL_INPUT_INVALID does not rewind a counter that moved on', () async {
+ // The daemon refuses every in-flight frame it cannot take, each naming
+ // the same expected seq. By the time the second refusal lands, the
+ // first has already realigned us and frames under the new numbers have
+ // been accepted — rewinding again would reuse them.
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('prompt> '), keyframe: true, cols: 80, rows: 24),
+ );
+ session.terminal.onOutput?.call('a');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ session.terminal.onOutput?.call('b');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ session.terminal.onOutput?.call('c');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(binarySent.map((f) => f.seq), [0, 1, 2]);
+
+ // Both 'b' (1) and 'c' (2) were refused: the daemon still wants 1.
+ final refusal = {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_INPUT_INVALID',
+ 'reason': 'seq',
+ 'expectedSeq': 1,
+ };
+ await session.handleFrame('terminal_error', refusal);
+ session.terminal.onOutput?.call('d'); // goes out as 1, accepted
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ await session.handleFrame('terminal_error', refusal); // the stale one
+ session.terminal.onOutput?.call('e');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+
+ final after = binarySent.skip(3).toList();
+ expect(after.map((f) => utf8.decode(f.bytes)), ['d', 'e']);
+ expect(after.map((f) => f.seq), [1, 2]);
+ expect(session.status, TerminalSessionStatus.controlling);
+
+ // A genuinely new gap names a larger seq and is honoured.
+ await session.handleFrame('terminal_error', {...refusal, 'expectedSeq': 2});
+ session.terminal.onOutput?.call('f');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(binarySent.last.seq, 2);
+ });
+
+ test(
+ 'TERMINAL_INPUT_INVALID from an older daemon reopens instead of freezing',
+ () async {
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('prompt> '), keyframe: true, cols: 80, rows: 24),
+ );
+ session.terminal.onOutput?.call('a');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+
+ // No expectedSeq: a daemon from before it said. The only way back in step is a new stream.
+ await session.handleFrame('terminal_error', {
+ 'streamId': streamId,
+ 'code': 'TERMINAL_INPUT_INVALID',
+ });
+ expect(session.status, isNot(TerminalSessionStatus.error));
+ expect(sent.where((f) => f.type == 'terminal_close'), hasLength(1));
+ final opens = sent.where((f) => f.type == 'terminal_open').toList();
+ expect(opens, hasLength(2));
+
+ const newStream = 'ffeeddcc-bbaa-9988-7766-554433221100';
+ await session.handleFrame('terminal_ready', {
+ 'requestId': opens.last.payload['requestId'],
+ 'protocolVersion': 3,
+ 'streamId': newStream,
+ 'agentId': 'agent-1',
+ });
+ await session.handleBinary(
+ TerminalBinaryFrame(
+ kind: TerminalBinaryKind.keyframe,
+ streamId: newStream,
+ seq: 0,
+ bytes: Uint8List.fromList(utf8.encode('prompt> ')),
+ compressed: false,
+ cols: 80,
+ rows: 24,
+ ),
+ );
+ expect(session.status, TerminalSessionStatus.controlling);
+ session.terminal.onOutput?.call('b');
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ expect(binarySent.last.streamId, newStream);
+ expect(binarySent.last.seq, 0);
+ },
+ );
+
+ test(
+ 'input larger than the daemon accepts is split, never refused',
+ () async {
+ await ready();
+ await session.handleBinary(
+ output(0, utf8.encode('prompt> '), keyframe: true, cols: 80, rows: 24),
+ );
+ // Bypass the 8 KiB chunker's boundaries: one contiguous burst of 70,000 bytes.
+ session.terminal.onOutput?.call('x' * 70000);
+ await Future<void>.delayed(const Duration(milliseconds: 12));
+ final inputs = binarySent
+ .where((f) => f.kind == TerminalBinaryKind.input)
+ .toList();
+ expect(
+ inputs.every(
+ (f) => f.bytes.length <= TerminalSession.kInputFrameMaxBytes,
+ ),
+ isTrue,
+ );
+ expect(inputs.map((f) => f.seq), List.generate(inputs.length, (i) => i));
+ expect(inputs.fold<int>(0, (n, f) => n + f.bytes.length), 70000);
+ },
+ );
+
test(
'idle sync advances sequence and is acknowledged after render',
() async {