Skip to content

Node-based terminal manager for AI coding agents — tmux-backed terminals and parallel agent sessions as draggable nodes on an infinite pan/zoom canvas. macOS, Linux, and a browser Server Edition.

fix(watch-link): the output filter has no length cap

review6c02b5f613d6
diff --git a/src/core/watch-link/__fixtures__/xterm-vt500.ts b/src/core/watch-link/__fixtures__/xterm-vt500.ts
index fb77d47a7..e8cf5c113 100644
--- a/src/core/watch-link/__fixtures__/xterm-vt500.ts
+++ b/src/core/watch-link/__fixtures__/xterm-vt500.ts
@@ -7,7 +7,7 @@
// VT500_TRANSITION_TABLE
// `stream-filter.xterm.test.ts` compares each block with that source, so an xterm upgrade that
// changes the table fails there instead of silently testing the filter against an old model.
-// Only the code after the END marker is ours.
+// Only the code after the END marker is ours; the lines of `parse()` it models are pinned there too.
//
// The copied blocks: Copyright (c) 2017-2018 The xterm.js authors. MIT license.
@@ -233,7 +233,8 @@ const STRING_STATES = new Set<number>([
// PARAM, OSC_PUT, DCS_PUT) only take codes whose table entry keeps the state they are in, and an
// OSC or DCS ended by ESC goes to ESCAPE although the table says GROUND
// (`transition |= ParserState.ESCAPE`). An ERROR (a non-ASCII code in SOS/PM/APC or a DCS header)
-// goes to GROUND with the default error handler, as the table says.
+// goes to GROUND with the default error handler, as the table says. The lines of parse() this
+// stands for are pinned in `stream-filter.xterm.test.ts`.
function step(state: number, code: number): number {
let transition = VT500_TRANSITION_TABLE.table[state << TableAccess.INDEX_STATE_SHIFT | (code < 0xa0 ? code : NON_ASCII_PRINTABLE)]
const action = transition >> TableAccess.TRANSITION_ACTION_SHIFT
diff --git a/src/core/watch-link/stream-filter.test.ts b/src/core/watch-link/stream-filter.test.ts
index 1b7178508..28cf1dbfe 100644
--- a/src/core/watch-link/stream-filter.test.ts
+++ b/src/core/watch-link/stream-filter.test.ts
@@ -1,24 +1,33 @@
import { describe, it, expect } from 'vitest'
+import { readFileSync } from 'node:fs'
+import { join } from 'node:path'
import { createStreamFilter } from './stream-filter'
const ESC = '\x1b'
const BEL = '\x07'
-const run = (chunks: string[], max?: number): string => {
- const f = createStreamFilter(max)
+const run = (chunks: string[]): string => {
+ const f = createStreamFilter()
return chunks.map((c) => f.push(c)).join('')
}
+// One string pushed in pieces of `size` chars, the way a pty delivers a large write.
+const pushInPieces = (text: string, size: number, midStream = false): string => {
+ const f = createStreamFilter({ midStream })
+ let out = ''
+ for (let i = 0; i < text.length; i += size) out += f.push(text.slice(i, i + size))
+ return out
+}
+// 2 MiB of base64, the shape of a large OSC 52.
+const BIG = 'QUJD'.repeat(512 * 1024)
// The filter's rules, one UTF-16 unit at a time and without a fast path: the oracle the fuzz test
// holds the production parser to.
-function oneCharAtATime(input: string, max: number, midStream: boolean): string {
+function oneCharAtATime(input: string, midStream: boolean): string {
// midStream: as if an 8-bit DCS introducer had just been read.
let mode: 'text' | 'esc' | 'string' | 'stringEsc' = midStream ? 'string' : 'text'
let osc = false
- let len = 0
let out = ''
const start = (ch: string): 'string' => {
osc = ch === ']' || ch === '\x9d'
- len = 0
return 'string'
}
for (let i = 0; i < input.length; i++) {
@@ -33,10 +42,6 @@ function oneCharAtATime(input: string, max: number, midStream: boolean): string
if (ch === '\x9c' || (ch === BEL && osc)) mode = 'text'
else if (ch === ESC) mode = 'stringEsc'
else if (intro8) mode = start(ch)
- else if (++len > max) {
- mode = 'text'
- len = 0
- }
} else if (']PX^_'.includes(ch) || intro8) mode = start(ch)
else if (ch === ESC) mode = 'esc'
else if (n <= 0x17 || n === 0x19 || (n >= 0x1c && n <= 0x1f)) out += ch
@@ -76,12 +81,42 @@ describe('createStreamFilter', () => {
expect(run([`a${ESC}]52;c;xx${ESC}[31mred`])).toBe(`a${ESC}[31mred`)
})
- it('keeps swallowing an unterminated string until the cap, then resumes text', () => {
- const f = createStreamFilter(10)
- expect(f.push(`a${ESC}]52;c;`)).toBe('a')
- expect(f.push('12345')).toBe('')
- // The 11th swallowed char ('6') trips the cap and is dropped with the rest; text resumes after it.
- expect(f.push('67890XYZ')).toBe('7890XYZ')
+ // No length cap: xterm has none, so any cap would show a viewer bytes the owner's screen did not
+ // (measured: a 900 KB tmux copy is one 1.2 MB OSC 52). The sizes and push sizes are a pty's.
+ it('a 2 MiB OSC 52, BEL- or ST-terminated, is swallowed whole in 64 KiB or 4 KiB pushes', () => {
+ for (const end of [BEL, `${ESC}\\`, '\x9c']) {
+ for (const size of [65536, 4096]) {
+ expect(pushInPieces(`a${ESC}]52;c;${BIG}${end}b${ESC}[mc`, size), `${JSON.stringify(end)} / ${size}`).toBe(
+ `ab${ESC}[mc`
+ )
+ }
+ }
+ })
+
+ it('a 2 MiB DCS is swallowed whole, a BEL in its data included', () => {
+ const half = BIG.slice(0, BIG.length / 2)
+ for (const size of [65536, 4096]) {
+ expect(pushInPieces(`a${ESC}Pq${half}${BEL}${half}${ESC}\\b`, size)).toBe('ab')
+ expect(pushInPieces(`a\x90q${half}${BEL}${half}\x9cb`, size)).toBe('ab')
+ }
+ })
+
+ it("is swallowed whole even past xterm's own 10,000,000-char payload limit", () => {
+ // xterm stops handing a string that long to its handler but stays in it until the terminator.
+ const huge = 'QUJD'.repeat(4 * 1024 * 1024)
+ expect(pushInPieces(`a${ESC}]52;c;${huge}${BEL}b`, 65536)).toBe('ab')
+ })
+
+ it('the string branch counts nothing, so no length can end a string (source)', () => {
+ // The tests above catch a cap up to the longest string they push (16 MiB); a cap past that
+ // would need a longer test to see. A cap needs a counter and a comparison, and the branch that
+ // reads a string's payload has neither.
+ const src = readFileSync(join(__dirname, 'stream-filter.ts'), 'utf8').replace(/\r\n/g, '\n')
+ const start = src.indexOf("} else if (mode === 'string') {")
+ const branch = src.slice(start, src.indexOf('} else {', start)).replace(/\/\/.*$/gm, '')
+ expect(start).toBeGreaterThan(-1)
+ expect(branch).toContain('.exec(chunk)')
+ expect(branch).not.toMatch(/\+\+|--|\+=|-=|[<>]/)
})
it('reset forgets a half-read sequence', () => {
@@ -91,15 +126,6 @@ describe('createStreamFilter', () => {
expect(f.push('visible')).toBe('visible')
})
- it('gives up only past 1 MiB by default: a string of exactly 1 MiB is still swallowed whole', () => {
- const cap = 1_048_576
- // `52;c;` counts too: the string's payload starts right after the introducer.
- const payload = '52;c;' + 'A'.repeat(cap - 5)
- expect(run([`x${ESC}]${payload}${BEL}y`])).toBe('xy')
- // One more char trips the cap; it is dropped and text resumes after it.
- expect(run([`x${ESC}]${payload}Ztail`])).toBe('xtail')
- })
-
it('a keyframe screen (capture-pane -e) loses its OSC 8 links and keeps the link text', () => {
// tmux 3.4 `capture-pane -p -e` emits hyperlinks verbatim (measured), ST-terminated; programs
// may end them with BEL instead. A keyframe is one push of a whole screen through a FRESH filter.
@@ -157,11 +183,8 @@ describe('createStreamFilter', () => {
it('an 8-bit introducer inside a string starts a new string of its own kind', () => {
// OSC → DCS: the BEL is now DCS data, not a terminator.
expect(run([`a${ESC}]0;t\x90q${BEL}hidden\x9cb`])).toBe('ab')
- // ...and it starts a fresh count toward the cap.
- const f = createStreamFilter(10)
- expect(f.push(`a${ESC}]` + '12345678')).toBe('a')
- expect(f.push('\x9d' + '12345678')).toBe('')
- expect(f.push(`${BEL}b`)).toBe('b')
+ // DCS → OSC: and now it is one.
+ expect(run([`a${ESC}Pq\x9d0;t${BEL}b`])).toBe('ab')
})
it('reset also forgets a pending ESC', () => {
@@ -200,18 +223,17 @@ describe('createStreamFilter', () => {
for (let iter = 0; iter < 4000; iter++) {
let input = ''
for (let i = rand(60); i > 0; i--) input += ALPHABET[rand(ALPHABET.length)]
- const cap = [2, 5, 1_048_576][rand(3)]
const midStream = rand(2) === 1
- // Cut anywhere, a surrogate pair included: the cap counts UTF-16 units, so nothing depends on it.
+ // Cut anywhere, a surrogate pair included: nothing in the filter counts or splits on one.
const cuts = [0, input.length]
for (let k = rand(5); k > 0; k--) cuts.push(rand(input.length + 1))
cuts.sort((x, y) => x - y)
const chunks = cuts.slice(1).map((end, i) => input.slice(cuts[i], end))
- const f = createStreamFilter(cap, { midStream })
+ const f = createStreamFilter({ midStream })
const outs = chunks.map((c) => f.push(c))
for (const out of outs) {
- const ctx = JSON.stringify({ input, chunks, cap, midStream, out })
+ const ctx = JSON.stringify({ input, chunks, midStream, out })
expect(INTRO_8.test(out), ctx).toBe(false)
expect(out.endsWith(ESC), ctx).toBe(false)
for (let i = out.indexOf(ESC); i !== -1; i = out.indexOf(ESC, i + 1)) {
@@ -220,7 +242,7 @@ describe('createStreamFilter', () => {
}
// However it is chunked, the result is the char-by-char statement of the same rules: the
// parser's slicing and fast path neither drop, reorder nor add a char.
- expect(outs.join(''), JSON.stringify({ chunks, cap, midStream })).toBe(oneCharAtATime(input, cap, midStream))
+ expect(outs.join(''), JSON.stringify({ chunks, midStream })).toBe(oneCharAtATime(input, midStream))
}
})
})
@@ -229,8 +251,8 @@ describe('createStreamFilter, joined mid-stream', () => {
// A watcher co-attaches to a RUNNING session, so its first byte can land anywhere: inside an OSC 52,
// or right after an ESC that the previous read ended on. A filter that starts in text would print
// the rest of that string, or read `]52;…` as text.
- const mid = (chunks: string[], max?: number): string => {
- const f = createStreamFilter(max, { midStream: true })
+ const mid = (chunks: string[]): string => {
+ const f = createStreamFilter({ midStream: true })
return chunks.map((c) => f.push(c)).join('')
}
@@ -278,16 +300,19 @@ describe('createStreamFilter, joined mid-stream', () => {
expect(mid([`DATA\x9d52;c;x${BEL}after`])).toBe('after')
})
- it('the cap counts from the join, on creation and on reset', () => {
- const f = createStreamFilter(5, { midStream: true })
- expect(f.push('12345')).toBe('')
- expect(f.push('6789')).toBe('789')
+ it('a join into the tail of a 2 MiB OSC 52 swallows all of it', () => {
+ const tail = BIG.slice(12345)
+ for (const size of [65536, 4096]) {
+ expect(pushInPieces(`${tail}${BEL}rest${ESC}[mafter`, size, true)).toBe(`${ESC}[mafter`)
+ expect(pushInPieces(`${tail}${ESC}\\after`, size, true)).toBe('after')
+ }
+ })
- const g = createStreamFilter(10)
- expect(g.push(`a${ESC}]12345678`)).toBe('a')
+ it('reset({ midStream }) inside an OSC forgets that it was one: BEL no longer ends it', () => {
+ const g = createStreamFilter()
+ expect(g.push(`a${ESC}]0;t`)).toBe('a')
g.reset({ midStream: true })
- expect(g.push('1234567890')).toBe('')
- expect(g.push('Xab')).toBe('ab')
+ expect(g.push(`x${BEL}y${ESC}[mz`)).toBe(`${ESC}[mz`)
})
it('reset({ midStream }) drops what was being read and starts as a join', () => {
diff --git a/src/core/watch-link/stream-filter.ts b/src/core/watch-link/stream-filter.ts
index c208746d9..781158cc6 100644
--- a/src/core/watch-link/stream-filter.ts
+++ b/src/core/watch-link/stream-filter.ts
@@ -6,8 +6,11 @@
// Stateful across chunks, and it must see EVERY byte of the stream even while nothing is being
// forwarded: a frame that skipped the parser would leave it mid-sequence and print the tail of an
// OSC 52 (the clipboard) as text. That is also why it is reset only for a new pty session.
-// An unterminated string is swallowed until its terminator; only past `maxStringChars` (UTF-16
-// units, i.e. string length) does the parser give up and return to text.
+// A string is swallowed until its terminator, however long. There is no length cap: xterm has none
+// (it stays in a string until ESC, ST, BEL for an OSC, CAN or SUB), so a cap could only ever show a
+// viewer bytes the owner's screen does not. Measured: past the 1 MiB cap an earlier version had, a
+// 900 KB tmux copy (one 1.2 MB OSC 52) put 151,428 chars of the clipboard's base64 on the viewer's
+// screen. And the filter keeps no buffer, so there is nothing for a cap to bound.
//
// Where a string starts and ends follows xterm.js 5.5's VT500 table (EscapeSequenceParser.ts), which
// renders the owner's node, the session host's emulator and the viewer page. Every rule below that
@@ -18,7 +21,7 @@
// - In ESCAPE, xterm executes C0 controls and ignores DEL without leaving it, restarts it on a second
// ESC, and takes an 8-bit introducer as a string start: `ESC \n ]52;…` is still an OSC 52.
// - An 8-bit introducer inside a string starts a new string of ITS kind (an OSC turned DCS no longer
-// ends at BEL), with a fresh count toward the cap.
+// ends at BEL).
// Where xterm ENDS a string on something this parser does not (CAN, SUB, other C1 controls, and a
// non-ASCII char, U+00A0 and up, inside SOS/PM/APC or a DCS before its final byte: ERROR → GROUND),
// the viewer misses a little text until the next terminator; it never sees more than the owner.
@@ -35,15 +38,20 @@
// viewer's parser can never be put into a string state, even by output that is dropped in part
// (drop-and-redraw) or starts on a keyframe.
//
-// It runs on the owner's process for every byte, once per viewer, so text is copied by the slice
-// between escapes rather than char by char: about 3x faster, measured on a captured tmux stream of
-// colored output (an ESC every ~8 bytes) on a loaded host: 22-39 MB/s char by char, 64-130 MB/s by
-// slice. Such a flood reaches a tmux 3.4 client at ~2.7 MB/s.
+// It runs on the owner's process for every byte, once per viewer, so text is copied, and a string's
+// payload skipped, by searching for the next char that matters rather than char by char: about 3x
+// faster on a captured tmux stream of colored output (an ESC every ~8 bytes) on a loaded host,
+// 22-39 MB/s char by char, 64-130 MB/s by slice. Such a flood reaches a tmux 3.4 client at
+// ~2.7 MB/s. One tmux copy of a large clipboard is a single OSC 52 of over a megabyte, whose
+// payload is skipped at ~390 MB/s (~53 MB/s char by char).
const ESC = '\x1b'
// Fast path: a chunk with none of these, read in text mode, is returned as it is.
const TEXT_SPECIAL = /[\x1b\x90\x98\x9d-\x9f]/
const INTRO_8_ANY = /[\x90\x98\x9d-\x9f]/
+// What can end a string: ESC, ST, an 8-bit introducer, and BEL for an OSC only.
+const STRING_STOP = /[\x1b\x90\x98\x9c-\x9f]/g
+const OSC_STOP = /[\x07\x1b\x90\x98\x9c-\x9f]/g
// OSC, DCS, SOS, PM, APC: `ESC ] P X ^ _` and their 8-bit forms.
function isIntro7(c: number): boolean {
@@ -64,7 +72,7 @@ export interface StreamFilterOptions {
* without it a join inside a string, or right after an ESC, sends the string's payload as text.
* Starts exactly as if an 8-bit DCS introducer had just been read: nothing is shown until the
* next ESC (an `ESC \` ST is dropped), ST or 8-bit introducer; BEL does not end it (the unknown
- * string may be a DCS or APC, where BEL is data); the cap counts from the join.
+ * string may be a DCS or APC, where BEL is data). Like every string, it has no length limit.
*/
midStream?: boolean
}
@@ -75,23 +83,25 @@ export interface StreamFilter {
reset(opts?: StreamFilterOptions): void
}
-export function createStreamFilter(maxStringChars = 1_048_576, opts?: StreamFilterOptions): StreamFilter {
+/**
+ * A string sequence is swallowed until its terminator however long it is: xterm has no length
+ * limit, so a cap would only show a viewer what the owner's screen does not, and nothing is
+ * buffered, so there is no memory for one to bound.
+ */
+export function createStreamFilter(opts?: StreamFilterOptions): StreamFilter {
// 'esc' is ESCAPE after a plain ESC; 'stringEsc' is ESCAPE after the ESC that ended a string,
// where `ESC \` is that string's ST and is dropped with it.
let mode: 'text' | 'esc' | 'string' | 'stringEsc' = 'text'
let osc = false
- let len = 0
const enterString = (introducer: number): void => {
mode = 'string'
osc = introducer === 0x5d || introducer === 0x9d
- len = 0
}
const start = (o: StreamFilterOptions | undefined): void => {
if (o?.midStream) enterString(0x90)
else {
mode = 'text'
osc = false
- len = 0
}
}
start(opts)
@@ -126,14 +136,16 @@ export function createStreamFilter(maxStringChars = 1_048_576, opts?: StreamFilt
else enterString(c)
i = j + 1
} else if (mode === 'string') {
- const c = chunk.charCodeAt(i++)
- if (c === 0x9c || (c === 0x07 && osc)) mode = 'text'
- else if (c === 0x1b) mode = 'stringEsc'
- else if (isIntro8(c)) enterString(c)
- else if (++len > maxStringChars) {
- mode = 'text'
- len = 0
- }
+ const stop = osc ? OSC_STOP : STRING_STOP
+ stop.lastIndex = i
+ const m = stop.exec(chunk)
+ // No terminator in this chunk: all of the rest is payload.
+ if (m === null) break
+ const c = chunk.charCodeAt(m.index)
+ i = m.index + 1
+ if (c === 0x1b) mode = 'stringEsc'
+ else if (c === 0x9c || c === 0x07) mode = 'text'
+ else enterString(c)
} else {
// ESCAPE. The ESC is held until the char that decides what it starts arrives.
const c = chunk.charCodeAt(i)
diff --git a/src/core/watch-link/stream-filter.xterm.test.ts b/src/core/watch-link/stream-filter.xterm.test.ts
index 46fea3a9b..153904546 100644
--- a/src/core/watch-link/stream-filter.xterm.test.ts
+++ b/src/core/watch-link/stream-filter.xterm.test.ts
@@ -45,7 +45,7 @@ const flipLetter = (c: string): string => LETTERS[(LETTERS.indexOf(c) + 1) % LET
// other C1, and non-ASCII (U+00A0 is where xterm's NON_ASCII_PRINTABLE starts).
const SINGLES = [
'\x00', '\t', '\n', '\r', BEL, '\x18', '\x1a', '\x19', '\x1c', '\x7f', ESC, ']', 'P', 'X', '^', '_', '[',
- '\\', ';', '0', '9', ' ', '(', 'é', ' ', '🙂', '\x85', '\x99', '\x9b', '\x9c', '\x9d', '\x90', '\x98',
+ '\\', ';', '0', '9', ' ', '(', 'é', '\u00a0', '🙂', '\x85', '\x99', '\x9b', '\x9c', '\x9d', '\x90', '\x98',
'\x9e', '\x9f'
]
const SNIPPETS = [
@@ -68,7 +68,7 @@ function randomStream(rand: (n: number) => number): string[] {
}
function filterInChunks(text: string, cuts: number[], midStream: boolean): string[] {
- const f = createStreamFilter(undefined, { midStream })
+ const f = createStreamFilter({ midStream })
const outs: string[] = []
let from = 0
for (const to of [...cuts, text.length]) {
@@ -124,6 +124,33 @@ describe('createStreamFilter against xterm 5.5', () => {
}
})
+ it("the model's walk is parse()'s: the lines of it that step() stands for are unchanged", () => {
+ // step() is not a copy but a statement of these lines of EscapeSequenceParser.parse()'s sync
+ // loop, so an upgrade that changes only parse() must reach this test too.
+ const parser = read(join(XTERM_PARSER, 'EscapeSequenceParser.ts'))
+ const loop = block(parser, '// continue with main sync loop', 'this.currentState = transition & TableAccess.TRANSITION_STATE_MASK;')
+ const actionCase = (action: string): string => {
+ const i = loop.indexOf(`case ParserAction.${action}:`)
+ const j = loop.indexOf('case ParserAction.', i + 1)
+ return i === -1 ? `<missing case ${action}>` : loop.slice(i, j === -1 ? loop.length : j)
+ }
+ const lines = (text: string, line: string): number => text.split('\n').filter((l) => l.trim() === line).length
+ // The table lookup and the state it moves to.
+ expect(lines(loop, 'transition = this._transitions.table[this.currentState << TableAccess.INDEX_STATE_SHIFT | (code < 0xa0 ? code : NON_ASCII_PRINTABLE)];')).toBe(1)
+ expect(loop.endsWith('this.currentState = transition & TableAccess.TRANSITION_STATE_MASK;')).toBe(true)
+ // An OSC or DCS ended by ESC goes to ESCAPE, not the table's GROUND; nothing else is patched.
+ const escPatch = 'if (code === 0x1b) transition |= ParserState.ESCAPE;'
+ expect(lines(actionCase('OSC_END'), escPatch)).toBe(1)
+ expect(lines(actionCase('DCS_UNHOOK'), escPatch)).toBe(1)
+ expect(lines(loop, escPatch)).toBe(2)
+ // The read-ahead loops stop exactly where the table would leave the state they are in.
+ expect(lines(actionCase('PRINT'), 'if (j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) {')).toBe(1)
+ expect(lines(actionCase('PRINT'), 'if (++j >= length || (code = data[j]) < 0x20 || (code > 0x7e && code < NON_ASCII_PRINTABLE)) {')).toBe(3)
+ expect(lines(actionCase('PARAM'), '} while (++i < length && (code = data[i]) > 0x2f && code < 0x3c);')).toBe(1)
+ expect(lines(actionCase('DCS_PUT'), 'if (j >= length || (code = data[j]) === 0x18 || code === 0x1a || code === 0x1b || (code > 0x7f && code < NON_ASCII_PRINTABLE)) {')).toBe(1)
+ expect(lines(actionCase('OSC_PUT'), 'if (j >= length || (code = data[j]) < 0x20 || (code > 0x7f && code < NON_ASCII_PRINTABLE)) {')).toBe(1)
+ })
+
it('nothing xterm hides reaches the output, from the start of a stream', () => {
const rand = rng(0xc0ffee)
let flips = 0