Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Reduce terminal parsing allocations for Latin-1 output (#673)

Deefa44a1fa92ca
diff --git a/desktop/test/terminal_byte_consumer_test.dart b/desktop/test/terminal_byte_consumer_test.dart
new file mode 100644
index 000000000..7f798be94
--- /dev/null
+++ b/desktop/test/terminal_byte_consumer_test.dart
@@ -0,0 +1,194 @@
+import 'dart:convert';
+import 'dart:math';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:xterm/core.dart';
+import 'package:xterm/src/utils/byte_consumer.dart';
+
+List<int> drain(ByteConsumer consumer) => [
+ for (; consumer.isNotEmpty;) consumer.consume(),
+];
+
+Object screen(Terminal terminal) => {
+ 'cursor': [terminal.buffer.cursorX, terminal.buffer.cursorY],
+ 'visible': terminal.cursorVisibleMode,
+ 'alternate': terminal.isUsingAltBuffer,
+ 'lines': [
+ for (var i = 0; i < terminal.buffer.lines.length; i++)
+ {
+ 'data': terminal.buffer.lines[i].data.toList(),
+ 'wrapped': terminal.buffer.lines[i].isWrapped,
+ 'links': [
+ for (var x = 0; x < terminal.buffer.lines[i].length; x++)
+ terminal.buffer.lines[i].getHyperlink(x),
+ ],
+ },
+ ],
+};
+
+void main() {
+ test(
+ 'Latin-1 includes controls and accents without changing code points',
+ () {
+ final expected = List.generate(256, (code) => code);
+ final consumer = ByteConsumer()..add(String.fromCharCodes(expected));
+ expect(consumer.length, 256);
+ expect(drain(consumer), expected);
+ consumer.rollback(256);
+ expect(drain(consumer), expected);
+ },
+ );
+
+ test('every non-surrogate BMP code point retains its value', () {
+ final expected = [
+ for (var code = 0; code <= 0xffff; code++)
+ if (code < 0xd800 || code > 0xdfff) code,
+ ];
+ final consumer = ByteConsumer()..add(String.fromCharCodes(expected));
+ expect(consumer.length, expected.length);
+ expect(drain(consumer), expected);
+ expect(consumer.totalConsumed, expected.length);
+ expect(consumer.isEmpty, isTrue);
+ });
+
+ test(
+ 'supplementary and unpaired surrogates keep per-chunk rune semantics',
+ () {
+ final chunks = [
+ '',
+ 'ASCII\x00\x1b[32m',
+ 'Việt Nam 漢字 │',
+ '😀🐙\u{10000}\u{10ffff}',
+ '\ud800x\udfff',
+ '\ud83d',
+ '\ude00',
+ ];
+ final expected = chunks.expand((chunk) => chunk.runes).toList();
+ final consumer = ByteConsumer();
+ for (final chunk in chunks) {
+ consumer.add(chunk);
+ }
+ expect(consumer.length, expected.length);
+ expect(drain(consumer), expected);
+ },
+ );
+
+ test('peek and rollback preserve rune positions across mixed blocks', () {
+ final consumer = ByteConsumer()
+ ..add('abc')
+ ..add('😀é')
+ ..add('漢\x1b[32m');
+ final expected = 'abc😀é漢\x1b[32m'.runes.toList();
+ expect(drain(consumer), expected);
+ consumer.rollback(expected.length);
+ expect(consumer.totalConsumed, 0);
+ for (final code in expected) {
+ final length = consumer.length;
+ expect(consumer.peek(), code);
+ expect(consumer.peek(), code);
+ expect(consumer.length, length);
+ expect(consumer.consume(), code);
+ }
+ consumer.rollbackTo(4);
+ expect(drain(consumer), expected.sublist(expected.length - 4));
+ });
+
+ test('forgetting consumed blocks and resetting do not retain old text', () {
+ final consumer = ByteConsumer()
+ ..add('abc')
+ ..add('😀xyz');
+ for (var i = 0; i < 4; i++) {
+ consumer.consume();
+ }
+ consumer.unrefConsumedBlocks();
+ consumer.rollback();
+ expect(consumer.peek(), 0x1f600);
+ expect(drain(consumer), '😀xyz'.runes.toList());
+ consumer.reset();
+ expect(consumer.length, 0);
+ expect(consumer.totalConsumed, 0);
+ consumer.add('next');
+ expect(drain(consumer), 'next'.codeUnits);
+ });
+
+ test('mixed append/consume/rewind operations match a flat rune stream', () {
+ final random = Random(670);
+ final consumer = ByteConsumer();
+ final expected = <int>[];
+ var position = 0;
+ const chunks = ['abc', '漢é', '😀z', '\ud800', '\udfff', '\x1b[32m', ''];
+ for (var step = 0; step < 5000; step++) {
+ switch (random.nextInt(4)) {
+ case 0:
+ final chunk = chunks[random.nextInt(chunks.length)];
+ consumer.add(chunk);
+ expected.addAll(chunk.runes);
+ case 1:
+ if (position < expected.length) {
+ expect(consumer.consume(), expected[position++]);
+ }
+ case 2:
+ if (position > 0) {
+ final count = random.nextInt(position + 1);
+ consumer.rollback(count);
+ position -= count;
+ }
+ case 3:
+ if (position < expected.length) {
+ expect(consumer.peek(), expected[position]);
+ }
+ }
+ expect(consumer.totalConsumed, position);
+ expect(consumer.length, expected.length - position);
+ expect(consumer.isEmpty, position == expected.length);
+ }
+ expect(drain(consumer), expected.sublist(position));
+ });
+
+ test(
+ 'fragmented ANSI, Unicode and hyperlinks preserve screen and replies',
+ () {
+ const text =
+ 'plain Việt Nam 漢字 😀\r\n'
+ '\x1b[32mgreen\x1b[0m\r\n'
+ '\x1b]0;title 漢 😀\x07'
+ '\x1b]8;;https://example.com/ảnh\x1b\\linked\x1b]8;;\x1b\\'
+ '\x1bPtmux;\x1b\x1b]11;?\x07\x1b\\'
+ '\x1b7\x1b[2;3Hwrite\x1b8\x1b[6n'
+ '\x1b[?25l\x1b[?25h tail';
+ final expectedReplies = <String>[];
+ final expectedTitles = <String>[];
+ final expected = Terminal(
+ onOutput: expectedReplies.add,
+ onTitleChange: expectedTitles.add,
+ )..write(text);
+ final scalars = text.runes.map(String.fromCharCode).toList();
+ for (final chunkSize in [1, 2, 3, 7, 31, 128]) {
+ final replies = <String>[];
+ final titles = <String>[];
+ final terminal = Terminal(
+ onOutput: replies.add,
+ onTitleChange: titles.add,
+ );
+ for (var start = 0; start < scalars.length; start += chunkSize) {
+ terminal.write(
+ scalars
+ .sublist(start, min(start + chunkSize, scalars.length))
+ .join(),
+ );
+ }
+ expect(
+ jsonEncode(screen(terminal)),
+ jsonEncode(screen(expected)),
+ reason: 'chunk size $chunkSize',
+ );
+ expect(replies, expectedReplies);
+ expect(titles, expectedTitles);
+ }
+ expect(expected.buffer.getText(), contains('linked'));
+ expect(expected.buffer.getText(), isNot(contains('tmux;')));
+ expect(expectedTitles, ['title 漢 😀']);
+ expect(expectedReplies, isNotEmpty);
+ },
+ );
+}
diff --git a/desktop/third_party/xterm/README.autonomous.md b/desktop/third_party/xterm/README.autonomous.md
index 14931c7c7..6764ff514 100644
--- a/desktop/third_party/xterm/README.autonomous.md
+++ b/desktop/third_party/xterm/README.autonomous.md
@@ -13,6 +13,20 @@ fix and the regression in `test/terminal_session_test.dart` passes against it.
## Local patches
+- **Single-byte output avoids a code-point list copy**
+ (`lib/src/utils/byte_consumer.dart`). On native runtimes, Latin-1 strings use
+ their immutable code-unit view, whose entries already equal their Unicode
+ code points.
+ Wider text and browser runtimes retain the original rune decoder: mixed
+ queue-list types slowed Unicode parsing in the JavaScript benchmark.
+ Queue positions, rollback,
+ split escape sequences and supplementary characters keep their existing
+ semantics. Regressions: `test/terminal_byte_consumer_test.dart`, also run in
+ Chrome. Compare an earlier revision with the working tree using
+ `python3 tool/benchmark_terminal_parser.py --flutter /path/to/flutter
+ --baseline REVISION --output /tmp/parser-result.json` from `desktop/`.
+ That benchmark measures parsing, not rendering or whole-app energy.
+
- **OSC 8 hyperlinks are kept per cell** (`lib/src/core/escape/parser.dart`,
`lib/src/core/cursor.dart`, `lib/src/core/buffer/line.dart`,
`lib/src/core/buffer/buffer.dart`, `lib/src/terminal.dart`). Upstream drops
diff --git a/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart b/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart
index 4a2ad382c..e68ab1273 100644
--- a/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart
+++ b/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart
@@ -1,6 +1,8 @@
import 'dart:collection';
class ByteConsumer {
+ static final _wideCharacter = RegExp(r'[\u0100-\uffff]');
+
final _queue = ListQueue<List<int>>();
final _consumed = ListQueue<List<int>>();
@@ -13,7 +15,13 @@ class ByteConsumer {
void add(String data) {
if (data.isEmpty) return;
- final runes = data.runes.toList(growable: false);
+ // On native runtimes, Latin-1 code units avoid copying complete runes into
+ // an int list. Wider text keeps the original decoder. JavaScript also
+ // keeps it: mixing code-unit views and rune lists slows its Unicode path.
+ final runes = const bool.fromEnvironment('dart.library.io') &&
+ !_wideCharacter.hasMatch(data)
+ ? data.codeUnits
+ : data.runes.toList(growable: false);
_queue.addLast(runes);
_length += runes.length;
}
diff --git a/desktop/tool/benchmark_terminal_parser.py b/desktop/tool/benchmark_terminal_parser.py
new file mode 100644
index 000000000..c55bbe930
--- /dev/null
+++ b/desktop/tool/benchmark_terminal_parser.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+"""Compare exact Git-baseline and working-tree xterm parsing in one Dart AOT VM."""
+import argparse
+import hashlib
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import tarfile
+import tempfile
+from urllib.parse import urljoin
+
+parser = argparse.ArgumentParser(description=__doc__)
+parser.add_argument('--flutter', required=True, type=Path)
+parser.add_argument('--baseline', required=True)
+parser.add_argument('--output', required=True, type=Path)
+args = parser.parse_args()
+desktop = Path(__file__).resolve().parents[1]
+output = args.output.resolve()
+if output.exists():
+ parser.error('Output already exists')
+revision = subprocess.check_output(
+ ['git', 'rev-parse', '--verify', f'{args.baseline}^{{commit}}'],
+ cwd=desktop, text=True).strip()
+archive = subprocess.check_output(
+ ['git', 'archive', revision, 'desktop/third_party/xterm/lib'], cwd=desktop.parent)
+fixture = desktop / 'tool/terminal_parser_benchmark.dart'
+
+
+def sources():
+ return {str(p.relative_to(desktop)): hashlib.sha256(p.read_bytes()).hexdigest()
+ for p in [fixture, Path(__file__).resolve(),
+ *sorted((desktop / 'third_party/xterm/lib').rglob('*.dart'))]}
+
+
+before = sources()
+with tempfile.TemporaryDirectory(prefix='harness-parser-') as directory:
+ root = Path(directory)
+ with tarfile.open(fileobj=io.BytesIO(archive)) as tar:
+ tar.extractall(root, filter='data')
+ baseline = root / 'desktop/third_party/xterm'
+ # The separate package name lets the exact baseline and candidate coexist.
+ # Rewrite import URIs only; parser/buffer executable code stays unchanged.
+ for path in (baseline / 'lib').rglob('*.dart'):
+ path.write_text(path.read_text().replace('package:xterm/', 'package:xterm_before/'))
+ config_path = desktop / '.dart_tool/package_config.json'
+ config = json.loads(config_path.read_text())
+ for package in config['packages']:
+ package['rootUri'] = urljoin(config_path.as_uri(), package['rootUri'])
+ xterm = next(package for package in config['packages'] if package['name'] == 'xterm')
+ config['packages'].append(dict(xterm, name='xterm_before', rootUri=baseline.as_uri() + '/'))
+ packages = root / 'package_config.json'
+ packages.write_text(json.dumps(config))
+ driver = root / 'comparison.dart'
+ driver.write_text(
+ "import 'package:xterm_before/core.dart' as before;\n"
+ f"import '{fixture.as_uri()}' as fixture;\n"
+ 'void main() => fixture.runParserBenchmark(() => fixture.ParserTarget(\n'
+ ' before.Terminal(maxLines: 1000, reflowEnabled: false)..resize(120, 30)));\n')
+ dart = args.flutter.resolve() / 'bin/cache/dart-sdk/bin/dart'
+ executable = root / 'comparison'
+ subprocess.run([str(dart), 'compile', 'exe', f'--packages={packages}', str(driver),
+ '-o', str(executable)], cwd=desktop, check=True, timeout=120)
+ subprocess.run([str(executable)], cwd=desktop, check=True, timeout=120,
+ env=dict(os.environ, HARNESS_PARSER_BENCH_OUTPUT=str(output)))
+if sources() != before:
+ raise RuntimeError('Benchmark inputs changed during measurement')
+result = json.loads(output.read_text())
+result.update({
+ 'boundary': 'Dart AOT synchronous ANSI parsing only; excludes rendering, transport, daemon and battery',
+ 'baselineCommit': revision,
+ 'baselineArchiveSha256': hashlib.sha256(archive).hexdigest(),
+ 'candidateAndFixtureSources': before,
+ 'baselineAdaptation': 'package:xterm/ import URIs renamed to package:xterm_before/ only',
+})
+output.write_text(json.dumps(result, indent=2) + '\n')
+print(output)
diff --git a/desktop/tool/terminal_parser_benchmark.dart b/desktop/tool/terminal_parser_benchmark.dart
new file mode 100644
index 000000000..a42dc2d1f
--- /dev/null
+++ b/desktop/tool/terminal_parser_benchmark.dart
@@ -0,0 +1,128 @@
+// Synchronous ANSI parsing only. No renderer, transport, daemon or battery data.
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math';
+
+import 'package:xterm/core.dart';
+
+class ParserTarget {
+ ParserTarget(dynamic terminal)
+ : write = terminal.write as void Function(String),
+ snapshot = (() => _screen(terminal)) {
+ terminal.onOutput = replies.add;
+ terminal.onTitleChange = titles.add;
+ }
+
+ final void Function(String) write;
+ final Object Function() snapshot;
+ final replies = <String>[];
+ final titles = <String>[];
+
+ Object get state => [snapshot(), replies, titles];
+}
+
+// Dynamic here lets the comparison runner use a separate, unmodified baseline
+// Terminal class. Snapshot work is outside the timed parsing loop.
+Object _screen(dynamic terminal) => {
+ 'grid': [terminal.viewWidth, terminal.viewHeight],
+ 'cursor': [terminal.buffer.cursorX, terminal.buffer.cursorY],
+ 'visible': terminal.cursorVisibleMode,
+ 'alternate': terminal.isUsingAltBuffer,
+ 'lines': [
+ for (var i = 0; i < terminal.buffer.lines.length; i++)
+ _line(terminal.buffer.lines[i]),
+ ],
+};
+
+Object _line(dynamic line) => {
+ 'data': line.data.toList(),
+ 'wrapped': line.isWrapped,
+ 'textVersion': line.textVersion,
+ 'paintVersion': line.paintVersion,
+ 'links': [for (var x = 0; x < line.length; x++) line.getHyperlink(x)],
+};
+
+void runParserBenchmark(ParserTarget Function() createBaseline) {
+ final results = <String, Object>{};
+ var comparisons = 0;
+ String rows(String text) =>
+ '\x1b7\x1b[H${List.generate(8, (row) => '\x1b[2K\x1b[32m$row: $text\x1b[0m\r\n').join()}\x1b8';
+ for (final (name, text, chunkSize) in [
+ ('ascii_short', '\r\x1b[2KAgent update: tests passed', 100000),
+ ('ascii_screen', rows('Agent output ${'x' * 80}'), 100000),
+ ('latin1_screen', rows('résumé café £ ${'x' * 60}'), 100000),
+ ('bmp_screen', rows('Việt Nam 漢字 │ ${'x' * 40}'), 100000),
+ ('emoji_screen', rows('Test 😀 🐙 𐐷 ${'x' * 40}'), 100000),
+ (
+ 'fragmented_ansi',
+ '\x1b[H\x1b]8;;https://example.com/ảnh\x1b\\Label 漢 😀'
+ '\x1b]8;;\x1b\\\x1bPtmux;\x1b\x1b]11;?\x07\x1b\\\x1b[6n done',
+ 7,
+ ),
+ ]) {
+ final chunks = [
+ for (var i = 0; i < text.length; i += chunkSize)
+ text.substring(i, min(i + chunkSize, text.length)),
+ ];
+ final targets = [
+ createBaseline(),
+ ParserTarget(
+ Terminal(maxLines: 1000, reflowEnabled: false)..resize(120, 30),
+ ),
+ ];
+ void check() {
+ if (jsonEncode(targets[0].state) != jsonEncode(targets[1].state)) {
+ throw StateError('$name: parser states or replies differ');
+ }
+ comparisons++;
+ }
+
+ for (var i = 0; i < 5; i++) {
+ for (final chunk in chunks) {
+ for (final target in targets) {
+ target.write(chunk);
+ }
+ check();
+ }
+ }
+ void batch(int version, int count) {
+ final write = targets[version].write;
+ for (var i = 0; i < count; i++) {
+ for (final chunk in chunks) {
+ write(chunk);
+ }
+ }
+ }
+
+ for (var version = 0; version < 2; version++) {
+ batch(version, 1000);
+ }
+ final samples = [<int>[], <int>[]];
+ for (var sample = 0; sample < 7; sample++) {
+ for (final version in sample.isEven ? [0, 1] : [1, 0]) {
+ final watch = Stopwatch()..start();
+ batch(version, 3000);
+ watch.stop();
+ samples[version].add(watch.elapsedMicroseconds);
+ }
+ }
+ check();
+ results[name] = {
+ 'chunksPerIteration': chunks.length,
+ 'codeUnitsPerIteration': text.length,
+ 'iterations': 3000,
+ 'baselineMicros': samples[0],
+ 'candidateMicros': samples[1],
+ };
+ }
+ final output = File(Platform.environment['HARNESS_PARSER_BENCH_OUTPUT']!);
+ if (output.existsSync()) throw StateError('Output already exists');
+ output.writeAsStringSync(
+ jsonEncode({
+ 'schema': 1,
+ 'dartVersion': Platform.version,
+ 'stateComparisons': comparisons,
+ 'results': results,
+ }),
+ );
+}