The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Reduce terminal parsing allocations for Latin-1 output (#673)
Deefa44a1fa92ca
diff --git a/desktop/test/terminal_byte_consumer_test.dart b/desktop/test/terminal_byte_consumer_test.dartnew file mode 100644index 000000000..7f798be94--- /dev/null+++ b/desktop/test/terminal_byte_consumer_test.dart@@ -0,0 +1,194 @@+import 'dart:convert';+import 'dart:math';++import 'package:flutter_test/flutter_test.dart';+import 'package:xterm/core.dart';+import 'package:xterm/src/utils/byte_consumer.dart';++List<int> drain(ByteConsumer consumer) => [+ for (; consumer.isNotEmpty;) consumer.consume(),+];++Object screen(Terminal terminal) => {+ 'cursor': [terminal.buffer.cursorX, terminal.buffer.cursorY],+ 'visible': terminal.cursorVisibleMode,+ 'alternate': terminal.isUsingAltBuffer,+ 'lines': [+ for (var i = 0; i < terminal.buffer.lines.length; i++)+ {+ 'data': terminal.buffer.lines[i].data.toList(),+ 'wrapped': terminal.buffer.lines[i].isWrapped,+ 'links': [+ for (var x = 0; x < terminal.buffer.lines[i].length; x++)+ terminal.buffer.lines[i].getHyperlink(x),+ ],+ },+ ],+};++void main() {+ test(+ 'Latin-1 includes controls and accents without changing code points',+ () {+ final expected = List.generate(256, (code) => code);+ final consumer = ByteConsumer()..add(String.fromCharCodes(expected));+ expect(consumer.length, 256);+ expect(drain(consumer), expected);+ consumer.rollback(256);+ expect(drain(consumer), expected);+ },+ );++ test('every non-surrogate BMP code point retains its value', () {+ final expected = [+ for (var code = 0; code <= 0xffff; code++)+ if (code < 0xd800 || code > 0xdfff) code,+ ];+ final consumer = ByteConsumer()..add(String.fromCharCodes(expected));+ expect(consumer.length, expected.length);+ expect(drain(consumer), expected);+ expect(consumer.totalConsumed, expected.length);+ expect(consumer.isEmpty, isTrue);+ });++ test(+ 'supplementary and unpaired surrogates keep per-chunk rune semantics',+ () {+ final chunks = [+ '',+ 'ASCII\x00\x1b[32m',+ 'Việt Nam 漢字 │',+ '😀🐙\u{10000}\u{10ffff}',+ '\ud800x\udfff',+ '\ud83d',+ '\ude00',+ ];+ final expected = chunks.expand((chunk) => chunk.runes).toList();+ final consumer = ByteConsumer();+ for (final chunk in chunks) {+ consumer.add(chunk);+ }+ expect(consumer.length, expected.length);+ expect(drain(consumer), expected);+ },+ );++ test('peek and rollback preserve rune positions across mixed blocks', () {+ final consumer = ByteConsumer()+ ..add('abc')+ ..add('😀é')+ ..add('漢\x1b[32m');+ final expected = 'abc😀é漢\x1b[32m'.runes.toList();+ expect(drain(consumer), expected);+ consumer.rollback(expected.length);+ expect(consumer.totalConsumed, 0);+ for (final code in expected) {+ final length = consumer.length;+ expect(consumer.peek(), code);+ expect(consumer.peek(), code);+ expect(consumer.length, length);+ expect(consumer.consume(), code);+ }+ consumer.rollbackTo(4);+ expect(drain(consumer), expected.sublist(expected.length - 4));+ });++ test('forgetting consumed blocks and resetting do not retain old text', () {+ final consumer = ByteConsumer()+ ..add('abc')+ ..add('😀xyz');+ for (var i = 0; i < 4; i++) {+ consumer.consume();+ }+ consumer.unrefConsumedBlocks();+ consumer.rollback();+ expect(consumer.peek(), 0x1f600);+ expect(drain(consumer), '😀xyz'.runes.toList());+ consumer.reset();+ expect(consumer.length, 0);+ expect(consumer.totalConsumed, 0);+ consumer.add('next');+ expect(drain(consumer), 'next'.codeUnits);+ });++ test('mixed append/consume/rewind operations match a flat rune stream', () {+ final random = Random(670);+ final consumer = ByteConsumer();+ final expected = <int>[];+ var position = 0;+ const chunks = ['abc', '漢é', '😀z', '\ud800', '\udfff', '\x1b[32m', ''];+ for (var step = 0; step < 5000; step++) {+ switch (random.nextInt(4)) {+ case 0:+ final chunk = chunks[random.nextInt(chunks.length)];+ consumer.add(chunk);+ expected.addAll(chunk.runes);+ case 1:+ if (position < expected.length) {+ expect(consumer.consume(), expected[position++]);+ }+ case 2:+ if (position > 0) {+ final count = random.nextInt(position + 1);+ consumer.rollback(count);+ position -= count;+ }+ case 3:+ if (position < expected.length) {+ expect(consumer.peek(), expected[position]);+ }+ }+ expect(consumer.totalConsumed, position);+ expect(consumer.length, expected.length - position);+ expect(consumer.isEmpty, position == expected.length);+ }+ expect(drain(consumer), expected.sublist(position));+ });++ test(+ 'fragmented ANSI, Unicode and hyperlinks preserve screen and replies',+ () {+ const text =+ 'plain Việt Nam 漢字 😀\r\n'+ '\x1b[32mgreen\x1b[0m\r\n'+ '\x1b]0;title 漢 😀\x07'+ '\x1b]8;;https://example.com/ảnh\x1b\\linked\x1b]8;;\x1b\\'+ '\x1bPtmux;\x1b\x1b]11;?\x07\x1b\\'+ '\x1b7\x1b[2;3Hwrite\x1b8\x1b[6n'+ '\x1b[?25l\x1b[?25h tail';+ final expectedReplies = <String>[];+ final expectedTitles = <String>[];+ final expected = Terminal(+ onOutput: expectedReplies.add,+ onTitleChange: expectedTitles.add,+ )..write(text);+ final scalars = text.runes.map(String.fromCharCode).toList();+ for (final chunkSize in [1, 2, 3, 7, 31, 128]) {+ final replies = <String>[];+ final titles = <String>[];+ final terminal = Terminal(+ onOutput: replies.add,+ onTitleChange: titles.add,+ );+ for (var start = 0; start < scalars.length; start += chunkSize) {+ terminal.write(+ scalars+ .sublist(start, min(start + chunkSize, scalars.length))+ .join(),+ );+ }+ expect(+ jsonEncode(screen(terminal)),+ jsonEncode(screen(expected)),+ reason: 'chunk size $chunkSize',+ );+ expect(replies, expectedReplies);+ expect(titles, expectedTitles);+ }+ expect(expected.buffer.getText(), contains('linked'));+ expect(expected.buffer.getText(), isNot(contains('tmux;')));+ expect(expectedTitles, ['title 漢 😀']);+ expect(expectedReplies, isNotEmpty);+ },+ );+}diff --git a/desktop/third_party/xterm/README.autonomous.md b/desktop/third_party/xterm/README.autonomous.mdindex 14931c7c7..6764ff514 100644--- a/desktop/third_party/xterm/README.autonomous.md+++ b/desktop/third_party/xterm/README.autonomous.md@@ -13,6 +13,20 @@ fix and the regression in `test/terminal_session_test.dart` passes against it.## Local patches+- **Single-byte output avoids a code-point list copy**+ (`lib/src/utils/byte_consumer.dart`). On native runtimes, Latin-1 strings use+ their immutable code-unit view, whose entries already equal their Unicode+ code points.+ Wider text and browser runtimes retain the original rune decoder: mixed+ queue-list types slowed Unicode parsing in the JavaScript benchmark.+ Queue positions, rollback,+ split escape sequences and supplementary characters keep their existing+ semantics. Regressions: `test/terminal_byte_consumer_test.dart`, also run in+ Chrome. Compare an earlier revision with the working tree using+ `python3 tool/benchmark_terminal_parser.py --flutter /path/to/flutter+ --baseline REVISION --output /tmp/parser-result.json` from `desktop/`.+ That benchmark measures parsing, not rendering or whole-app energy.+- **OSC 8 hyperlinks are kept per cell** (`lib/src/core/escape/parser.dart`,`lib/src/core/cursor.dart`, `lib/src/core/buffer/line.dart`,`lib/src/core/buffer/buffer.dart`, `lib/src/terminal.dart`). Upstream dropsdiff --git a/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart b/desktop/third_party/xterm/lib/src/utils/byte_consumer.dartindex 4a2ad382c..e68ab1273 100644--- a/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart+++ b/desktop/third_party/xterm/lib/src/utils/byte_consumer.dart@@ -1,6 +1,8 @@import 'dart:collection';class ByteConsumer {+ static final _wideCharacter = RegExp(r'[\u0100-\uffff]');+final _queue = ListQueue<List<int>>();final _consumed = ListQueue<List<int>>();@@ -13,7 +15,13 @@ class ByteConsumer {void add(String data) {if (data.isEmpty) return;- final runes = data.runes.toList(growable: false);+ // On native runtimes, Latin-1 code units avoid copying complete runes into+ // an int list. Wider text keeps the original decoder. JavaScript also+ // keeps it: mixing code-unit views and rune lists slows its Unicode path.+ final runes = const bool.fromEnvironment('dart.library.io') &&+ !_wideCharacter.hasMatch(data)+ ? data.codeUnits+ : data.runes.toList(growable: false);_queue.addLast(runes);_length += runes.length;}diff --git a/desktop/tool/benchmark_terminal_parser.py b/desktop/tool/benchmark_terminal_parser.pynew file mode 100644index 000000000..c55bbe930--- /dev/null+++ b/desktop/tool/benchmark_terminal_parser.py@@ -0,0 +1,78 @@+#!/usr/bin/env python3+"""Compare exact Git-baseline and working-tree xterm parsing in one Dart AOT VM."""+import argparse+import hashlib+import io+import json+import os+from pathlib import Path+import subprocess+import tarfile+import tempfile+from urllib.parse import urljoin++parser = argparse.ArgumentParser(description=__doc__)+parser.add_argument('--flutter', required=True, type=Path)+parser.add_argument('--baseline', required=True)+parser.add_argument('--output', required=True, type=Path)+args = parser.parse_args()+desktop = Path(__file__).resolve().parents[1]+output = args.output.resolve()+if output.exists():+ parser.error('Output already exists')+revision = subprocess.check_output(+ ['git', 'rev-parse', '--verify', f'{args.baseline}^{{commit}}'],+ cwd=desktop, text=True).strip()+archive = subprocess.check_output(+ ['git', 'archive', revision, 'desktop/third_party/xterm/lib'], cwd=desktop.parent)+fixture = desktop / 'tool/terminal_parser_benchmark.dart'+++def sources():+ return {str(p.relative_to(desktop)): hashlib.sha256(p.read_bytes()).hexdigest()+ for p in [fixture, Path(__file__).resolve(),+ *sorted((desktop / 'third_party/xterm/lib').rglob('*.dart'))]}+++before = sources()+with tempfile.TemporaryDirectory(prefix='harness-parser-') as directory:+ root = Path(directory)+ with tarfile.open(fileobj=io.BytesIO(archive)) as tar:+ tar.extractall(root, filter='data')+ baseline = root / 'desktop/third_party/xterm'+ # The separate package name lets the exact baseline and candidate coexist.+ # Rewrite import URIs only; parser/buffer executable code stays unchanged.+ for path in (baseline / 'lib').rglob('*.dart'):+ path.write_text(path.read_text().replace('package:xterm/', 'package:xterm_before/'))+ config_path = desktop / '.dart_tool/package_config.json'+ config = json.loads(config_path.read_text())+ for package in config['packages']:+ package['rootUri'] = urljoin(config_path.as_uri(), package['rootUri'])+ xterm = next(package for package in config['packages'] if package['name'] == 'xterm')+ config['packages'].append(dict(xterm, name='xterm_before', rootUri=baseline.as_uri() + '/'))+ packages = root / 'package_config.json'+ packages.write_text(json.dumps(config))+ driver = root / 'comparison.dart'+ driver.write_text(+ "import 'package:xterm_before/core.dart' as before;\n"+ f"import '{fixture.as_uri()}' as fixture;\n"+ 'void main() => fixture.runParserBenchmark(() => fixture.ParserTarget(\n'+ ' before.Terminal(maxLines: 1000, reflowEnabled: false)..resize(120, 30)));\n')+ dart = args.flutter.resolve() / 'bin/cache/dart-sdk/bin/dart'+ executable = root / 'comparison'+ subprocess.run([str(dart), 'compile', 'exe', f'--packages={packages}', str(driver),+ '-o', str(executable)], cwd=desktop, check=True, timeout=120)+ subprocess.run([str(executable)], cwd=desktop, check=True, timeout=120,+ env=dict(os.environ, HARNESS_PARSER_BENCH_OUTPUT=str(output)))+if sources() != before:+ raise RuntimeError('Benchmark inputs changed during measurement')+result = json.loads(output.read_text())+result.update({+ 'boundary': 'Dart AOT synchronous ANSI parsing only; excludes rendering, transport, daemon and battery',+ 'baselineCommit': revision,+ 'baselineArchiveSha256': hashlib.sha256(archive).hexdigest(),+ 'candidateAndFixtureSources': before,+ 'baselineAdaptation': 'package:xterm/ import URIs renamed to package:xterm_before/ only',+})+output.write_text(json.dumps(result, indent=2) + '\n')+print(output)diff --git a/desktop/tool/terminal_parser_benchmark.dart b/desktop/tool/terminal_parser_benchmark.dartnew file mode 100644index 000000000..a42dc2d1f--- /dev/null+++ b/desktop/tool/terminal_parser_benchmark.dart@@ -0,0 +1,128 @@+// Synchronous ANSI parsing only. No renderer, transport, daemon or battery data.+import 'dart:convert';+import 'dart:io';+import 'dart:math';++import 'package:xterm/core.dart';++class ParserTarget {+ ParserTarget(dynamic terminal)+ : write = terminal.write as void Function(String),+ snapshot = (() => _screen(terminal)) {+ terminal.onOutput = replies.add;+ terminal.onTitleChange = titles.add;+ }++ final void Function(String) write;+ final Object Function() snapshot;+ final replies = <String>[];+ final titles = <String>[];++ Object get state => [snapshot(), replies, titles];+}++// Dynamic here lets the comparison runner use a separate, unmodified baseline+// Terminal class. Snapshot work is outside the timed parsing loop.+Object _screen(dynamic terminal) => {+ 'grid': [terminal.viewWidth, terminal.viewHeight],+ 'cursor': [terminal.buffer.cursorX, terminal.buffer.cursorY],+ 'visible': terminal.cursorVisibleMode,+ 'alternate': terminal.isUsingAltBuffer,+ 'lines': [+ for (var i = 0; i < terminal.buffer.lines.length; i++)+ _line(terminal.buffer.lines[i]),+ ],+};++Object _line(dynamic line) => {+ 'data': line.data.toList(),+ 'wrapped': line.isWrapped,+ 'textVersion': line.textVersion,+ 'paintVersion': line.paintVersion,+ 'links': [for (var x = 0; x < line.length; x++) line.getHyperlink(x)],+};++void runParserBenchmark(ParserTarget Function() createBaseline) {+ final results = <String, Object>{};+ var comparisons = 0;+ String rows(String text) =>+ '\x1b7\x1b[H${List.generate(8, (row) => '\x1b[2K\x1b[32m$row: $text\x1b[0m\r\n').join()}\x1b8';+ for (final (name, text, chunkSize) in [+ ('ascii_short', '\r\x1b[2KAgent update: tests passed', 100000),+ ('ascii_screen', rows('Agent output ${'x' * 80}'), 100000),+ ('latin1_screen', rows('résumé café £ ${'x' * 60}'), 100000),+ ('bmp_screen', rows('Việt Nam 漢字 │ ${'x' * 40}'), 100000),+ ('emoji_screen', rows('Test 😀 🐙 𐐷 ${'x' * 40}'), 100000),+ (+ 'fragmented_ansi',+ '\x1b[H\x1b]8;;https://example.com/ảnh\x1b\\Label 漢 😀'+ '\x1b]8;;\x1b\\\x1bPtmux;\x1b\x1b]11;?\x07\x1b\\\x1b[6n done',+ 7,+ ),+ ]) {+ final chunks = [+ for (var i = 0; i < text.length; i += chunkSize)+ text.substring(i, min(i + chunkSize, text.length)),+ ];+ final targets = [+ createBaseline(),+ ParserTarget(+ Terminal(maxLines: 1000, reflowEnabled: false)..resize(120, 30),+ ),+ ];+ void check() {+ if (jsonEncode(targets[0].state) != jsonEncode(targets[1].state)) {+ throw StateError('$name: parser states or replies differ');+ }+ comparisons++;+ }++ for (var i = 0; i < 5; i++) {+ for (final chunk in chunks) {+ for (final target in targets) {+ target.write(chunk);+ }+ check();+ }+ }+ void batch(int version, int count) {+ final write = targets[version].write;+ for (var i = 0; i < count; i++) {+ for (final chunk in chunks) {+ write(chunk);+ }+ }+ }++ for (var version = 0; version < 2; version++) {+ batch(version, 1000);+ }+ final samples = [<int>[], <int>[]];+ for (var sample = 0; sample < 7; sample++) {+ for (final version in sample.isEven ? [0, 1] : [1, 0]) {+ final watch = Stopwatch()..start();+ batch(version, 3000);+ watch.stop();+ samples[version].add(watch.elapsedMicroseconds);+ }+ }+ check();+ results[name] = {+ 'chunksPerIteration': chunks.length,+ 'codeUnitsPerIteration': text.length,+ 'iterations': 3000,+ 'baselineMicros': samples[0],+ 'candidateMicros': samples[1],+ };+ }+ final output = File(Platform.environment['HARNESS_PARSER_BENCH_OUTPUT']!);+ if (output.existsSync()) throw StateError('Output already exists');+ output.writeAsStringSync(+ jsonEncode({+ 'schema': 1,+ 'dartVersion': Platform.version,+ 'stateComparisons': comparisons,+ 'results': results,+ }),+ );+}