A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
fix(tui): scan standard ANSI control sequences (#460)
Sebastian Müller33c8df923d21
diff --git a/packages/tui/CHANGELOG.md b/packages/tui/CHANGELOG.mdindex fc1434aa30..0e8f4c4ce4 100644--- a/packages/tui/CHANGELOG.md+++ b/packages/tui/CHANGELOG.md@@ -6,6 +6,7 @@- Removed the fullscreen image fallback instruction and kept fullscreen overlays from emitting terminal graphics ([#437](https://github.com/PrimeIntellect-ai/prime-agent/pull/437) by [@snimu](https://github.com/snimu)).- Changed the fullscreen follow shortcut from `Alt+Down` to `Ctrl+Shift+Down` for more reliable terminal input ([ENG-4684](https://linear.app/primeintellect/issue/ENG-4684/altdown-doesnt-work)).+- Fixed terminal width, slicing, and wrapping helpers misreading standard CSI and DCS escape sequences.- Fixed autocomplete popups overlapping the prompt's top edge.## [0.3.1] - 2026-07-15diff --git a/packages/tui/src/selection-metadata.ts b/packages/tui/src/selection-metadata.tsindex 329ba3337f..8922243da1 100644--- a/packages/tui/src/selection-metadata.ts+++ b/packages/tui/src/selection-metadata.ts@@ -1,4 +1,4 @@-import { extractAnsiCode, visibleWidth } from "./utils.js";+import { createAnsiCodeExtractor, visibleWidth } from "./utils.js";const TABLE_MARKER_PREFIX = "\x1b_pi:table:";const TABLE_START_MARKER = `${TABLE_MARKER_PREFIX}start\x07`;@@ -99,9 +99,10 @@ export function extractTableCellSelectionRegions(let clean = "";let activeCell: (CellMarker & { col: number }) | null = null;let offset = 0;+ const extractAnsi = createAnsiCodeExtractor(source);while (offset < source.length) {- const ansi = extractAnsiCode(source, offset);+ const ansi = extractAnsi(offset);if (!ansi) {clean += source[offset];offset++;diff --git a/packages/tui/src/utils.ts b/packages/tui/src/utils.tsindex 3fbdc10e37..ba02305e05 100644--- a/packages/tui/src/utils.ts+++ b/packages/tui/src/utils.ts@@ -77,9 +77,10 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: stringlet width = 0;let i = 0;let pendingAnsi = "";+ const extractAnsi = createAnsiCodeExtractor(text);while (i < text.length) {- const ansi = extractAnsiCode(text, i);+ const ansi = extractAnsi(i);if (ansi) {pendingAnsi += ansi.code;i += ansi.length;@@ -102,7 +103,7 @@ function truncateFragmentToWidth(text: string, maxWidth: number): { text: stringlet end = i;while (end < text.length && text[end] !== "\t") {- const nextAnsi = extractAnsiCode(text, end);+ const nextAnsi = extractAnsi(end);if (nextAnsi) {break;}@@ -224,8 +225,9 @@ export function visibleWidth(str: string): number {// and APC sequences like CURSOR_MARKER.let stripped = "";let i = 0;+ const extractAnsi = createAnsiCodeExtractor(clean);while (i < clean.length) {- const ansi = extractAnsiCode(clean, i);+ const ansi = extractAnsi(i);if (ansi) {i += ansi.length;continue;@@ -265,9 +267,10 @@ export function visibleContentSpan(line: string, maxWidth: number): { from: numblet to = -1;let currentCol = 0;let i = 0;+ const extractAnsi = createAnsiCodeExtractor(line);while (i < line.length && currentCol < limit) {- const ansi = extractAnsiCode(line, i);+ const ansi = extractAnsi(i);if (ansi) {i += ansi.length;continue;@@ -280,7 +283,7 @@ export function visibleContentSpan(line: string, maxWidth: number): { from: numb}let textEnd = i;- while (textEnd < line.length && line[textEnd] !== "\t" && !extractAnsiCode(line, textEnd)) {+ while (textEnd < line.length && line[textEnd] !== "\t" && !extractAnsi(textEnd)) {textEnd++;}@@ -329,49 +332,130 @@ export function normalizeTerminalOutput(str: string): string {return normalized;}-/**- * Extract ANSI escape sequences from a string at the given position.- */-export function extractAnsiCode(str: string, pos: number): { code: string; length: number } | null {- if (pos >= str.length || str[pos] !== "\x1b") return null;+interface AnsiCode {+ code: string;+ length: number;+}- const next = str[pos + 1];+type ControlStringEnds = Map<number, number | null>;- // CSI sequence: ESC [ ... m/G/K/H/J- if (next === "[") {- let j = pos + 2;- while (j < str.length && !/[mGKHJ]/.test(str[j]!)) j++;- if (j < str.length) return { code: str.substring(pos, j + 1), length: j + 1 - pos };- return null;+interface AnsiScanState {+ controlStringEnds?: ControlStringEnds;+}++function cacheControlStringEnds(str: string, from: number, ends: ControlStringEnds): void {+ let nextBel = -1;+ let nextSt = -1;++ for (let i = str.length - 1; i >= from; i--) {+ if (str[i] === "\x07") {+ nextBel = i;+ }+ if (str[i] !== "\x1b") {+ continue;+ }++ const next = str[i + 1];+ if (next === "\\") {+ nextSt = i;+ } else if (next === "]" || next === "_") {+ if (nextBel !== -1 && (nextSt === -1 || nextBel < nextSt)) {+ ends.set(i, nextBel + 1);+ } else {+ ends.set(i, nextSt === -1 ? null : nextSt + 2);+ }+ } else if (next === "P" || next === "^" || next === "X") {+ ends.set(i, nextSt === -1 ? null : nextSt + 2);+ }}+}- // OSC sequence: ESC ] ... BEL or ESC ] ... ST (ESC \)- // Used for hyperlinks (OSC 8), window titles, etc.- if (next === "]") {- let j = pos + 2;- while (j < str.length) {- if (str[j] === "\x07") return { code: str.substring(pos, j + 1), length: j + 1 - pos };- if (str[j] === "\x1b" && str[j + 1] === "\\") return { code: str.substring(pos, j + 2), length: j + 2 - pos };- j++;+function extractControlStringEnd(str: string, pos: number, allowBel: boolean, state?: AnsiScanState): number | null {+ if (state?.controlStringEnds?.has(pos)) {+ const end = state.controlStringEnds.get(pos);+ return end ?? null;+ }++ let j = pos + 2;+ while (j < str.length) {+ if (allowBel && str[j] === "\x07") {+ return j + 1;}- return null;+ if (str[j] === "\x1b" && str[j + 1] === "\\") {+ return j + 2;+ }+ j++;+ }+ if (state) {+ const ends: ControlStringEnds = new Map();+ cacheControlStringEnds(str, pos, ends);+ state.controlStringEnds = ends;}+ return null;+}- // APC sequence: ESC _ ... BEL or ESC _ ... ST (ESC \)- // Used for cursor marker and application-specific commands- if (next === "_") {+function extractAnsiEndAt(str: string, pos: number, state?: AnsiScanState): number | null {+ if (pos >= str.length || str[pos] !== "\x1b") return null;++ const next = str[pos + 1];++ // CSI: parameter bytes, then intermediate bytes, then one final byte.+ if (next === "[") {let j = pos + 2;+ let hasIntermediate = false;while (j < str.length) {- if (str[j] === "\x07") return { code: str.substring(pos, j + 1), length: j + 1 - pos };- if (str[j] === "\x1b" && str[j + 1] === "\\") return { code: str.substring(pos, j + 2), length: j + 2 - pos };- j++;+ const byte = str.charCodeAt(j);+ if (byte >= 0x30 && byte <= 0x3f && !hasIntermediate) {+ j++;+ continue;+ }+ if (byte >= 0x20 && byte <= 0x2f) {+ hasIntermediate = true;+ j++;+ continue;+ }+ if (byte >= 0x40 && byte <= 0x7e) {+ return j + 1;+ }+ return null;}return null;}+ // OSC uses BEL or ST. APC also accepts BEL for existing private markers.+ if (next === "]" || next === "_") {+ return extractControlStringEnd(str, pos, true, state);+ }++ // DCS, PM, and SOS are terminated by ST.+ if (next === "P" || next === "^" || next === "X") {+ return extractControlStringEnd(str, pos, false, state);+ }+return null;}+function extractAnsiCodeAt(str: string, pos: number, state?: AnsiScanState): AnsiCode | null {+ const end = extractAnsiEndAt(str, pos, state);+ return end === null ? null : { code: str.substring(pos, end), length: end - pos };+}++/** Extract an ANSI escape sequence from a string at the given position. */+export function extractAnsiCode(str: string, pos: number): AnsiCode | null {+ return extractAnsiCodeAt(str, pos);+}++/** Create a scanner whose malformed-control-string cache is released after this string scan. */+export function createAnsiCodeExtractor(str: string): (pos: number) => AnsiCode | null {+ const state: AnsiScanState = {};+ return (pos) => extractAnsiCodeAt(str, pos, state);+}++function createAnsiEndExtractor(str: string): (pos: number) => number | null {+ const state: AnsiScanState = {};+ return (pos) => extractAnsiEndAt(str, pos, state);+}+type Osc8Terminator = "\x07" | "\x1b\\";interface ActiveHyperlink {@@ -635,8 +719,9 @@ class AnsiCodeTracker {function updateTrackerFromText(text: string, tracker: AnsiCodeTracker): void {let i = 0;+ const extractAnsi = createAnsiCodeExtractor(text);while (i < text.length) {- const ansiResult = extractAnsiCode(text, i);+ const ansiResult = extractAnsi(i);if (ansiResult) {tracker.process(ansiResult.code);i += ansiResult.length;@@ -655,9 +740,10 @@ function splitIntoTokensWithAnsi(text: string): string[] {let pendingAnsi = ""; // ANSI codes waiting to be attached to next visible contentlet inWhitespace = false;let i = 0;+ const extractAnsi = createAnsiCodeExtractor(text);while (i < text.length) {- const ansiResult = extractAnsiCode(text, i);+ const ansiResult = extractAnsi(i);if (ansiResult) {// Hold ANSI codes separately - they'll be attached to the next visible charpendingAnsi += ansiResult.code;@@ -810,10 +896,37 @@ function wrapSingleLine(line: string, width: number): string[] {}const PUNCTUATION_REGEX = /[(){}[\]<>.,;:'"!?+\-=*/\\|&%^$#@~`]/;+// Fast path for the common CSI form. The shared scanner handles the wider CSI+// grammar, control strings, malformed sequences, and ordinary two-byte escapes.+const COMMON_CSI_REGEX = /\x1b\[[0-9;:?<=>]*[\x40-\x7e]/g;/** Remove all escape sequences (CSI, OSC, DCS/APC, two-char) leaving plain text. */export function stripAnsi(str: string): string {- return str.replace(/\x1b(?:\[[0-9;:?<=>]*[\x40-\x7e]|\][^\x07\x1b]*(?:\x07|\x1b\\)|[P_^X][^\x1b]*\x1b\\|.)/g, "");+ if (!str.includes("\x1b")) return str;++ const input = str.replace(COMMON_CSI_REGEX, "");+ let escapeIndex = input.indexOf("\x1b");+ if (escapeIndex === -1) return input;++ const result: string[] = [];+ let plainStart = 0;+ const extractAnsiEnd = createAnsiEndExtractor(input);+ while (escapeIndex !== -1) {+ const ansiEnd = extractAnsiEnd(escapeIndex);+ if (ansiEnd !== null) {+ if (plainStart < escapeIndex) result.push(input.slice(plainStart, escapeIndex));+ plainStart = ansiEnd;+ } else {+ const next = input.charCodeAt(escapeIndex + 1);+ if (escapeIndex + 1 < input.length && next !== 0x0a && next !== 0x0d && next !== 0x2028 && next !== 0x2029) {+ if (plainStart < escapeIndex) result.push(input.slice(plainStart, escapeIndex));+ plainStart = escapeIndex + 2;+ }+ }+ escapeIndex = input.indexOf("\x1b", Math.max(escapeIndex + 1, plainStart));+ }+ if (plainStart < input.length) result.push(input.slice(plainStart));+ return result.join("");}/**@@ -839,9 +952,10 @@ function breakLongWord(word: string, width: number, tracker: AnsiCodeTracker): s// We need to handle ANSI codes specially since they're not graphemeslet i = 0;const segments: Array<{ type: "ansi" | "grapheme"; value: string }> = [];+ const extractAnsi = createAnsiCodeExtractor(word);while (i < word.length) {- const ansiResult = extractAnsiCode(word, i);+ const ansiResult = extractAnsi(i);if (ansiResult) {segments.push({ type: "ansi", value: ansiResult.code });i += ansiResult.length;@@ -849,7 +963,7 @@ function breakLongWord(word: string, width: number, tracker: AnsiCodeTracker): s// Find the next ANSI code or end of stringlet end = i;while (end < word.length) {- const nextAnsi = extractAnsiCode(word, end);+ const nextAnsi = extractAnsi(end);if (nextAnsi) break;end++;}@@ -994,8 +1108,9 @@ export function truncateToWidth(exhaustedInput = !overflowed;} else {let i = 0;+ const extractAnsi = createAnsiCodeExtractor(text);while (i < text.length) {- const ansi = extractAnsiCode(text, i);+ const ansi = extractAnsi(i);if (ansi) {pendingAnsi += ansi.code;i += ansi.length;@@ -1025,7 +1140,7 @@ export function truncateToWidth(let end = i;while (end < text.length && text[end] !== "\t") {- const nextAnsi = extractAnsiCode(text, end);+ const nextAnsi = extractAnsi(end);if (nextAnsi) {break;}@@ -1089,9 +1204,10 @@ export function sliceWithWidth(currentCol = 0,i = 0,pendingAnsi = "";+ const extractAnsi = createAnsiCodeExtractor(line);while (i < line.length) {- const ansi = extractAnsiCode(line, i);+ const ansi = extractAnsi(i);if (ansi) {if (currentCol >= startCol && currentCol < endCol) result += ansi.code;else if (currentCol < startCol) pendingAnsi += ansi.code;@@ -1100,7 +1216,7 @@ export function sliceWithWidth(}let textEnd = i;- while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++;+ while (textEnd < line.length && !extractAnsi(textEnd)) textEnd++;for (const { segment } of segmenter.segment(line.slice(i, textEnd))) {const w = graphemeWidth(segment);@@ -1147,12 +1263,13 @@ export function extractSegments(let pendingAnsiBefore = "";let afterStarted = false;const afterEnd = afterStart + afterLen;+ const extractAnsi = createAnsiCodeExtractor(line);// Track styling state so "after" inherits styling from before the overlaypooledStyleTracker.clear();while (i < line.length) {- const ansi = extractAnsiCode(line, i);+ const ansi = extractAnsi(i);if (ansi) {// Track all SGR codes to know styling state at afterStartpooledStyleTracker.process(ansi.code);@@ -1168,7 +1285,7 @@ export function extractSegments(}let textEnd = i;- while (textEnd < line.length && !extractAnsiCode(line, textEnd)) textEnd++;+ while (textEnd < line.length && !extractAnsi(textEnd)) textEnd++;for (const { segment } of segmenter.segment(line.slice(i, textEnd))) {const w = graphemeWidth(segment);diff --git a/packages/tui/test/wrap-ansi.test.ts b/packages/tui/test/wrap-ansi.test.tsindex bd24e81599..40d87cef62 100644--- a/packages/tui/test/wrap-ansi.test.ts+++ b/packages/tui/test/wrap-ansi.test.ts@@ -1,6 +1,7 @@import assert from "node:assert";+import { performance } from "node:perf_hooks";import { describe, it } from "node:test";-import { visibleWidth, wrapTextWithAnsi } from "../src/utils.js";+import { extractAnsiCode, sliceByColumn, stripAnsi, visibleWidth, wrapTextWithAnsi } from "../src/utils.js";describe("wrapTextWithAnsi", () => {describe("underline styling", () => {@@ -220,3 +221,94 @@ describe("wrapTextWithAnsi with OSC 8 hyperlinks", () => {assert.strictEqual(closeCount, 1);});});++describe("ANSI sequence scanning", () => {+ it("accepts the full CSI final-byte range and valid byte classes", () => {+ const sequences = ["\x1b[@", "\x1b[2A", "\x1b[?25l", "\x1b[1 q", "\x1b[15~"];++ for (const sequence of sequences) {+ assert.deepStrictEqual(extractAnsiCode(sequence, 0), {+ code: sequence,+ length: sequence.length,+ });+ }+ });++ it("rejects incomplete and malformed CSI sequences", () => {+ for (const sequence of ["\x1b[", "\x1b[31", "\x1b[1 ", "\x1b[1 2m", "\x1b[1\x7fm"]) {+ assert.strictEqual(extractAnsiCode(sequence, 0), null);+ }+ });++ it("keeps OSC and DCS payloads distinct from CSI scanning", () => {+ const oscBel = "\x1b]0;title\x07";+ const oscSt = "\x1b]0;title\x1b\\";+ const dcs = "\x1bP1;2|payload[m~\x1b\\";+ const apcBel = "\x1b_private-marker\x07";++ for (const sequence of [oscBel, oscSt, dcs, apcBel]) {+ assert.deepStrictEqual(extractAnsiCode(sequence, 0), {+ code: sequence,+ length: sequence.length,+ });+ }+ assert.strictEqual(extractAnsiCode("\x1b]0;title", 0), null);+ assert.strictEqual(extractAnsiCode("\x1bPpayload", 0), null);+ assert.strictEqual(extractAnsiCode("\x1bPpayload\x07", 0), null);+ });++ it("ignores standard CSI and DCS sequences when measuring width", () => {+ const text = "a\x1b[2Ab\x1b[?25lc\x1b[1 qd\x1bPpayload[m~\x1b\\e";+ assert.strictEqual(visibleWidth(text), 5);+ });++ it("preserves scanned sequences while slicing by visible columns", () => {+ const hideCursor = "\x1b[?25l";+ const dcs = "\x1bPpayload\x1b\\";+ const text = `a${hideCursor}bc${dcs}d`;++ assert.strictEqual(sliceByColumn(text, 1, 3), `${hideCursor}bc${dcs}d`);+ });++ it("wraps by visible width without splitting standard CSI sequences", () => {+ const hideCursor = "\x1b[?25l";+ const showCursor = "\x1b[?25h";++ assert.deepStrictEqual(wrapTextWithAnsi(`${hideCursor}abcdef${showCursor}`, 3), [+ `${hideCursor}abc`,+ `def${showCursor}`,+ ]);+ });++ it("strips every scanned sequence from copied text", () => {+ const cursorStyle = "\x1b[1 q";+ const dcs = "\x1bPone\x1bXtwo\x1b\\";+ const dcsWithCsiPayload = "\x1bPbefore\x1b[31mafter\x1b\\";+ const text = `a${cursorStyle}b${dcs}c`;++ assert.strictEqual(stripAnsi(sliceByColumn(text, 0, 3)), "abc");+ assert.strictEqual(stripAnsi(`a${dcsWithCsiPayload}b`), "ab");+ assert.strictEqual(stripAnsi("a\x1bc b"), "a b");+ assert.strictEqual(stripAnsi("a\x1b\nb"), "a\x1b\nb");+ });++ it("strips long ANSI-heavy copied text", () => {+ const styled = "plain-text-1234567890\x1b[31mcolored\x1b[0m".repeat(10_000);++ assert.strictEqual(stripAnsi(styled), "plain-text-1234567890colored".repeat(10_000));+ });++ it("scans repeated incomplete control strings without quadratic slowdown", () => {+ const text = "\x1bP".repeat(50_000);+ const start = performance.now();++ assert.strictEqual(visibleWidth(text), 50_000);+ assert.ok(performance.now() - start < 1_000);+ });++ it("still finds valid control strings after an incomplete prefix", () => {+ const text = "\x1bPunterminated\x1b]0;title\x07x";++ assert.strictEqual(visibleWidth(text), visibleWidth("Punterminatedx"));+ });+});