Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(cli): preserve Unicode and text across token windows (#5814)
STDquantum769546608148
diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.tsindex 65eb80cb3..a7a7bf43e 100644--- a/src/cli/cliResultOutput.ts+++ b/src/cli/cliResultOutput.ts@@ -2,7 +2,7 @@ import { constants } from "node:buffer";import type { Writable } from "node:stream";import { Filter, Formatter } from "incur";-import { estimateTokenCount, sliceByTokens } from "tokenx";+import { estimateTokenCount } from "tokenx";import {parseCliOutputArguments,@@ -13,6 +13,7 @@ import {cliOperationFailurePayload,} from "../cliLogging.js";import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";+import { sliceTextByTokenWindow } from "./tokenWindow.js";import {projectAnalysisError,type AnalysisErrorProjection,@@ -33,6 +34,7 @@ export interface CliResultOutputMetadata {readonly command: string;readonly duration: string;readonly format: string;+ readonly renderOutput?: boolean;}/** Optional executable-owned result surface for one CLI invocation. */@@ -50,8 +52,8 @@ export const streamCliCommandResults = (cli: CliInstance,output: CliResultOutput,): void => {- visitRegisteredCommands(cli, (command, path) =>- wrapCommand(command, path, output),+ visitRegisteredCommands(cli, (command, path, inheritedPolicy) =>+ wrapCommand(command, path, output, inheritedPolicy),);};@@ -59,6 +61,7 @@ const wrapCommand = (command: Pick<RegisteredCommand, "run">,name: string,output: CliResultOutput,+ inheritedPolicy?: "all" | "agent-only",): void => {const run = command.run;command.run = (context) => {@@ -82,6 +85,7 @@ const wrapCommand = (!(await output.write(value, {command: name,format: context.format,+ renderOutput: context.agent || inheritedPolicy !== "agent-only",duration: `${Math.round(performance.now() - started)}ms`,})))@@ -213,9 +217,24 @@ export const createCliResultOutput = (? cliOperationFailurePayload(value): undefined;const failure = options.fullOutput ? operationFailure : undefined;+ // Incur owns suppressed human presentation, but a filter must never erase+ // the typed failure that this boundary already owns.+ if (+ operationFailure === undefined &&+ metadata.renderOutput === false &&+ !options.fullOutput &&+ !options.tokenCount+ )+ return false;const filtered: unknown =operationFailure ??(filterPaths === undefined ? value : Filter.apply(value, filterPaths));+ const ownsSuccessfulTokenWindow =+ operationFailure === undefined &&+ options.tokenWindow &&+ !options.tokenCount &&+ format !== "json" &&+ format !== "jsonl";if (operationFailure === undefined &&filterPaths !== undefined &&@@ -230,7 +249,8 @@ export const createCliResultOutput = (if (format !== "json" &&format !== "jsonl" &&- operationFailure === undefined+ operationFailure === undefined &&+ !ownsSuccessfulTokenWindow)return false;if (options.tokenCount) {@@ -309,6 +329,53 @@ export const createCliResultOutput = (typeof filtered === "symbol"? {}: filtered;+ if (ownsSuccessfulTokenWindow) {+ const formatted = Formatter.format(filtered, format);+ const window = sliceTextByTokenWindow(+ formatted,+ options.tokenOffset ?? 0,+ options.tokenLimit,+ );+ // Keep Incur's empty human/agent presentation semantics. Nonempty text+ // already belongs to this formatter, even when the whole result fits.+ if (!window.truncated && !options.fullOutput && formatted === "")+ return false;++ const sliced = window.truncated+ ? `${window.text}\n[truncated: showing tokens ${options.tokenOffset ?? 0}–${window.end} of ${window.total}]`+ : formatted;+ const outputDocument = options.fullOutput+ ? Formatter.format(+ window.truncated+ ? {+ ok: true,+ data: sliced,+ meta: {+ command: metadata.command,+ duration: metadata.duration,+ ...(window.end < window.total+ ? { nextOffset: window.end }+ : {}),+ },+ }+ : { ok: true, data: filtered, meta },+ format,+ )+ : sliced;+ handled = true;+ try {+ await writeOutputText(+ destination,+ outputDocument.endsWith("\n")+ ? outputDocument+ : `${outputDocument}\n`,+ );+ } catch (cause: unknown) {+ failed = true;+ throw cause;+ }+ return true;+ }// Once writing starts, a failed destination must not receive a second// document from Incur's error formatter.const failureText =@@ -350,15 +417,13 @@ const fullFailureEnvelope = (if (tokenLimit === undefined && tokenOffset === undefined)return { ok: false, error: failure, meta };const formatted = Formatter.format(failure, format);- const total = estimateTokenCount(formatted);const offset = tokenOffset ?? 0;- const end = tokenLimit === undefined ? total : offset + tokenLimit;- if (offset === 0 && end >= total) return { ok: false, error: failure, meta };- const actualEnd = Math.min(end, total);- const nextOffset = actualEnd < total ? actualEnd : undefined;+ const window = sliceTextByTokenWindow(formatted, offset, tokenLimit);+ if (!window.truncated) return { ok: false, error: failure, meta };+ const nextOffset = window.end < window.total ? window.end : undefined;return {ok: false,- error: `${sliceByTokens(formatted, offset, end)}\n[truncated: showing tokens ${offset}–${actualEnd} of ${total}]`,+ error: `${window.text}\n[truncated: showing tokens ${offset}–${window.end} of ${window.total}]`,meta: { ...meta, ...(nextOffset === undefined ? {} : { nextOffset }) },};};diff --git a/src/cli/tokenWindow.ts b/src/cli/tokenWindow.tsnew file mode 100644index 000000000..0b85e6a96--- /dev/null+++ b/src/cli/tokenWindow.ts@@ -0,0 +1,81 @@+import { estimateTokenCount } from "tokenx";++export interface TokenWindow {+ readonly text: string;+ readonly total: number;+ readonly end: number;+ readonly truncated: boolean;+}++// tokenx 1.3.0 uses this split expression before estimating and slicing.+// Its public API does not expose the segments, so retain the same boundaries+// here while mapping both ends of each segment to a disjoint text range.+const TOKEN_SPLIT_PATTERN = /(\s+|[.,!?;(){}[\]<>:/\\|@#$%^&*+=`~_-]+)/;++const codePointSafeBoundary = (text: string, offset: number): number =>+ offset > 0 &&+ offset < text.length &&+ text.charCodeAt(offset - 1)! >= 0xd800 &&+ text.charCodeAt(offset - 1)! <= 0xdbff &&+ text.charCodeAt(offset)! >= 0xdc00 &&+ text.charCodeAt(offset)! <= 0xdfff+ ? offset - 1+ : offset;++const segmentBoundary = (+ segment: string,+ tokenOffset: number,+ tokenCount: number,+): number =>+ codePointSafeBoundary(+ segment,+ Math.floor((tokenOffset / tokenCount) * segment.length),+ );++/** Slice an approximate token range without overlapping pages or splitting a surrogate pair. */+export const sliceTextByTokenWindow = (+ text: string,+ offset: number,+ limit: number | undefined,+): TokenWindow => {+ const total = estimateTokenCount(text);+ const end = limit === undefined ? total : offset + limit;+ if (offset === 0 && end >= total)+ return { text, total, end: total, truncated: false };++ const actualEnd = Math.min(end, total);+ if (offset >= actualEnd)+ return { text: "", total, end: actualEnd, truncated: true };++ const parts: string[] = [];+ let tokenPosition = 0;+ for (const segment of text.split(TOKEN_SPLIT_PATTERN).filter(Boolean)) {+ if (tokenPosition >= actualEnd && actualEnd < total) break;+ const tokenCount = estimateTokenCount(segment);+ if (tokenCount === 0) {+ if (+ tokenPosition >= offset &&+ (tokenPosition < actualEnd || actualEnd === total)+ )+ parts.push(segment);+ continue;+ }++ const segmentEnd = tokenPosition + tokenCount;+ if (tokenPosition < actualEnd && segmentEnd > offset) {+ const localStart = Math.max(0, offset - tokenPosition);+ const localEnd = Math.min(tokenCount, actualEnd - tokenPosition);+ const charStart = segmentBoundary(segment, localStart, tokenCount);+ const charEnd = segmentBoundary(segment, localEnd, tokenCount);+ parts.push(segment.slice(charStart, charEnd));+ }+ tokenPosition = segmentEnd;+ }++ return {+ text: parts.join(""),+ total,+ end: actualEnd,+ truncated: true,+ };+};diff --git a/src/cli/visitRegisteredCommands.ts b/src/cli/visitRegisteredCommands.tsindex 895f9e16a..efe7989e8 100644--- a/src/cli/visitRegisteredCommands.ts+++ b/src/cli/visitRegisteredCommands.ts@@ -5,6 +5,7 @@ import type { CliInstance } from "./types.js";type CommandMap = NonNullable<ReturnType<typeof Cli.toCommands.get>>;type CommandEntry = CommandMap extends Map<string, infer Entry> ? Entry : never;export type RegisteredCommand = Extract<CommandEntry, { run: unknown }>;+type OutputPolicy = "all" | "agent-only";// Incur types its router key without vars, env, or globals; accept any CLI.const registeredCommands: WeakMap<object, CommandMap> = Cli.toCommands;@@ -12,24 +13,35 @@ const registeredCommands: WeakMap<object, CommandMap> = Cli.toCommands;/** Visit ordinary commands in map order, children before each group's root. */export const visitRegisteredCommands = (cli: CliInstance,- visit: (command: RegisteredCommand, path: string) => void,+ visit: (+ command: RegisteredCommand,+ path: string,+ outputPolicy: OutputPolicy | undefined,+ ) => void,): void => {const commands = registeredCommands.get(cli);- if (commands !== undefined) visitCommands(commands, [], visit);+ if (commands !== undefined) visitCommands(commands, [], undefined, visit);};const visitCommands = (commands: CommandMap,parents: readonly string[],- visit: (command: RegisteredCommand, path: string) => void,+ inheritedPolicy: OutputPolicy | undefined,+ visit: (+ command: RegisteredCommand,+ path: string,+ outputPolicy: OutputPolicy | undefined,+ ) => void,): void => {for (const [name, entry] of commands) {const path = [...parents, name];if ("_group" in entry) {- visitCommands(entry.commands, path, visit);- if (entry.root !== undefined) visit(entry.root, path.join(" "));+ const policy = entry.outputPolicy ?? inheritedPolicy;+ visitCommands(entry.commands, path, policy, visit);+ if (entry.root !== undefined)+ visit(entry.root, path.join(" "), entry.root.outputPolicy ?? policy);} else if (!("_alias" in entry) && !("_fetch" in entry)) {- visit(entry, path.join(" "));+ visit(entry, path.join(" "), entry.outputPolicy ?? inheritedPolicy);}}};diff --git a/tests/acceptance/cliTokenWindowUnicode.test.ts b/tests/acceptance/cliTokenWindowUnicode.test.tsnew file mode 100644index 000000000..246365745--- /dev/null+++ b/tests/acceptance/cliTokenWindowUnicode.test.ts@@ -0,0 +1,107 @@+import { writeFile } from "node:fs/promises";+import { resolve } from "node:path";++import { expect } from "vitest";++import { workspaceCliTest } from "../support/cli/workspaceCliFixture.js";++workspaceCliTest(+ "preserves successful TOON text across CLI token pages",+ async ({ processes, workspace }) => {+ const fileName = `${"😀é".repeat(20)}.plist`;+ const plist = workspace.path(fileName);+ await writeFile(+ plist,+ `<?xml version="1.0" encoding="UTF-8"?>\n` ++ `<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" ` ++ `"http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n` ++ `<plist version="1.0"><dict>` ++ `<key>Long</key><string>${"abcdefghij".repeat(300)}</string>` ++ `<key>Emoji</key><string>${"😀é".repeat(300)}</string>` ++ `</dict></plist>`,+ );+ const run = async (filter: string, ...window: string[]) =>+ processes.run(+ process.execPath,+ [+ resolve("scripts/rea.mjs"),+ "inspect-artifact",+ plist,+ "--format",+ "toon",+ "--filter-output",+ filter,+ ...window,+ ],+ {+ cwd: workspace.root,+ env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },+ },+ );++ const fullDigest = await run("subject.digest.sha256");+ expect(+ fullDigest.exitCode,+ `${fullDigest.stderr}\n${fullDigest.stdout}`,+ ).toBe(0);+ const firstPage = await run(+ "subject.digest.sha256",+ "--token-limit",+ "10",+ "--token-offset",+ "0",+ );+ expect(firstPage.exitCode, firstPage.stderr).toBe(0);+ const firstMarker = firstPage.stdout.match(+ /\n\[truncated: showing tokens 0–10 of (\d+)\]\n?$/,+ );+ expect(firstMarker).not.toBeNull();+ const total = Number(firstMarker?.[1]);+ let joinedDigest = "";+ for (let offset = 0; offset < total; offset += 10) {+ const page =+ offset === 0+ ? firstPage+ : await run(+ "subject.digest.sha256",+ "--token-limit",+ "10",+ "--token-offset",+ String(offset),+ );+ expect(page.exitCode, page.stderr).toBe(0);+ joinedDigest += page.stdout.replace(+ /\n\[truncated: showing tokens \d+–\d+ of \d+\]\n?$/,+ "",+ );+ }+ expect(joinedDigest).toBe(fullDigest.stdout.trimEnd());++ // Artifact inspection exposes the actual Unicode filename on every platform.+ const fullName = await run("subject.name");+ expect(fullName.exitCode, fullName.stderr).toBe(0);+ expect(fullName.stdout).toContain(fileName);+ let joinedName = "";+ let nameTotal = Number.POSITIVE_INFINITY;+ for (let offset = 0; offset < nameTotal; offset += 7) {+ const page = await run(+ "subject.name",+ "--token-limit",+ "7",+ "--token-offset",+ String(offset),+ );+ expect(page.exitCode, page.stderr).toBe(0);+ const marker = page.stdout.match(+ /\n\[truncated: showing tokens \d+–\d+ of (\d+)\]\n?$/,+ );+ expect(marker).not.toBeNull();+ nameTotal = Number(marker?.[1]);+ const body = page.stdout.slice(0, marker?.index);+ expect(body.length).toBeGreaterThan(0);+ expect(body).not.toContain("\uFFFD");+ joinedName += body;+ }+ expect(joinedName).toBe(fullName.stdout.trimEnd());+ },+);diff --git a/tests/boundary/cli/cliTokenWindowOutput.test.ts b/tests/boundary/cli/cliTokenWindowOutput.test.tsnew file mode 100644index 000000000..48e16bcaf--- /dev/null+++ b/tests/boundary/cli/cliTokenWindowOutput.test.ts@@ -0,0 +1,288 @@+import { Writable } from "node:stream";++import { Cli, Formatter } from "incur";+import { estimateTokenCount } from "tokenx";+import { describe, expect, it } from "vitest";++import {+ createCliResultOutput,+ streamCliCommandResults,+} from "../../../src/cli/cliResultOutput.js";++const renderWindow = async (+ value: unknown,+ offset: number,+ limit: number,+ fullOutput = false,+ terminal = false,+ formatExplicit = true,+): Promise<{ readonly handled: boolean; readonly text: string }> => {+ const chunks: Buffer[] = [];+ const destination = new Writable({+ write(chunk: Buffer, _encoding, callback) {+ chunks.push(chunk);+ callback();+ },+ });+ if (terminal) Object.defineProperty(destination, "isTTY", { value: true });+ const argv = [+ "fixture",+ ...(formatExplicit ? ["--format", "toon"] : []),+ ...(fullOutput ? ["--full-output"] : []),+ "--token-limit",+ String(limit),+ "--token-offset",+ String(offset),+ ];+ const output = createCliResultOutput(argv, destination);+ if (output === undefined) throw new Error("Missing TOON output surface");+ const handled = await output.write(value, {+ command: "fixture",+ duration: "0ms",+ format: "toon",+ });+ return { handled, text: Buffer.concat(chunks).toString("utf8") };+};++describe("successful CLI token windows", () => {+ it.each([undefined, null, ""])(+ "preserves the full-output data field for a fitting empty value (%j)",+ async (value) => {+ const page = await renderWindow(value, 0, 1000, true);+ expect(page.handled).toBe(true);+ expect(page.text).toBe(+ `${Formatter.format(+ {+ ok: true,+ data: value,+ meta: { command: "fixture", duration: "0ms" },+ },+ "toon",+ )}\n`,+ );+ },+ );++ it("preserves an existing final newline when the whole result fits", async () => {+ const page = await renderWindow("abcdefghij\n", 0, 1000);+ expect(page.handled).toBe(true);+ expect(page.text).toBe("abcdefghij\n");+ });++ it.each([false, true])(+ "owns fitting windows and preserves full-output structure (%s)",+ async (fullOutput) => {+ const value = { text: "😀é", nested: { value: 42 } };+ const page = await renderWindow(value, 0, 1000, fullOutput);+ expect(page.handled).toBe(true);+ expect(page.text).toBe(+ `${Formatter.format(+ fullOutput+ ? {+ ok: true,+ data: value,+ meta: { command: "fixture", duration: "0ms" },+ }+ : value,+ "toon",+ )}\n`,+ );+ expect(page.text).not.toContain("nextOffset");+ expect(page.text).not.toContain("[truncated:");+ },+ );++ it("does not reformat a fitting ordinary result", async () => {+ let reads = 0;+ const page = await renderWindow(+ {+ get text() {+ reads++;+ return "abcdefghij";+ },+ },+ 0,+ 1000,+ );+ expect(page.handled).toBe(true);+ // TOON reads this property once per formatting pass.+ expect(reads).toBe(1);+ expect(page.text).toBe("text: abcdefghij\n");+ });++ it.each([+ { terminal: false, formatExplicit: false },+ { terminal: true, formatExplicit: false },+ { terminal: true, formatExplicit: true },+ ])(+ "preserves digest and Unicode pages ($terminal, $formatExplicit)",+ async ({ terminal, formatExplicit }) => {+ const value = {+ sha256:+ "6764c190e641f1c59c638f9bf32fee05f66992811c5605ebd25056156f21e221",+ text: "😀é".repeat(300),+ };+ const full = Formatter.format(value, "toon");+ const total = estimateTokenCount(full);+ const pages: string[] = [];+ for (let offset = 0; offset < total; offset += 10) {+ const page = await renderWindow(+ value,+ offset,+ 10,+ false,+ terminal,+ formatExplicit,+ );+ expect(page.handled).toBe(true);+ const marker = page.text.lastIndexOf("\n[truncated: showing tokens ");+ expect(marker).toBeGreaterThanOrEqual(0);+ pages.push(page.text.slice(0, marker));+ }+ const joined = pages.join("");+ expect(joined).toBe(full);+ expect(joined).toContain(value.sha256);+ expect(joined).not.toContain("\uFFFD");+ },+ );++ it("retains full-output metadata while returning a truncated page", async () => {+ const page = await renderWindow(+ { value: "abcdefghij" },+ 0,+ 2,+ true,+ true,+ false,+ );+ expect(page.handled).toBe(true);+ expect(page.text).toContain("ok: true");+ expect(page.text).toContain("nextOffset: 2");+ expect(page.text).toContain("showing tokens 0–2 of");+ });++ it.each(["abcdefghij abcdefghij abcdefghij \n", " 😀é abcdefghij\t\r\n"])(+ "preserves all whitespace across text pages: %j",+ async (value) => {+ let joined = "";+ const total = estimateTokenCount(value);+ for (let offset = 0; offset < total; offset += 1) {+ const page = await renderWindow(value, offset, 1);+ expect(page.handled).toBe(true);+ joined += page.text.slice(+ 0,+ page.text.lastIndexOf("\n[truncated: showing tokens "),+ );+ }+ expect(joined).toBe(value);+ },+ );+});++describe("token windows through Incur output policies", () => {+ it.each([{ windowFlags: [] }, { windowFlags: ["--token-limit", "2"] }])(+ "retains filtered typed failures despite suppressed human results ($windowFlags)",+ async ({ windowFlags }) => {+ let text = "";+ const destination = new Writable({+ write(chunk, _encoding, callback) {+ text += chunk.toString();+ callback();+ },+ });+ const output = createCliResultOutput(+ ["fixture", "--filter-output", "missing", ...windowFlags],+ destination,+ )!;+ const handled = await output.write(+ {+ error: "Analysis failed",+ code: "target_unavailable",+ category: "unavailable",+ message: "Artifact missing",+ retryable: false,+ remediation: { action: "Check the path." },+ details: {},+ },+ {+ command: "fixture",+ duration: "0ms",+ format: "toon",+ renderOutput: false,+ },+ );+ expect(handled).toBe(true);+ expect(text).toContain("target_unavailable");+ expect(text).toContain("Artifact missing");+ },+ );+ it.each([+ { terminal: true, flags: [], visible: false },+ { terminal: false, flags: [], visible: true },+ { terminal: true, flags: ["--format", "toon"], visible: true },+ { terminal: true, flags: ["--full-output"], visible: true },+ { terminal: true, flags: ["--token-count"], visible: true },+ ])(+ "honors agent-only presentation ($terminal, $flags)",+ async ({ terminal, flags, visible }) => {+ const descriptor = Object.getOwnPropertyDescriptor(+ process.stdout,+ "isTTY",+ );+ Object.defineProperty(process.stdout, "isTTY", {+ value: terminal,+ configurable: true,+ });+ try {+ for (const grouped of [false, true]) {+ const argv = [+ ...(grouped ? ["group"] : []),+ "fixture",+ "--token-limit",+ "2",+ ...flags,+ ];+ let text = "";+ const destination = new Writable({+ write(chunk, _encoding, callback) {+ text += chunk.toString();+ callback();+ },+ });+ const output = createCliResultOutput(argv, destination)!;+ const cli = Cli.create("policy-test", { sync: false });+ if (grouped) {+ const group = Cli.create("group", {+ sync: false,+ outputPolicy: "agent-only",+ });+ group.command("fixture", {+ run: () => ({ value: "😀é".repeat(100) }),+ });+ cli.command(group);+ } else {+ cli.command("fixture", {+ outputPolicy: "agent-only",+ run: () => ({ value: "😀é".repeat(100) }),+ });+ }+ streamCliCommandResults(cli, output);+ await cli.serve(argv, {+ stdout: (chunk) => {+ if (!output.handled) text += chunk;+ },+ exit: () => {},+ });+ expect(text.length > 0).toBe(visible);+ expect(text).not.toContain("\uFFFD");+ if (visible && !flags.includes("--token-count"))+ expect(text).toContain("showing tokens 0–2 of");+ }+ } finally {+ if (descriptor)+ Object.defineProperty(process.stdout, "isTTY", descriptor);+ else Reflect.deleteProperty(process.stdout, "isTTY");+ }+ },+ );+});