Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(cli): stream large JavaScript JSON results (#1024)
Ysopwnable0e25befe78f2
diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.mdindex 42d40e938..d354dc570 100644--- a/docs/javascript-artifact-reconstruction.md+++ b/docs/javascript-artifact-reconstruction.md@@ -65,11 +65,19 @@ identifiers remain unchanged. The opt-in regression check is`npm run verify:javascript:digests`;it hashes a value larger than the running Node engine's single-string limit.-Output formatters and MCP transport serialization still assemble whole strings.-For a large CLI result, select the needed fields-before formatting, for example `--format json --filter-output-evidence_id,normalized_result.statistics`. Complete serialization of a result-beyond the engine's string limit is not established by this digest check.+Both application CLI routes stream complete JSON and JSONL output in bounded+chunks, including field filters and `--full-output`. They wait for each stdout+write instead of constructing or reparsing a document-sized string. The opt-in+`npm run verify:javascript:output` checks the actual CLI formatting boundary with+a temporary output file larger than the running engine's string limit, verifies+its bytes and digest, and removes it. Use `-- jsonl` for the compact JSONL check.+Each check needs space for one output file plus a 1 GiB free-space reserve.++Other CLI formats, `--token-count`, and MCP transport serialization still+assemble whole strings. Field selection remains useful when the caller needs a+smaller view, for example `--format json --filter-output+evidence_id,normalized_result.statistics`. Streaming output does not bound the+memory needed to construct the analysis graph itself.Client framing limits also apply: the pinned Node MCP SDK's stdio transportdefaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodatethe complete response, including text and structured Evidence projections.diff --git a/docs/testing.md b/docs/testing.mdindex 740d7958e..feb2bdf54 100644--- a/docs/testing.md+++ b/docs/testing.md@@ -561,6 +561,18 @@ conditional `real-web-source-map` CI job supplies Chrome and an isolated pinnedfixture compiler; static/unit checks do not acquire a browser. See[the source location guide](web-source-location.md) for the verified decoder profile.+### JavaScript large-output lane++`npm run verify:javascript:output` exercises the CLI JSON result surface beyond+the running Node engine's single-string limit. Shared input leaves keep the+fixture's graph small; the verifier writes one temporary output file, checks its+complete byte count and an independent digest, then removes it. It requires only+Node and the built REA runtime, with space for the output plus a 1 GiB reserve.+Run `npm run verify:javascript:output -- jsonl` for compact JSONL coverage. This+opt-in lane is separate from routine tests and the canonical hash check+`npm run verify:javascript:digests`. It verifies serialization rather than an+arbitrary third-party application's parsing cost or MCP client capacity.+### Website runtime attribution lane`npm run verify:browser:runtime` uses caller-supplieddiff --git a/package.json b/package.jsonindex 4b2b72f45..3a98037bc 100644--- a/package.json+++ b/package.json@@ -141,6 +141,7 @@"verify:inspector": "npm run build:cached && node scripts/verify/inspector/runtime-observation.mjs","verify:javascript-runtime": "npm run build:cached && node scripts/verify/javascript/runtime-observation.mjs","verify:javascript:digests": "npm run build:cached && node scripts/verify/javascript/digests.mjs",+ "verify:javascript:output": "npm run build:cached && node scripts/verify/javascript/output.mjs","verify:application-workflows": "npm run build:cached && node scripts/verify/javascript/application-workflows.mjs","verify:managed": "npm run build:cached && node scripts/verify/managed/conformance.mjs","verify:firmware": "npm run build:cached && node scripts/verify/firmware/analysis.mjs",diff --git a/scripts/rea.mjs b/scripts/rea.mjsindex 4e310bf08..1f720a4d1 100755--- a/scripts/rea.mjs+++ b/scripts/rea.mjs@@ -19,7 +19,7 @@ const runtimeFiles = isMcpMode? ["dist/main.js"]: isMcpDoctorMode? ["dist/main.js", "dist/mcpDoctor.js"]- : ["dist/cli.js", "dist/cliOutput.js"];+ : ["dist/cli.js", "dist/cliOutput.js", "dist/cli/streamedJsonOutput.js"];if (!(await compiledRuntimeExists(runtimeFiles))) {process.stderr.write(@@ -39,6 +39,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {process.exitCode = result.exitCode;} else {const { createCli } = await import("../dist/cli.js");+ const { createStreamedCliJsonOutput } =+ await import("../dist/cli/streamedJsonOutput.js");const {renderCliOutputArgumentError,renderEmptyFilteredCliOutput,@@ -51,14 +53,20 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {process.exitCode = 1;} else {let wroteOutput = false;- await createCli().serve(args, {+ const resultOutput = createStreamedCliJsonOutput(args, process.stdout);+ await createCli(process.env, resultOutput).serve(args, {stdout: (output) => {+ if (resultOutput?.handled) {+ if (resultOutput.failed)+ process.stderr.write(sanitizeCliOutput(output));+ return;+ }const sanitized = sanitizeCliOutput(output);if (sanitized.length > 0) wroteOutput = true;process.stdout.write(sanitized);},});- if (!wroteOutput)+ if (!wroteOutput && !resultOutput?.handled)process.stdout.write(renderEmptyFilteredCliOutput(args) ?? "");}}diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjsnew file mode 100644index 000000000..0e6d5635e--- /dev/null+++ b/scripts/verify/javascript/output.mjs@@ -0,0 +1,134 @@+import assert from "node:assert/strict";+import { constants } from "node:buffer";+import { createHash } from "node:crypto";+import { once } from "node:events";+import { createReadStream, createWriteStream } from "node:fs";+import { mkdtemp, rm, stat, statfs } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { finished } from "node:stream/promises";++import { Cli } from "incur";++import { createStreamedCliJsonOutput } from "../../../dist/cli/streamedJsonOutput.js";+import { createEvidence } from "../../../dist/domain/evidence.js";++// This opt-in format-boundary check writes one temporary file larger than the+// actual engine string limit. Shared leaves keep the analysis input small.+const format = process.argv[2] ?? "json";+assert.ok(format === "json" || format === "jsonl", "Use json or jsonl");+const leaf = "x".repeat(64 * 1024);+const encodedLeaf = JSON.stringify(leaf);+const count = Math.ceil(constants.MAX_STRING_LENGTH / (leaf.length + 3)) + 1;+const volume = await statfs(tmpdir());+assert.ok(+ volume.bavail * volume.bsize > count * encodedLeaf.length + 1024 ** 3,+ "Large-output verification needs room for one output file and a 1-GiB reserve",+);+const workspace = await mkdtemp(join(tmpdir(), "rea-javascript-output-"));+const path = join(workspace, "result.json");+let destination;+let peakRss = process.memoryUsage().rss;+const sampleRss = () => {+ peakRss = Math.max(peakRss, process.memoryUsage().rss);+};+const sampler = setInterval(sampleRss, 25);+sampler.unref();++try {+ const evidence = createEvidence(+ undefined,+ { id: "large-output-fixture", name: "Large output fixture", version: "1" },+ {+ operation: "inspect_fixture",+ parameters: {},+ result: { payload: Array.from({ length: count }, () => leaf) },+ },+ );+ sampleRss();+ const metadata = { command: "inspect-output-fixture", duration: "0ms" };+ const marker = "__REA_OUTPUT_ENCODING_MARKER__";+ const template = {+ ok: true,+ data: { ...evidence, normalized_result: { payload: [marker] } },+ meta: metadata,+ };+ const skeleton = JSON.stringify(template, null, format === "json" ? 2 : 0);+ const encodedMarker = JSON.stringify(marker);+ const markerIndex = skeleton.indexOf(encodedMarker);+ assert.ok(markerIndex >= 0);+ assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker));+ const prefix = skeleton.slice(0, markerIndex);+ const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`;+ const indentation = prefix.slice(prefix.lastIndexOf("\n") + 1);+ const separator = format === "json" ? `,\n${indentation}` : ",";+ const expectedHash = createHash("sha256");+ expectedHash.update(prefix);+ for (let index = 0; index < count; index += 1) {+ if (index > 0) expectedHash.update(separator);+ expectedHash.update(encodedLeaf);+ }+ expectedHash.update(suffix);+ const expectedBytes =+ Buffer.byteLength(prefix) ++ count * Buffer.byteLength(encodedLeaf) ++ (count - 1) * Buffer.byteLength(separator) ++ Buffer.byteLength(suffix);+ assert.ok(expectedBytes > constants.MAX_STRING_LENGTH);++ destination = createWriteStream(path, { flags: "wx", mode: 0o600 });+ const argv = [metadata.command, "--format", format, "--full-output"];+ const output = createStreamedCliJsonOutput(argv, destination);+ assert.ok(output);+ let unexpectedOutputBytes = 0;+ let exitCode = 0;+ await Cli.create("rea-output-verifier", { sync: false })+ .command(metadata.command, {+ run: async ({ format: selectedFormat }) => {+ assert.equal(+ await output.write(evidence, { ...metadata, format: selectedFormat }),+ true,+ );+ },+ })+ .serve(argv, {+ stdout: (text) => {+ if (!output.handled) unexpectedOutputBytes += Buffer.byteLength(text);+ },+ exit: (code) => {+ exitCode = code;+ },+ });+ destination.end();+ await finished(destination);+ assert.equal(exitCode, 0);+ assert.equal(output.handled, true);+ assert.equal(output.failed, false);+ assert.equal(unexpectedOutputBytes, 0);++ const actualHash = createHash("sha256");+ for await (const chunk of createReadStream(path)) actualHash.update(chunk);+ const digest = actualHash.digest("hex");+ assert.equal(digest, expectedHash.digest("hex"));+ assert.equal((await stat(path)).size, expectedBytes);+ sampleRss();+ process.stdout.write(+ `${JSON.stringify({+ json_output_verified: true,+ format,+ output_bytes: expectedBytes,+ engine_string_limit: constants.MAX_STRING_LENGTH,+ output_sha256: digest,+ evidence_id: evidence.evidence_id,+ rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },+ })}\n`,+ );+} finally {+ clearInterval(sampler);+ if (destination !== undefined && !destination.closed) {+ const closed = once(destination, "close");+ destination.destroy();+ await closed;+ }+ await rm(workspace, { recursive: true, force: true });+}diff --git a/src/cli.ts b/src/cli.tsindex 8ce590f05..cd9b683e5 100644--- a/src/cli.ts+++ b/src/cli.ts@@ -23,6 +23,7 @@ import { registerElectronCommands } from "./cli/electronCommands.js";import { registerJavaScriptRuntimeObservationCommands } from "./cliJavaScriptRuntimeCommands.js";import { registerApplicationCommands } from "./cli/applicationCommands.js";import type { CliInstance } from "./cli/types.js";+import type { CliResultOutput } from "./cli/streamedJsonOutput.js";/*** Build the one-shot Incur CLI without starting Hopper at import time.@@ -31,6 +32,7 @@ import type { CliInstance } from "./cli/types.js";*/export const createCli = (environment: Readonly<Record<string, string | undefined>> = process.env,+ resultOutput?: CliResultOutput,): CliInstance => {const logger = createLogger("cli",@@ -50,7 +52,7 @@ export const createCli = (});registerSetupCommands(cli, logger);- registerCoreAnalysisCommands(cli, logger);+ registerCoreAnalysisCommands(cli, logger, resultOutput);registerUtilityCommands(cli, logger, environment);registerArtifactCommands(cli, logger);registerManagedCommands(cli, logger);@@ -66,7 +68,7 @@ export const createCli = (registerJavaScriptRecoveryCommands(cli, logger, environment);registerAdvancedBrowserCommands(cli, logger);registerBrowserScenarioCommands(cli, logger);- registerElectronCommands(cli, logger);+ registerElectronCommands(cli, logger, resultOutput);registerJavaScriptRuntimeObservationCommands(cli, logger);registerApplicationCommands(cli, logger);return cli;diff --git a/src/cli/coreAnalysisCommands.ts b/src/cli/coreAnalysisCommands.tsindex da50ca830..e323348da 100644--- a/src/cli/coreAnalysisCommands.ts+++ b/src/cli/coreAnalysisCommands.ts@@ -2,12 +2,14 @@ import type { Logger } from "../logger.js";import { registerCoreBinaryCommands } from "./coreBinaryCommands.js";import { registerCoreNativeCommands } from "./coreNativeCommands.js";import type { CliInstance } from "./types.js";+import type { CliResultOutput } from "./streamedJsonOutput.js";/** Register core binary and native deep-analysis CLI commands. */export const registerCoreAnalysisCommands = (cli: CliInstance,logger: Logger,+ resultOutput?: CliResultOutput,): void => {- registerCoreBinaryCommands(cli, logger);+ registerCoreBinaryCommands(cli, logger, resultOutput);registerCoreNativeCommands(cli, logger);};diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.tsindex 6c70a2626..441d0e147 100644--- a/src/cli/coreBinaryCommands.ts+++ b/src/cli/coreBinaryCommands.ts@@ -13,13 +13,18 @@ import {} from "./options.js";import type { CliInstance } from "./types.js";import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";+import type {+ CliCommandOutput,+ CliResultOutput,+} from "./streamedJsonOutput.js";/** Register provider-neutral binary overview and procedure CLI commands. */export const registerCoreBinaryCommands = (cli: CliInstance,logger: Logger,+ resultOutput?: CliResultOutput,): void => {- registerOverviewCommands(cli, logger);+ registerOverviewCommands(cli, logger, resultOutput);registerFunctionCommand(cli, logger);registerInstructionsCommand(cli, logger);registerSearchCommand(cli, logger);@@ -27,7 +32,11 @@ export const registerCoreBinaryCommands = (registerTraceCommand(cli, logger);};-const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {+const registerOverviewCommands = (+ cli: CliInstance,+ logger: Logger,+ resultOutput?: CliResultOutput,+): void => {const overviewOptions = z.object({snapshot: z.string()@@ -43,9 +52,16 @@ const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {path: z.string().describe("App, program, or analysis database path"),}),options: overviewOptions,- run: ({ args, options }) =>+ run: ({ args, options, format }) =>logCliCommand(logger, "analyze", () =>- runRoutedOverview(args.path, options, logger),+ runRoutedOverview(+ args.path,+ options,+ logger,+ resultOutput === undefined+ ? undefined+ : { output: resultOutput, command: CLI_COMMANDS.analyze, format },+ ),),});cli.command(CLI_COMMANDS.inspect, {@@ -109,6 +125,7 @@ const runRoutedOverview = async (readonly provider?: string | undefined;},logger: Logger,+ output?: CliCommandOutput,) => {if (options.provider === undefined &&@@ -116,9 +133,10 @@ const runRoutedOverview = async (options["target-format"] === undefined &&(await isJavaScriptApplicationPath(path)))- return runCliJavaScriptApplicationAnalysis({- input_path: resolve(path),- });+ return runCliJavaScriptApplicationAnalysis(+ { input_path: resolve(path) },+ output,+ );return runDirectAnalysis(path,"binary_overview",diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.tsindex e91b1ff63..e747dc5c9 100644--- a/src/cli/electronCommands.ts+++ b/src/cli/electronCommands.ts@@ -24,15 +24,17 @@ import {javascriptApplicationOptions,} from "../cliObservationOptions.js";import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";+import type { CliResultOutput } from "./streamedJsonOutput.js";/** Register CLI equivalents of the Electron MCP tools. */export const registerElectronCommands = (cli: ReturnType<typeof Cli.create>,logger: Logger,+ resultOutput?: CliResultOutput,): void => {registerElectronObservationCommands(cli, logger);registerElectronActiveCommand(cli, logger);- registerJavaScriptApplicationCommand(cli, logger);+ registerJavaScriptApplicationCommand(cli, logger, resultOutput);registerJavaScriptRuntimeReconciliationCommand(cli, logger);};@@ -172,6 +174,7 @@ const registerElectronPageInspection = (const registerJavaScriptApplicationCommand = (cli: ReturnType<typeof Cli.create>,logger: Logger,+ resultOutput?: CliResultOutput,): void => {cli.command(CLI_COMMANDS.analyzeJavaScriptApplication, {description:@@ -180,12 +183,18 @@ const registerJavaScriptApplicationCommand = (path: z.string().describe("ASAR or extracted application path"),}),options: javascriptApplicationOptions,- run: ({ args, options }) =>+ run: ({ args, options, format }) =>logCliCommand(logger, CLI_COMMANDS.analyzeJavaScriptApplication, () =>- runCliJavaScriptApplicationAnalysis({- input_path: args.path,- format: options.artifactFormat,- }),+ runCliJavaScriptApplicationAnalysis(+ { input_path: args.path, format: options.artifactFormat },+ resultOutput === undefined+ ? undefined+ : {+ output: resultOutput,+ command: CLI_COMMANDS.analyzeJavaScriptApplication,+ format,+ },+ ),),});};diff --git a/src/cli/javascriptApplicationAnalysis.ts b/src/cli/javascriptApplicationAnalysis.tsindex 62a8bc5b0..92d4696c4 100644--- a/src/cli/javascriptApplicationAnalysis.ts+++ b/src/cli/javascriptApplicationAnalysis.ts@@ -2,11 +2,14 @@ import { analyzeJavaScriptApplication } from "../application/javascript/JavaScriimport { createProgressReporter } from "../application/ProgressReporter.js";import { projectAnalysisError } from "../domain/analysisErrorProjection.js";import type { JsonValue } from "../domain/jsonValue.js";+import type { CliCommandOutput } from "./streamedJsonOutput.js";/** Execute the shared one-shot CLI boundary for static JavaScript analysis. */export const runCliJavaScriptApplicationAnalysis = async (input: unknown,-): Promise<JsonValue> => {+ output?: CliCommandOutput,+): Promise<JsonValue | undefined> => {+ const started = performance.now();const progress = createProgressReporter(async (update) => {process.stderr.write(`${JSON.stringify({ rea_progress: update })}\n`);@@ -14,7 +17,17 @@ export const runCliJavaScriptApplicationAnalysis = async ({ minimumIntervalMs: 0 },);const result = await analyzeJavaScriptApplication(input, { progress });- return result.ok ? result.value : cliError(result.error);+ const value = result.ok ? result.value : cliError(result.error);+ if (+ output !== undefined &&+ (await output.output.write(value, {+ command: output.command,+ format: output.format,+ duration: `${Math.round(performance.now() - started)}ms`,+ }))+ )+ return undefined;+ return value;};const cliError = (diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.tsnew file mode 100644index 000000000..0e7e75503--- /dev/null+++ b/src/cli/jsonOutput.ts@@ -0,0 +1,149 @@+import type { Writable } from "node:stream";++type JsonOutputFormat = "json" | "jsonl";++/** Write ordinary JSON without constructing a document-sized string. */+export const writeJsonOutput = async (+ value: unknown,+ destination: Writable,+ format: JsonOutputFormat = "json",+): Promise<void> => {+ for (const part of bufferedParts(documentParts(value, format)))+ await writePart(destination, part);+};++const writePart = (destination: Writable, part: string): Promise<void> =>+ new Promise((resolve, reject) => {+ const onError = (cause: unknown): void => reject(cause);+ destination.once("error", onError);+ try {+ destination.write(part, (cause) => {+ if (cause) {+ // A Writable can emit its error after invoking the write callback.+ setImmediate(() => destination.off("error", onError));+ reject(cause);+ } else {+ destination.off("error", onError);+ resolve();+ }+ });+ } catch (cause: unknown) {+ destination.off("error", onError);+ reject(cause);+ }+ });++function* documentParts(+ value: unknown,+ format: JsonOutputFormat,+): Generator<string> {+ if (format === "jsonl" && Array.isArray(value)) {+ for (const item of value) {+ yield* encodeValue(item, false, 0, new Set());+ yield "\n";+ }+ return;+ }+ yield* encodeValue(value, format === "json", 0, new Set());+ yield "\n";+}++function* bufferedParts(parts: Iterable<string>): Generator<string> {+ let buffer = "";+ for (const part of parts) {+ if (buffer.length + part.length > 64 * 1024 && buffer !== "") {+ yield buffer;+ buffer = "";+ }+ buffer += part;+ }+ if (buffer !== "") yield buffer;+}++function* encodeString(value: string): Generator<string> {+ yield '"';+ for (let start = 0; start < value.length;) {+ let end = Math.min(start + 8192, value.length);+ const last = value.charCodeAt(end - 1);+ const next = value.charCodeAt(end);+ // Keep a surrogate pair in the same JSON.stringify and UTF-8 write.+ if (+ end < value.length &&+ last >= 0xd800 &&+ last <= 0xdbff &&+ next >= 0xdc00 &&+ next <= 0xdfff+ )+ end -= 1;+ yield JSON.stringify(value.slice(start, end)).slice(1, -1);+ start = end;+ }+ yield '"';+}++function* encodeValue(+ value: unknown,+ pretty: boolean,+ depth: number,+ ancestors: Set<object>,+): Generator<string> {+ if (typeof value === "string") {+ yield* encodeString(value);+ return;+ }+ if (value === null || typeof value === "boolean") {+ yield value === null ? "null" : String(value);+ return;+ }+ if (typeof value === "number" && Number.isFinite(value)) {+ yield String(value);+ return;+ }+ if (typeof value !== "object" || value === null)+ throw new TypeError("CLI JSON output requires ordinary JSON values");+ if (ancestors.has(value))+ throw new TypeError("CLI JSON output contains a circular reference");+ if (+ !Array.isArray(value) &&+ Object.prototype.toString.call(value) !== "[object Object]"+ )+ throw new TypeError("CLI JSON output requires ordinary JSON objects");+ ancestors.add(value);+ try {+ const array = Array.isArray(value);+ const keys = array+ ? undefined+ : Object.keys(value).filter((key) => {+ const item: unknown = Reflect.get(value, key);+ // Incur's filter can select inherited methods. Match native JSON's+ // omission of fields that have no JSON representation.+ return (+ item !== undefined &&+ typeof item !== "function" &&+ typeof item !== "symbol"+ );+ });+ const count = array ? value.length : (keys?.length ?? 0);+ yield array ? "[" : "{";+ for (let index = 0; index < count; index += 1) {+ if (index > 0) yield ",";+ if (pretty) yield `\n${" ".repeat(depth + 1)}`;+ let item: unknown;+ if (array) {+ item = value[index];+ } else {+ const key = keys?.[index];+ if (key === undefined)+ throw new TypeError("CLI JSON object changed during serialization");+ yield* encodeString(key);+ yield pretty ? ": " : ":";+ item = Reflect.get(value, key);+ }+ yield* encodeValue(item, pretty, depth + 1, ancestors);+ }+ if (pretty && count > 0) yield `\n${" ".repeat(depth)}`;+ yield array ? "]" : "}";+ } finally {+ ancestors.delete(value);+ }+}diff --git a/src/cli/streamedJsonOutput.ts b/src/cli/streamedJsonOutput.tsnew file mode 100644index 000000000..282316d0a--- /dev/null+++ b/src/cli/streamedJsonOutput.ts@@ -0,0 +1,84 @@+import { type Writable } from "node:stream";++import { Filter } from "incur";++import { parseCliOutputArguments } from "../cliOutput.js";+import type { JsonValue } from "../domain/jsonValue.js";+import { writeJsonOutput } from "./jsonOutput.js";++/** Result metadata supplied by the command that actually ran. */+export interface CliResultOutputMetadata {+ readonly command: string;+ readonly duration: string;+ readonly format: string;+}++/** Optional executable-owned result surface for one CLI invocation. */+export interface CliResultOutput {+ readonly handled: boolean;+ readonly failed: boolean;+ write(value: JsonValue, metadata: CliResultOutputMetadata): Promise<boolean>;+}++/** Selected command output context, independent of application workflows. */+export interface CliCommandOutput {+ readonly output: CliResultOutput;+ readonly command: string;+ readonly format: string;+}++/** Preserve Incur's JSON controls while streaming the complete result. */+export const createStreamedCliJsonOutput = (+ arguments_: readonly string[],+ destination: Writable,+): CliResultOutput | undefined => {+ const options = parseCliOutputArguments(arguments_);+ if (+ options.parseError ||+ options.tokenCount ||+ options.tokenWindow ||+ (options.format !== "json" && options.format !== "jsonl")+ )+ return undefined;+ const format = options.format;+ let handled = false;+ let failed = false;+ return {+ get handled() {+ return handled;+ },+ get failed() {+ return failed;+ },+ async write(value, metadata) {+ if (metadata.format !== format) return false;+ const filtered: unknown = options.filterOutput+ ? Filter.apply(value, Filter.parse(options.filterOutput))+ : value;+ const meta = {+ command: metadata.command,+ duration: metadata.duration,+ };+ const document = options.fullOutput+ ? filtered === undefined+ ? { ok: true, meta }+ : { ok: true, data: filtered, meta }+ : filtered === undefined ||+ filtered === null ||+ typeof filtered === "function" ||+ typeof filtered === "symbol"+ ? {}+ : filtered;+ // Once writing starts, a failed destination must not receive a second+ // document from Incur's error formatter.+ handled = true;+ try {+ await writeJsonOutput(document, destination, format);+ } catch (cause: unknown) {+ failed = true;+ throw cause;+ }+ return true;+ },+ };+};diff --git a/src/cliOutput.ts b/src/cliOutput.tsindex 063a6d452..a5b359992 100644--- a/src/cliOutput.ts+++ b/src/cliOutput.ts@@ -20,9 +20,11 @@ export type CliOutputArgumentValidation =type JsonRecord = Record<string, unknown>;interface ParsedCliOutputArguments {- readonly filterOutput: boolean;+ readonly filterOutput: string | undefined;readonly format: string;+ readonly fullOutput: boolean;readonly parseError: boolean;+ readonly tokenCount: boolean;readonly tokenWindow: boolean;}@@ -36,12 +38,23 @@ const validationError = (value: JsonRecord): JsonRecord | undefined => {return undefined;};-const parseCliOutputArguments = (+/** Read effective Incur output controls at the executable boundary. */+export const parseCliOutputArguments = (arguments_: readonly string[],): ParsedCliOutputArguments => {let format: string = "toon";- let filterOutput = false;+ let filterOutput: string | undefined;+ let fullOutput = false;+ let tokenCount = false;let tokenWindow = false;+ const parsed = (parseError: boolean): ParsedCliOutputArguments => ({+ filterOutput,+ format,+ fullOutput,+ parseError,+ tokenCount,+ tokenWindow: parseError ? false : tokenWindow,+ });for (let index = 0; index < arguments_.length; index += 1) {const argument = arguments_[index];// Incur scans builtins across `--` and accepts only spaced value forms.@@ -55,8 +68,7 @@ const parseCliOutputArguments = (if (value) {// Incur stops parsing at an invalid format before applying any later// output controls. Let it report that malformed flag itself.- if (!INCUR_OUTPUT_FORMATS.has(value))- return { filterOutput, format, parseError: true, tokenWindow: false };+ if (!INCUR_OUTPUT_FORMATS.has(value)) return parsed(true);format = value;index += 1;}@@ -64,7 +76,7 @@ const parseCliOutputArguments = (}if (argument === "--filter-output") {if (arguments_[index + 1]) {- filterOutput = true;+ filterOutput = arguments_[index + 1];index += 1;}continue;@@ -74,14 +86,16 @@ const parseCliOutputArguments = (if (value) {const numericValue = Number(value);if (!Number.isFinite(numericValue) || value.trim() === "")- return { filterOutput, format, parseError: true, tokenWindow: false };+ return parsed(true);tokenWindow = true;index += 1;}continue;}+ if (argument === "--full-output") fullOutput = true;+ if (argument === "--token-count") tokenCount = true;}- return { filterOutput, format, parseError: false, tokenWindow };+ return parsed(false);};/** Reject text-window controls that would corrupt a structured document. */diff --git a/tests/boundary/cli/dispatcher.test.ts b/tests/boundary/cli/dispatcher.test.tsindex d94cb7ec7..79d2913b9 100644--- a/tests/boundary/cli/dispatcher.test.ts+++ b/tests/boundary/cli/dispatcher.test.ts@@ -11,6 +11,7 @@ describe("executable dispatcher", () => {async ({ processes, workspace }) => {const scripts = await workspace.mkdir("scripts");await workspace.mkdir("dist");+ await workspace.mkdir("dist/cli");await Promise.all([copyFile("scripts/rea.mjs", join(scripts, "rea.mjs")),workspace.write(@@ -29,6 +30,10 @@ describe("executable dispatcher", () => {"dist/cliOutput.js","export const sanitizeCliOutput = (output) => output; export const validateCliOutputArguments = () => ({ ok: true }); export const renderCliOutputArgumentError = () => ''; export const renderEmptyFilteredCliOutput = () => undefined;\n",),+ workspace.write(+ "dist/cli/streamedJsonOutput.js",+ "export const createStreamedCliJsonOutput = () => undefined;\n",+ ),workspace.write("dist/mcpDoctor.js",'export const runProductionMcpDoctorCli = (args) => Promise.resolve({ output: JSON.stringify({ adapter: "mcp-doctor", args }), exitCode: 0 });\n',diff --git a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts b/tests/boundary/cli/javascriptApplicationJsonOutput.test.tsnew file mode 100644index 000000000..05dc4fc06--- /dev/null+++ b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts@@ -0,0 +1,118 @@+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { Filter } from "incur";+import { expect } from "vitest";+import { z } from "zod";++import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++const filter = "evidence_id,normalized_result.graph.nodes[0:1].node_id";++for (const command of ["analyze-javascript-application", "analyze"]) {+ for (const format of ["json", "jsonl"]) {+ for (const fullOutput of [false, true]) {+ for (const filtered of [false, true]) {+ cliTest(+ `${command} preserves ${format}, full output ${String(fullOutput)}, filtering ${String(filtered)}`,+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-js-json-output-");+ await writeFile(join(root, "main.js"), "fetch('');\n");+ const direct = await analyzeJavaScriptApplication({+ input_path: root,+ });+ if (!direct.ok) throw direct.error;+ const output = await cli.run({+ arguments: [+ command,+ root,+ "--format",+ format,+ ...(fullOutput ? ["--full-output"] : []),+ ...(filtered ? ["--filter-output", filter] : []),+ ],+ });+ expect(output.exitCode).toBe(0);+ let data: unknown = output.json;+ if (fullOutput) {+ const envelope = z+ .object({+ ok: z.literal(true),+ data: z.unknown(),+ meta: z.object({+ command: z.literal(command),+ duration: z.string().regex(/^\d+ms$/u),+ }),+ })+ .parse(output.json);+ data = envelope.data;+ }+ expect(data).toEqual(+ filtered+ ? Filter.apply(direct.value, Filter.parse(filter))+ : direct.value,+ );+ },+ );+ }+ }+ }+}++cliTest(+ "preserves a complete empty filtered JSON document",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-js-json-empty-filter-");+ await writeFile(join(root, "main.js"), "export const value = 1;\n");+ const output = await cli.run({+ arguments: [+ "analyze-javascript-application",+ root,+ "--json",+ "--filter-output",+ "absent.field",+ ],+ });+ expect(output.exitCode).toBe(0);+ expect(output.json).toEqual({});+ },+);++cliTest(+ "preserves a typed failed analysis in the JSON result",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-js-json-failure-");+ const output = await cli.run({+ arguments: [+ "analyze-javascript-application",+ join(root, "absent"),+ "--json",+ ],+ });+ expect(output.json).toMatchObject({+ code: "artifact_operation_failed",+ details: { reason: "io" },+ });+ },+);++cliTest(+ "preserves JSON omission for inherited filter methods",+ async ({ cli }) => {+ const root = await createTestTempDirectory("rea-js-json-inherited-filter-");+ await writeFile(join(root, "main.js"), "export const value = 1;\n");+ const output = await cli.run({+ arguments: [+ "analyze-javascript-application",+ root,+ "--json",+ "--filter-output",+ "toString",+ ],+ });+ expect(output.exitCode).toBe(0);+ expect(output.json).toEqual({});+ },+);diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.tsnew file mode 100644index 000000000..7c1ff4ddd--- /dev/null+++ b/tests/boundary/cli/jsonOutput.test.ts@@ -0,0 +1,170 @@+import { Writable } from "node:stream";++import { describe, expect, it } from "vitest";++import { writeJsonOutput } from "../../../src/cli/jsonOutput.js";+import { createStreamedCliJsonOutput } from "../../../src/cli/streamedJsonOutput.js";++const collect = async (+ value: unknown,+ format: "json" | "jsonl" = "json",+): Promise<string> => {+ const chunks: Buffer[] = [];+ const destination = new Writable({+ write(chunk: unknown, _encoding, callback) {+ if (!Buffer.isBuffer(chunk)) {+ callback(new TypeError("Expected a UTF-8 output buffer"));+ return;+ }+ chunks.push(chunk);+ callback();+ },+ });+ await writeJsonOutput(value, destination, format);+ return Buffer.concat(chunks).toString("utf8");+};++describe("incremental CLI JSON", () => {+ it.each([+ null,+ true,+ -0,+ 1e-7,+ "",+ { empty: [], object: {}, nested: [{ value: null }] },+ { 'quoted"key': '\u0000\b\t\n\f\r\\"', unicode: "้ช๐ \ud800\udfff" },+ { text: `${"x".repeat(8191)}๐ ${"y".repeat(8191)}\ud800` },+ ])("matches native JSON encoding for %j", async (value) => {+ expect(await collect(value)).toBe(`${JSON.stringify(value, null, 2)}\n`);+ });++ it("preserves shared objects without interpreting them as cycles", async () => {+ const shared = { value: "shared" };+ const value = { copies: [shared, shared] };+ expect(await collect(value)).toBe(`${JSON.stringify(value, null, 2)}\n`);+ });++ it("writes compact JSONL records and preserves an empty record set", async () => {+ const value = [{ text: "first\nrecord" }, { text: "้ช๐ " }];+ expect(await collect(value, "jsonl")).toBe(+ `${value.map((item) => JSON.stringify(item)).join("\n")}\n`,+ );+ expect(await collect([], "jsonl")).toBe("");+ });++ it("bounds escaped leaf writes and waits for a slow destination", async () => {+ const leaf = `\u0000้ช๐ `.repeat(32 * 1024);+ const chunks: Buffer[] = [];+ let largestChunk = 0;+ const destination = new Writable({+ highWaterMark: 1,+ write(chunk: unknown, _encoding, callback) {+ if (!Buffer.isBuffer(chunk)) {+ callback(new TypeError("Expected UTF-8 bytes"));+ return;+ }+ largestChunk = Math.max(largestChunk, chunk.length);+ setTimeout(() => {+ chunks.push(chunk);+ callback();+ }, 1);+ },+ });+ await writeJsonOutput({ leaf }, destination);+ expect(largestChunk).toBeLessThan(Buffer.byteLength(leaf));+ const actual = Buffer.concat(chunks).toString("utf8");+ const expected = `${JSON.stringify({ leaf }, null, 2)}\n`;+ expect(actual.length).toBe(expected.length);+ expect(actual === expected).toBe(true);+ expect(destination.writableEnded).toBe(false);+ });++ it("waits for the final write even when it fits the destination buffer", async () => {+ let completed = false;+ const destination = new Writable({+ write(_chunk, _encoding, callback) {+ setTimeout(() => {+ completed = true;+ callback();+ }, 5);+ },+ });+ await writeJsonOutput({ result: "small" }, destination);+ expect(completed).toBe(true);+ });++ it.each([false, true])(+ "propagates a failed destination (deferred: %s)",+ async (deferred) => {+ const cause = new Error("Fixture destination failed");+ const destination = new Writable({+ write(_chunk, _encoding, callback) {+ if (deferred) setTimeout(() => callback(cause), 5);+ else callback(cause);+ },+ });+ await expect(+ writeJsonOutput({ result: "small" }, destination),+ ).rejects.toBe(cause);+ },+ );++ it("leaves a reusable destination open without adding error listeners", async () => {+ const destination = new Writable({+ write(_chunk, _encoding, callback) {+ callback();+ },+ });+ const listeners = destination.listenerCount("error");+ await writeJsonOutput({ result: "first" }, destination);+ await writeJsonOutput({ result: "second" }, destination);+ expect(destination.listenerCount("error")).toBe(listeners);+ expect(destination.writableEnded).toBe(false);+ });++ it("rejects a cycle at the serialization boundary", async () => {+ const cycle: Record<string, unknown> = {};+ cycle.self = cycle;+ await expect(collect(cycle)).rejects.toThrow("circular reference");+ });++ it.each([undefined, Number.NaN, Number.POSITIVE_INFINITY, 1n, new Date(0)])(+ "rejects non-JSON input %s",+ async (value) => {+ await expect(collect(value)).rejects.toThrow("ordinary JSON");+ },+ );+});++describe("streamed CLI result surface", () => {+ it.each([+ [],+ ["--format", "md"],+ ["--json", "--token-count"],+ ["--json", "--token-limit", "5"],+ ["--json", "--format", "invalid"],+ ])("delegates unsupported output controls %j", (...arguments_) => {+ expect(createStreamedCliJsonOutput(arguments_, new Writable())).toBe(+ undefined,+ );+ });++ it("keeps ownership after a failed write so no second document is emitted", async () => {+ const cause = new Error("Fixture destination failed");+ const destination = new Writable({+ write(_chunk, _encoding, callback) {+ callback(cause);+ },+ });+ const output = createStreamedCliJsonOutput(["--json"], destination);+ if (output === undefined) throw new Error("Missing JSON output surface");+ await expect(+ output.write(+ { result: "small" },+ { command: "fixture", duration: "1ms", format: "json" },+ ),+ ).rejects.toBe(cause);+ expect(output.handled).toBe(true);+ expect(output.failed).toBe(true);+ });+});