Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(cli): stream large JavaScript JSON results (#1024)

Ysopwnable0e25befe78f2
diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md
index 42d40e938..d354dc570 100644
--- a/docs/javascript-artifact-reconstruction.md
+++ b/docs/javascript-artifact-reconstruction.md
@@ -65,11 +65,19 @@ identifiers remain unchanged. The opt-in regression check is
`npm run verify:javascript:digests`;
it hashes a value larger than the running Node engine's single-string limit.
-Output formatters and MCP transport serialization still assemble whole strings.
-For a large CLI result, select the needed fields
-before formatting, for example `--format json --filter-output
-evidence_id,normalized_result.statistics`. Complete serialization of a result
-beyond the engine's string limit is not established by this digest check.
+Both application CLI routes stream complete JSON and JSONL output in bounded
+chunks, including field filters and `--full-output`. They wait for each stdout
+write instead of constructing or reparsing a document-sized string. The opt-in
+`npm run verify:javascript:output` checks the actual CLI formatting boundary with
+a temporary output file larger than the running engine's string limit, verifies
+its bytes and digest, and removes it. Use `-- jsonl` for the compact JSONL check.
+Each check needs space for one output file plus a 1 GiB free-space reserve.
+
+Other CLI formats, `--token-count`, and MCP transport serialization still
+assemble whole strings. Field selection remains useful when the caller needs a
+smaller view, for example `--format json --filter-output
+evidence_id,normalized_result.statistics`. Streaming output does not bound the
+memory needed to construct the analysis graph itself.
Client framing limits also apply: the pinned Node MCP SDK's stdio transport
defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate
the complete response, including text and structured Evidence projections.
diff --git a/docs/testing.md b/docs/testing.md
index 740d7958e..feb2bdf54 100644
--- a/docs/testing.md
+++ b/docs/testing.md
@@ -561,6 +561,18 @@ conditional `real-web-source-map` CI job supplies Chrome and an isolated pinned
fixture compiler; static/unit checks do not acquire a browser. See
[the source location guide](web-source-location.md) for the verified decoder profile.
+### JavaScript large-output lane
+
+`npm run verify:javascript:output` exercises the CLI JSON result surface beyond
+the running Node engine's single-string limit. Shared input leaves keep the
+fixture's graph small; the verifier writes one temporary output file, checks its
+complete byte count and an independent digest, then removes it. It requires only
+Node and the built REA runtime, with space for the output plus a 1 GiB reserve.
+Run `npm run verify:javascript:output -- jsonl` for compact JSONL coverage. This
+opt-in lane is separate from routine tests and the canonical hash check
+`npm run verify:javascript:digests`. It verifies serialization rather than an
+arbitrary third-party application's parsing cost or MCP client capacity.
+
### Website runtime attribution lane
`npm run verify:browser:runtime` uses caller-supplied
diff --git a/package.json b/package.json
index 4b2b72f45..3a98037bc 100644
--- a/package.json
+++ b/package.json
@@ -141,6 +141,7 @@
"verify:inspector": "npm run build:cached && node scripts/verify/inspector/runtime-observation.mjs",
"verify:javascript-runtime": "npm run build:cached && node scripts/verify/javascript/runtime-observation.mjs",
"verify:javascript:digests": "npm run build:cached && node scripts/verify/javascript/digests.mjs",
+ "verify:javascript:output": "npm run build:cached && node scripts/verify/javascript/output.mjs",
"verify:application-workflows": "npm run build:cached && node scripts/verify/javascript/application-workflows.mjs",
"verify:managed": "npm run build:cached && node scripts/verify/managed/conformance.mjs",
"verify:firmware": "npm run build:cached && node scripts/verify/firmware/analysis.mjs",
diff --git a/scripts/rea.mjs b/scripts/rea.mjs
index 4e310bf08..1f720a4d1 100755
--- a/scripts/rea.mjs
+++ b/scripts/rea.mjs
@@ -19,7 +19,7 @@ const runtimeFiles = isMcpMode
? ["dist/main.js"]
: isMcpDoctorMode
? ["dist/main.js", "dist/mcpDoctor.js"]
- : ["dist/cli.js", "dist/cliOutput.js"];
+ : ["dist/cli.js", "dist/cliOutput.js", "dist/cli/streamedJsonOutput.js"];
if (!(await compiledRuntimeExists(runtimeFiles))) {
process.stderr.write(
@@ -39,6 +39,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
process.exitCode = result.exitCode;
} else {
const { createCli } = await import("../dist/cli.js");
+ const { createStreamedCliJsonOutput } =
+ await import("../dist/cli/streamedJsonOutput.js");
const {
renderCliOutputArgumentError,
renderEmptyFilteredCliOutput,
@@ -51,14 +53,20 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
process.exitCode = 1;
} else {
let wroteOutput = false;
- await createCli().serve(args, {
+ const resultOutput = createStreamedCliJsonOutput(args, process.stdout);
+ await createCli(process.env, resultOutput).serve(args, {
stdout: (output) => {
+ if (resultOutput?.handled) {
+ if (resultOutput.failed)
+ process.stderr.write(sanitizeCliOutput(output));
+ return;
+ }
const sanitized = sanitizeCliOutput(output);
if (sanitized.length > 0) wroteOutput = true;
process.stdout.write(sanitized);
},
});
- if (!wroteOutput)
+ if (!wroteOutput && !resultOutput?.handled)
process.stdout.write(renderEmptyFilteredCliOutput(args) ?? "");
}
}
diff --git a/scripts/verify/javascript/output.mjs b/scripts/verify/javascript/output.mjs
new file mode 100644
index 000000000..0e6d5635e
--- /dev/null
+++ b/scripts/verify/javascript/output.mjs
@@ -0,0 +1,134 @@
+import assert from "node:assert/strict";
+import { constants } from "node:buffer";
+import { createHash } from "node:crypto";
+import { once } from "node:events";
+import { createReadStream, createWriteStream } from "node:fs";
+import { mkdtemp, rm, stat, statfs } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { finished } from "node:stream/promises";
+
+import { Cli } from "incur";
+
+import { createStreamedCliJsonOutput } from "../../../dist/cli/streamedJsonOutput.js";
+import { createEvidence } from "../../../dist/domain/evidence.js";
+
+// This opt-in format-boundary check writes one temporary file larger than the
+// actual engine string limit. Shared leaves keep the analysis input small.
+const format = process.argv[2] ?? "json";
+assert.ok(format === "json" || format === "jsonl", "Use json or jsonl");
+const leaf = "x".repeat(64 * 1024);
+const encodedLeaf = JSON.stringify(leaf);
+const count = Math.ceil(constants.MAX_STRING_LENGTH / (leaf.length + 3)) + 1;
+const volume = await statfs(tmpdir());
+assert.ok(
+ volume.bavail * volume.bsize > count * encodedLeaf.length + 1024 ** 3,
+ "Large-output verification needs room for one output file and a 1-GiB reserve",
+);
+const workspace = await mkdtemp(join(tmpdir(), "rea-javascript-output-"));
+const path = join(workspace, "result.json");
+let destination;
+let peakRss = process.memoryUsage().rss;
+const sampleRss = () => {
+ peakRss = Math.max(peakRss, process.memoryUsage().rss);
+};
+const sampler = setInterval(sampleRss, 25);
+sampler.unref();
+
+try {
+ const evidence = createEvidence(
+ undefined,
+ { id: "large-output-fixture", name: "Large output fixture", version: "1" },
+ {
+ operation: "inspect_fixture",
+ parameters: {},
+ result: { payload: Array.from({ length: count }, () => leaf) },
+ },
+ );
+ sampleRss();
+ const metadata = { command: "inspect-output-fixture", duration: "0ms" };
+ const marker = "__REA_OUTPUT_ENCODING_MARKER__";
+ const template = {
+ ok: true,
+ data: { ...evidence, normalized_result: { payload: [marker] } },
+ meta: metadata,
+ };
+ const skeleton = JSON.stringify(template, null, format === "json" ? 2 : 0);
+ const encodedMarker = JSON.stringify(marker);
+ const markerIndex = skeleton.indexOf(encodedMarker);
+ assert.ok(markerIndex >= 0);
+ assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker));
+ const prefix = skeleton.slice(0, markerIndex);
+ const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`;
+ const indentation = prefix.slice(prefix.lastIndexOf("\n") + 1);
+ const separator = format === "json" ? `,\n${indentation}` : ",";
+ const expectedHash = createHash("sha256");
+ expectedHash.update(prefix);
+ for (let index = 0; index < count; index += 1) {
+ if (index > 0) expectedHash.update(separator);
+ expectedHash.update(encodedLeaf);
+ }
+ expectedHash.update(suffix);
+ const expectedBytes =
+ Buffer.byteLength(prefix) +
+ count * Buffer.byteLength(encodedLeaf) +
+ (count - 1) * Buffer.byteLength(separator) +
+ Buffer.byteLength(suffix);
+ assert.ok(expectedBytes > constants.MAX_STRING_LENGTH);
+
+ destination = createWriteStream(path, { flags: "wx", mode: 0o600 });
+ const argv = [metadata.command, "--format", format, "--full-output"];
+ const output = createStreamedCliJsonOutput(argv, destination);
+ assert.ok(output);
+ let unexpectedOutputBytes = 0;
+ let exitCode = 0;
+ await Cli.create("rea-output-verifier", { sync: false })
+ .command(metadata.command, {
+ run: async ({ format: selectedFormat }) => {
+ assert.equal(
+ await output.write(evidence, { ...metadata, format: selectedFormat }),
+ true,
+ );
+ },
+ })
+ .serve(argv, {
+ stdout: (text) => {
+ if (!output.handled) unexpectedOutputBytes += Buffer.byteLength(text);
+ },
+ exit: (code) => {
+ exitCode = code;
+ },
+ });
+ destination.end();
+ await finished(destination);
+ assert.equal(exitCode, 0);
+ assert.equal(output.handled, true);
+ assert.equal(output.failed, false);
+ assert.equal(unexpectedOutputBytes, 0);
+
+ const actualHash = createHash("sha256");
+ for await (const chunk of createReadStream(path)) actualHash.update(chunk);
+ const digest = actualHash.digest("hex");
+ assert.equal(digest, expectedHash.digest("hex"));
+ assert.equal((await stat(path)).size, expectedBytes);
+ sampleRss();
+ process.stdout.write(
+ `${JSON.stringify({
+ json_output_verified: true,
+ format,
+ output_bytes: expectedBytes,
+ engine_string_limit: constants.MAX_STRING_LENGTH,
+ output_sha256: digest,
+ evidence_id: evidence.evidence_id,
+ rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
+ })}\n`,
+ );
+} finally {
+ clearInterval(sampler);
+ if (destination !== undefined && !destination.closed) {
+ const closed = once(destination, "close");
+ destination.destroy();
+ await closed;
+ }
+ await rm(workspace, { recursive: true, force: true });
+}
diff --git a/src/cli.ts b/src/cli.ts
index 8ce590f05..cd9b683e5 100644
--- a/src/cli.ts
+++ b/src/cli.ts
@@ -23,6 +23,7 @@ import { registerElectronCommands } from "./cli/electronCommands.js";
import { registerJavaScriptRuntimeObservationCommands } from "./cliJavaScriptRuntimeCommands.js";
import { registerApplicationCommands } from "./cli/applicationCommands.js";
import type { CliInstance } from "./cli/types.js";
+import type { CliResultOutput } from "./cli/streamedJsonOutput.js";
/**
* Build the one-shot Incur CLI without starting Hopper at import time.
@@ -31,6 +32,7 @@ import type { CliInstance } from "./cli/types.js";
*/
export const createCli = (
environment: Readonly<Record<string, string | undefined>> = process.env,
+ resultOutput?: CliResultOutput,
): CliInstance => {
const logger = createLogger(
"cli",
@@ -50,7 +52,7 @@ export const createCli = (
});
registerSetupCommands(cli, logger);
- registerCoreAnalysisCommands(cli, logger);
+ registerCoreAnalysisCommands(cli, logger, resultOutput);
registerUtilityCommands(cli, logger, environment);
registerArtifactCommands(cli, logger);
registerManagedCommands(cli, logger);
@@ -66,7 +68,7 @@ export const createCli = (
registerJavaScriptRecoveryCommands(cli, logger, environment);
registerAdvancedBrowserCommands(cli, logger);
registerBrowserScenarioCommands(cli, logger);
- registerElectronCommands(cli, logger);
+ registerElectronCommands(cli, logger, resultOutput);
registerJavaScriptRuntimeObservationCommands(cli, logger);
registerApplicationCommands(cli, logger);
return cli;
diff --git a/src/cli/coreAnalysisCommands.ts b/src/cli/coreAnalysisCommands.ts
index da50ca830..e323348da 100644
--- a/src/cli/coreAnalysisCommands.ts
+++ b/src/cli/coreAnalysisCommands.ts
@@ -2,12 +2,14 @@ import type { Logger } from "../logger.js";
import { registerCoreBinaryCommands } from "./coreBinaryCommands.js";
import { registerCoreNativeCommands } from "./coreNativeCommands.js";
import type { CliInstance } from "./types.js";
+import type { CliResultOutput } from "./streamedJsonOutput.js";
/** Register core binary and native deep-analysis CLI commands. */
export const registerCoreAnalysisCommands = (
cli: CliInstance,
logger: Logger,
+ resultOutput?: CliResultOutput,
): void => {
- registerCoreBinaryCommands(cli, logger);
+ registerCoreBinaryCommands(cli, logger, resultOutput);
registerCoreNativeCommands(cli, logger);
};
diff --git a/src/cli/coreBinaryCommands.ts b/src/cli/coreBinaryCommands.ts
index 6c70a2626..441d0e147 100644
--- a/src/cli/coreBinaryCommands.ts
+++ b/src/cli/coreBinaryCommands.ts
@@ -13,13 +13,18 @@ import {
} from "./options.js";
import type { CliInstance } from "./types.js";
import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";
+import type {
+ CliCommandOutput,
+ CliResultOutput,
+} from "./streamedJsonOutput.js";
/** Register provider-neutral binary overview and procedure CLI commands. */
export const registerCoreBinaryCommands = (
cli: CliInstance,
logger: Logger,
+ resultOutput?: CliResultOutput,
): void => {
- registerOverviewCommands(cli, logger);
+ registerOverviewCommands(cli, logger, resultOutput);
registerFunctionCommand(cli, logger);
registerInstructionsCommand(cli, logger);
registerSearchCommand(cli, logger);
@@ -27,7 +32,11 @@ export const registerCoreBinaryCommands = (
registerTraceCommand(cli, logger);
};
-const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {
+const registerOverviewCommands = (
+ cli: CliInstance,
+ logger: Logger,
+ resultOutput?: CliResultOutput,
+): void => {
const overviewOptions = z.object({
snapshot: z
.string()
@@ -43,9 +52,16 @@ const registerOverviewCommands = (cli: CliInstance, logger: Logger): void => {
path: z.string().describe("App, program, or analysis database path"),
}),
options: overviewOptions,
- run: ({ args, options }) =>
+ run: ({ args, options, format }) =>
logCliCommand(logger, "analyze", () =>
- runRoutedOverview(args.path, options, logger),
+ runRoutedOverview(
+ args.path,
+ options,
+ logger,
+ resultOutput === undefined
+ ? undefined
+ : { output: resultOutput, command: CLI_COMMANDS.analyze, format },
+ ),
),
});
cli.command(CLI_COMMANDS.inspect, {
@@ -109,6 +125,7 @@ const runRoutedOverview = async (
readonly provider?: string | undefined;
},
logger: Logger,
+ output?: CliCommandOutput,
) => {
if (
options.provider === undefined &&
@@ -116,9 +133,10 @@ const runRoutedOverview = async (
options["target-format"] === undefined &&
(await isJavaScriptApplicationPath(path))
)
- return runCliJavaScriptApplicationAnalysis({
- input_path: resolve(path),
- });
+ return runCliJavaScriptApplicationAnalysis(
+ { input_path: resolve(path) },
+ output,
+ );
return runDirectAnalysis(
path,
"binary_overview",
diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts
index e91b1ff63..e747dc5c9 100644
--- a/src/cli/electronCommands.ts
+++ b/src/cli/electronCommands.ts
@@ -24,15 +24,17 @@ import {
javascriptApplicationOptions,
} from "../cliObservationOptions.js";
import { runCliJavaScriptApplicationAnalysis } from "./javascriptApplicationAnalysis.js";
+import type { CliResultOutput } from "./streamedJsonOutput.js";
/** Register CLI equivalents of the Electron MCP tools. */
export const registerElectronCommands = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
+ resultOutput?: CliResultOutput,
): void => {
registerElectronObservationCommands(cli, logger);
registerElectronActiveCommand(cli, logger);
- registerJavaScriptApplicationCommand(cli, logger);
+ registerJavaScriptApplicationCommand(cli, logger, resultOutput);
registerJavaScriptRuntimeReconciliationCommand(cli, logger);
};
@@ -172,6 +174,7 @@ const registerElectronPageInspection = (
const registerJavaScriptApplicationCommand = (
cli: ReturnType<typeof Cli.create>,
logger: Logger,
+ resultOutput?: CliResultOutput,
): void => {
cli.command(CLI_COMMANDS.analyzeJavaScriptApplication, {
description:
@@ -180,12 +183,18 @@ const registerJavaScriptApplicationCommand = (
path: z.string().describe("ASAR or extracted application path"),
}),
options: javascriptApplicationOptions,
- run: ({ args, options }) =>
+ run: ({ args, options, format }) =>
logCliCommand(logger, CLI_COMMANDS.analyzeJavaScriptApplication, () =>
- runCliJavaScriptApplicationAnalysis({
- input_path: args.path,
- format: options.artifactFormat,
- }),
+ runCliJavaScriptApplicationAnalysis(
+ { input_path: args.path, format: options.artifactFormat },
+ resultOutput === undefined
+ ? undefined
+ : {
+ output: resultOutput,
+ command: CLI_COMMANDS.analyzeJavaScriptApplication,
+ format,
+ },
+ ),
),
});
};
diff --git a/src/cli/javascriptApplicationAnalysis.ts b/src/cli/javascriptApplicationAnalysis.ts
index 62a8bc5b0..92d4696c4 100644
--- a/src/cli/javascriptApplicationAnalysis.ts
+++ b/src/cli/javascriptApplicationAnalysis.ts
@@ -2,11 +2,14 @@ import { analyzeJavaScriptApplication } from "../application/javascript/JavaScri
import { createProgressReporter } from "../application/ProgressReporter.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import type { JsonValue } from "../domain/jsonValue.js";
+import type { CliCommandOutput } from "./streamedJsonOutput.js";
/** Execute the shared one-shot CLI boundary for static JavaScript analysis. */
export const runCliJavaScriptApplicationAnalysis = async (
input: unknown,
-): Promise<JsonValue> => {
+ output?: CliCommandOutput,
+): Promise<JsonValue | undefined> => {
+ const started = performance.now();
const progress = createProgressReporter(
async (update) => {
process.stderr.write(`${JSON.stringify({ rea_progress: update })}\n`);
@@ -14,7 +17,17 @@ export const runCliJavaScriptApplicationAnalysis = async (
{ minimumIntervalMs: 0 },
);
const result = await analyzeJavaScriptApplication(input, { progress });
- return result.ok ? result.value : cliError(result.error);
+ const value = result.ok ? result.value : cliError(result.error);
+ if (
+ output !== undefined &&
+ (await output.output.write(value, {
+ command: output.command,
+ format: output.format,
+ duration: `${Math.round(performance.now() - started)}ms`,
+ }))
+ )
+ return undefined;
+ return value;
};
const cliError = (
diff --git a/src/cli/jsonOutput.ts b/src/cli/jsonOutput.ts
new file mode 100644
index 000000000..0e7e75503
--- /dev/null
+++ b/src/cli/jsonOutput.ts
@@ -0,0 +1,149 @@
+import type { Writable } from "node:stream";
+
+type JsonOutputFormat = "json" | "jsonl";
+
+/** Write ordinary JSON without constructing a document-sized string. */
+export const writeJsonOutput = async (
+ value: unknown,
+ destination: Writable,
+ format: JsonOutputFormat = "json",
+): Promise<void> => {
+ for (const part of bufferedParts(documentParts(value, format)))
+ await writePart(destination, part);
+};
+
+const writePart = (destination: Writable, part: string): Promise<void> =>
+ new Promise((resolve, reject) => {
+ const onError = (cause: unknown): void => reject(cause);
+ destination.once("error", onError);
+ try {
+ destination.write(part, (cause) => {
+ if (cause) {
+ // A Writable can emit its error after invoking the write callback.
+ setImmediate(() => destination.off("error", onError));
+ reject(cause);
+ } else {
+ destination.off("error", onError);
+ resolve();
+ }
+ });
+ } catch (cause: unknown) {
+ destination.off("error", onError);
+ reject(cause);
+ }
+ });
+
+function* documentParts(
+ value: unknown,
+ format: JsonOutputFormat,
+): Generator<string> {
+ if (format === "jsonl" && Array.isArray(value)) {
+ for (const item of value) {
+ yield* encodeValue(item, false, 0, new Set());
+ yield "\n";
+ }
+ return;
+ }
+ yield* encodeValue(value, format === "json", 0, new Set());
+ yield "\n";
+}
+
+function* bufferedParts(parts: Iterable<string>): Generator<string> {
+ let buffer = "";
+ for (const part of parts) {
+ if (buffer.length + part.length > 64 * 1024 && buffer !== "") {
+ yield buffer;
+ buffer = "";
+ }
+ buffer += part;
+ }
+ if (buffer !== "") yield buffer;
+}
+
+function* encodeString(value: string): Generator<string> {
+ yield '"';
+ for (let start = 0; start < value.length;) {
+ let end = Math.min(start + 8192, value.length);
+ const last = value.charCodeAt(end - 1);
+ const next = value.charCodeAt(end);
+ // Keep a surrogate pair in the same JSON.stringify and UTF-8 write.
+ if (
+ end < value.length &&
+ last >= 0xd800 &&
+ last <= 0xdbff &&
+ next >= 0xdc00 &&
+ next <= 0xdfff
+ )
+ end -= 1;
+ yield JSON.stringify(value.slice(start, end)).slice(1, -1);
+ start = end;
+ }
+ yield '"';
+}
+
+function* encodeValue(
+ value: unknown,
+ pretty: boolean,
+ depth: number,
+ ancestors: Set<object>,
+): Generator<string> {
+ if (typeof value === "string") {
+ yield* encodeString(value);
+ return;
+ }
+ if (value === null || typeof value === "boolean") {
+ yield value === null ? "null" : String(value);
+ return;
+ }
+ if (typeof value === "number" && Number.isFinite(value)) {
+ yield String(value);
+ return;
+ }
+ if (typeof value !== "object" || value === null)
+ throw new TypeError("CLI JSON output requires ordinary JSON values");
+ if (ancestors.has(value))
+ throw new TypeError("CLI JSON output contains a circular reference");
+ if (
+ !Array.isArray(value) &&
+ Object.prototype.toString.call(value) !== "[object Object]"
+ )
+ throw new TypeError("CLI JSON output requires ordinary JSON objects");
+ ancestors.add(value);
+ try {
+ const array = Array.isArray(value);
+ const keys = array
+ ? undefined
+ : Object.keys(value).filter((key) => {
+ const item: unknown = Reflect.get(value, key);
+ // Incur's filter can select inherited methods. Match native JSON's
+ // omission of fields that have no JSON representation.
+ return (
+ item !== undefined &&
+ typeof item !== "function" &&
+ typeof item !== "symbol"
+ );
+ });
+ const count = array ? value.length : (keys?.length ?? 0);
+ yield array ? "[" : "{";
+ for (let index = 0; index < count; index += 1) {
+ if (index > 0) yield ",";
+ if (pretty) yield `\n${" ".repeat(depth + 1)}`;
+ let item: unknown;
+ if (array) {
+ item = value[index];
+ } else {
+ const key = keys?.[index];
+ if (key === undefined)
+ throw new TypeError("CLI JSON object changed during serialization");
+ yield* encodeString(key);
+ yield pretty ? ": " : ":";
+ item = Reflect.get(value, key);
+ }
+ yield* encodeValue(item, pretty, depth + 1, ancestors);
+ }
+ if (pretty && count > 0) yield `\n${" ".repeat(depth)}`;
+ yield array ? "]" : "}";
+ } finally {
+ ancestors.delete(value);
+ }
+}
diff --git a/src/cli/streamedJsonOutput.ts b/src/cli/streamedJsonOutput.ts
new file mode 100644
index 000000000..282316d0a
--- /dev/null
+++ b/src/cli/streamedJsonOutput.ts
@@ -0,0 +1,84 @@
+import { type Writable } from "node:stream";
+
+import { Filter } from "incur";
+
+import { parseCliOutputArguments } from "../cliOutput.js";
+import type { JsonValue } from "../domain/jsonValue.js";
+import { writeJsonOutput } from "./jsonOutput.js";
+
+/** Result metadata supplied by the command that actually ran. */
+export interface CliResultOutputMetadata {
+ readonly command: string;
+ readonly duration: string;
+ readonly format: string;
+}
+
+/** Optional executable-owned result surface for one CLI invocation. */
+export interface CliResultOutput {
+ readonly handled: boolean;
+ readonly failed: boolean;
+ write(value: JsonValue, metadata: CliResultOutputMetadata): Promise<boolean>;
+}
+
+/** Selected command output context, independent of application workflows. */
+export interface CliCommandOutput {
+ readonly output: CliResultOutput;
+ readonly command: string;
+ readonly format: string;
+}
+
+/** Preserve Incur's JSON controls while streaming the complete result. */
+export const createStreamedCliJsonOutput = (
+ arguments_: readonly string[],
+ destination: Writable,
+): CliResultOutput | undefined => {
+ const options = parseCliOutputArguments(arguments_);
+ if (
+ options.parseError ||
+ options.tokenCount ||
+ options.tokenWindow ||
+ (options.format !== "json" && options.format !== "jsonl")
+ )
+ return undefined;
+ const format = options.format;
+ let handled = false;
+ let failed = false;
+ return {
+ get handled() {
+ return handled;
+ },
+ get failed() {
+ return failed;
+ },
+ async write(value, metadata) {
+ if (metadata.format !== format) return false;
+ const filtered: unknown = options.filterOutput
+ ? Filter.apply(value, Filter.parse(options.filterOutput))
+ : value;
+ const meta = {
+ command: metadata.command,
+ duration: metadata.duration,
+ };
+ const document = options.fullOutput
+ ? filtered === undefined
+ ? { ok: true, meta }
+ : { ok: true, data: filtered, meta }
+ : filtered === undefined ||
+ filtered === null ||
+ typeof filtered === "function" ||
+ typeof filtered === "symbol"
+ ? {}
+ : filtered;
+ // Once writing starts, a failed destination must not receive a second
+ // document from Incur's error formatter.
+ handled = true;
+ try {
+ await writeJsonOutput(document, destination, format);
+ } catch (cause: unknown) {
+ failed = true;
+ throw cause;
+ }
+ return true;
+ },
+ };
+};
diff --git a/src/cliOutput.ts b/src/cliOutput.ts
index 063a6d452..a5b359992 100644
--- a/src/cliOutput.ts
+++ b/src/cliOutput.ts
@@ -20,9 +20,11 @@ export type CliOutputArgumentValidation =
type JsonRecord = Record<string, unknown>;
interface ParsedCliOutputArguments {
- readonly filterOutput: boolean;
+ readonly filterOutput: string | undefined;
readonly format: string;
+ readonly fullOutput: boolean;
readonly parseError: boolean;
+ readonly tokenCount: boolean;
readonly tokenWindow: boolean;
}
@@ -36,12 +38,23 @@ const validationError = (value: JsonRecord): JsonRecord | undefined => {
return undefined;
};
-const parseCliOutputArguments = (
+/** Read effective Incur output controls at the executable boundary. */
+export const parseCliOutputArguments = (
arguments_: readonly string[],
): ParsedCliOutputArguments => {
let format: string = "toon";
- let filterOutput = false;
+ let filterOutput: string | undefined;
+ let fullOutput = false;
+ let tokenCount = false;
let tokenWindow = false;
+ const parsed = (parseError: boolean): ParsedCliOutputArguments => ({
+ filterOutput,
+ format,
+ fullOutput,
+ parseError,
+ tokenCount,
+ tokenWindow: parseError ? false : tokenWindow,
+ });
for (let index = 0; index < arguments_.length; index += 1) {
const argument = arguments_[index];
// Incur scans builtins across `--` and accepts only spaced value forms.
@@ -55,8 +68,7 @@ const parseCliOutputArguments = (
if (value) {
// Incur stops parsing at an invalid format before applying any later
// output controls. Let it report that malformed flag itself.
- if (!INCUR_OUTPUT_FORMATS.has(value))
- return { filterOutput, format, parseError: true, tokenWindow: false };
+ if (!INCUR_OUTPUT_FORMATS.has(value)) return parsed(true);
format = value;
index += 1;
}
@@ -64,7 +76,7 @@ const parseCliOutputArguments = (
}
if (argument === "--filter-output") {
if (arguments_[index + 1]) {
- filterOutput = true;
+ filterOutput = arguments_[index + 1];
index += 1;
}
continue;
@@ -74,14 +86,16 @@ const parseCliOutputArguments = (
if (value) {
const numericValue = Number(value);
if (!Number.isFinite(numericValue) || value.trim() === "")
- return { filterOutput, format, parseError: true, tokenWindow: false };
+ return parsed(true);
tokenWindow = true;
index += 1;
}
continue;
}
+ if (argument === "--full-output") fullOutput = true;
+ if (argument === "--token-count") tokenCount = true;
}
- return { filterOutput, format, parseError: false, tokenWindow };
+ return parsed(false);
};
/** Reject text-window controls that would corrupt a structured document. */
diff --git a/tests/boundary/cli/dispatcher.test.ts b/tests/boundary/cli/dispatcher.test.ts
index d94cb7ec7..79d2913b9 100644
--- a/tests/boundary/cli/dispatcher.test.ts
+++ b/tests/boundary/cli/dispatcher.test.ts
@@ -11,6 +11,7 @@ describe("executable dispatcher", () => {
async ({ processes, workspace }) => {
const scripts = await workspace.mkdir("scripts");
await workspace.mkdir("dist");
+ await workspace.mkdir("dist/cli");
await Promise.all([
copyFile("scripts/rea.mjs", join(scripts, "rea.mjs")),
workspace.write(
@@ -29,6 +30,10 @@ describe("executable dispatcher", () => {
"dist/cliOutput.js",
"export const sanitizeCliOutput = (output) => output; export const validateCliOutputArguments = () => ({ ok: true }); export const renderCliOutputArgumentError = () => ''; export const renderEmptyFilteredCliOutput = () => undefined;\n",
),
+ workspace.write(
+ "dist/cli/streamedJsonOutput.js",
+ "export const createStreamedCliJsonOutput = () => undefined;\n",
+ ),
workspace.write(
"dist/mcpDoctor.js",
'export const runProductionMcpDoctorCli = (args) => Promise.resolve({ output: JSON.stringify({ adapter: "mcp-doctor", args }), exitCode: 0 });\n',
diff --git a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts
new file mode 100644
index 000000000..05dc4fc06
--- /dev/null
+++ b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts
@@ -0,0 +1,118 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { Filter } from "incur";
+import { expect } from "vitest";
+import { z } from "zod";
+
+import { analyzeJavaScriptApplication } from "../../../src/application/javascript/JavaScriptApplicationService.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+const filter = "evidence_id,normalized_result.graph.nodes[0:1].node_id";
+
+for (const command of ["analyze-javascript-application", "analyze"]) {
+ for (const format of ["json", "jsonl"]) {
+ for (const fullOutput of [false, true]) {
+ for (const filtered of [false, true]) {
+ cliTest(
+ `${command} preserves ${format}, full output ${String(fullOutput)}, filtering ${String(filtered)}`,
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-js-json-output-");
+ await writeFile(join(root, "main.js"), "fetch('');\n");
+ const direct = await analyzeJavaScriptApplication({
+ input_path: root,
+ });
+ if (!direct.ok) throw direct.error;
+ const output = await cli.run({
+ arguments: [
+ command,
+ root,
+ "--format",
+ format,
+ ...(fullOutput ? ["--full-output"] : []),
+ ...(filtered ? ["--filter-output", filter] : []),
+ ],
+ });
+ expect(output.exitCode).toBe(0);
+ let data: unknown = output.json;
+ if (fullOutput) {
+ const envelope = z
+ .object({
+ ok: z.literal(true),
+ data: z.unknown(),
+ meta: z.object({
+ command: z.literal(command),
+ duration: z.string().regex(/^\d+ms$/u),
+ }),
+ })
+ .parse(output.json);
+ data = envelope.data;
+ }
+ expect(data).toEqual(
+ filtered
+ ? Filter.apply(direct.value, Filter.parse(filter))
+ : direct.value,
+ );
+ },
+ );
+ }
+ }
+ }
+}
+
+cliTest(
+ "preserves a complete empty filtered JSON document",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-js-json-empty-filter-");
+ await writeFile(join(root, "main.js"), "export const value = 1;\n");
+ const output = await cli.run({
+ arguments: [
+ "analyze-javascript-application",
+ root,
+ "--json",
+ "--filter-output",
+ "absent.field",
+ ],
+ });
+ expect(output.exitCode).toBe(0);
+ expect(output.json).toEqual({});
+ },
+);
+
+cliTest(
+ "preserves a typed failed analysis in the JSON result",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-js-json-failure-");
+ const output = await cli.run({
+ arguments: [
+ "analyze-javascript-application",
+ join(root, "absent"),
+ "--json",
+ ],
+ });
+ expect(output.json).toMatchObject({
+ code: "artifact_operation_failed",
+ details: { reason: "io" },
+ });
+ },
+);
+
+cliTest(
+ "preserves JSON omission for inherited filter methods",
+ async ({ cli }) => {
+ const root = await createTestTempDirectory("rea-js-json-inherited-filter-");
+ await writeFile(join(root, "main.js"), "export const value = 1;\n");
+ const output = await cli.run({
+ arguments: [
+ "analyze-javascript-application",
+ root,
+ "--json",
+ "--filter-output",
+ "toString",
+ ],
+ });
+ expect(output.exitCode).toBe(0);
+ expect(output.json).toEqual({});
+ },
+);
diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts
new file mode 100644
index 000000000..7c1ff4ddd
--- /dev/null
+++ b/tests/boundary/cli/jsonOutput.test.ts
@@ -0,0 +1,170 @@
+import { Writable } from "node:stream";
+
+import { describe, expect, it } from "vitest";
+
+import { writeJsonOutput } from "../../../src/cli/jsonOutput.js";
+import { createStreamedCliJsonOutput } from "../../../src/cli/streamedJsonOutput.js";
+
+const collect = async (
+ value: unknown,
+ format: "json" | "jsonl" = "json",
+): Promise<string> => {
+ const chunks: Buffer[] = [];
+ const destination = new Writable({
+ write(chunk: unknown, _encoding, callback) {
+ if (!Buffer.isBuffer(chunk)) {
+ callback(new TypeError("Expected a UTF-8 output buffer"));
+ return;
+ }
+ chunks.push(chunk);
+ callback();
+ },
+ });
+ await writeJsonOutput(value, destination, format);
+ return Buffer.concat(chunks).toString("utf8");
+};
+
+describe("incremental CLI JSON", () => {
+ it.each([
+ null,
+ true,
+ -0,
+ 1e-7,
+ "",
+ { empty: [], object: {}, nested: [{ value: null }] },
+ { 'quoted"key': '\u0000\b\t\n\f\r\\"', unicode: "้›ช๐Ÿ \ud800\udfff" },
+ { text: `${"x".repeat(8191)}๐Ÿ ${"y".repeat(8191)}\ud800` },
+ ])("matches native JSON encoding for %j", async (value) => {
+ expect(await collect(value)).toBe(`${JSON.stringify(value, null, 2)}\n`);
+ });
+
+ it("preserves shared objects without interpreting them as cycles", async () => {
+ const shared = { value: "shared" };
+ const value = { copies: [shared, shared] };
+ expect(await collect(value)).toBe(`${JSON.stringify(value, null, 2)}\n`);
+ });
+
+ it("writes compact JSONL records and preserves an empty record set", async () => {
+ const value = [{ text: "first\nrecord" }, { text: "้›ช๐Ÿ " }];
+ expect(await collect(value, "jsonl")).toBe(
+ `${value.map((item) => JSON.stringify(item)).join("\n")}\n`,
+ );
+ expect(await collect([], "jsonl")).toBe("");
+ });
+
+ it("bounds escaped leaf writes and waits for a slow destination", async () => {
+ const leaf = `\u0000้›ช๐Ÿ `.repeat(32 * 1024);
+ const chunks: Buffer[] = [];
+ let largestChunk = 0;
+ const destination = new Writable({
+ highWaterMark: 1,
+ write(chunk: unknown, _encoding, callback) {
+ if (!Buffer.isBuffer(chunk)) {
+ callback(new TypeError("Expected UTF-8 bytes"));
+ return;
+ }
+ largestChunk = Math.max(largestChunk, chunk.length);
+ setTimeout(() => {
+ chunks.push(chunk);
+ callback();
+ }, 1);
+ },
+ });
+ await writeJsonOutput({ leaf }, destination);
+ expect(largestChunk).toBeLessThan(Buffer.byteLength(leaf));
+ const actual = Buffer.concat(chunks).toString("utf8");
+ const expected = `${JSON.stringify({ leaf }, null, 2)}\n`;
+ expect(actual.length).toBe(expected.length);
+ expect(actual === expected).toBe(true);
+ expect(destination.writableEnded).toBe(false);
+ });
+
+ it("waits for the final write even when it fits the destination buffer", async () => {
+ let completed = false;
+ const destination = new Writable({
+ write(_chunk, _encoding, callback) {
+ setTimeout(() => {
+ completed = true;
+ callback();
+ }, 5);
+ },
+ });
+ await writeJsonOutput({ result: "small" }, destination);
+ expect(completed).toBe(true);
+ });
+
+ it.each([false, true])(
+ "propagates a failed destination (deferred: %s)",
+ async (deferred) => {
+ const cause = new Error("Fixture destination failed");
+ const destination = new Writable({
+ write(_chunk, _encoding, callback) {
+ if (deferred) setTimeout(() => callback(cause), 5);
+ else callback(cause);
+ },
+ });
+ await expect(
+ writeJsonOutput({ result: "small" }, destination),
+ ).rejects.toBe(cause);
+ },
+ );
+
+ it("leaves a reusable destination open without adding error listeners", async () => {
+ const destination = new Writable({
+ write(_chunk, _encoding, callback) {
+ callback();
+ },
+ });
+ const listeners = destination.listenerCount("error");
+ await writeJsonOutput({ result: "first" }, destination);
+ await writeJsonOutput({ result: "second" }, destination);
+ expect(destination.listenerCount("error")).toBe(listeners);
+ expect(destination.writableEnded).toBe(false);
+ });
+
+ it("rejects a cycle at the serialization boundary", async () => {
+ const cycle: Record<string, unknown> = {};
+ cycle.self = cycle;
+ await expect(collect(cycle)).rejects.toThrow("circular reference");
+ });
+
+ it.each([undefined, Number.NaN, Number.POSITIVE_INFINITY, 1n, new Date(0)])(
+ "rejects non-JSON input %s",
+ async (value) => {
+ await expect(collect(value)).rejects.toThrow("ordinary JSON");
+ },
+ );
+});
+
+describe("streamed CLI result surface", () => {
+ it.each([
+ [],
+ ["--format", "md"],
+ ["--json", "--token-count"],
+ ["--json", "--token-limit", "5"],
+ ["--json", "--format", "invalid"],
+ ])("delegates unsupported output controls %j", (...arguments_) => {
+ expect(createStreamedCliJsonOutput(arguments_, new Writable())).toBe(
+ undefined,
+ );
+ });
+
+ it("keeps ownership after a failed write so no second document is emitted", async () => {
+ const cause = new Error("Fixture destination failed");
+ const destination = new Writable({
+ write(_chunk, _encoding, callback) {
+ callback(cause);
+ },
+ });
+ const output = createStreamedCliJsonOutput(["--json"], destination);
+ if (output === undefined) throw new Error("Missing JSON output surface");
+ await expect(
+ output.write(
+ { result: "small" },
+ { command: "fixture", duration: "1ms", format: "json" },
+ ),
+ ).rejects.toBe(cause);
+ expect(output.handled).toBe(true);
+ expect(output.failed).toBe(true);
+ });
+});