Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(json): preserve runtime limits across JSON file readers (#1350)
Ysopwnable9c4a68bc19a0
diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.tsindex 487679070..4c7e310c7 100644--- a/src/application/EvidenceBundleFiles.ts+++ b/src/application/EvidenceBundleFiles.ts@@ -9,7 +9,10 @@ import {EvidenceIntegrityError,} from "../domain/evidenceErrors.js";import { err, ok, type Result } from "../domain/result.js";-import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";+import {+ AnalysisCancelledError,+ type AnalysisResourceConstraintError,+} from "../domain/analysisErrorCore.js";import { parseProcessCapture } from "../domain/process/processCaptureParsing.js";import {bufferedJsonParts,@@ -17,7 +20,10 @@ import {} from "../domain/jsonSerialization.js";import { readJsonFile, writeTextParts } from "./JsonFiles.js";-type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError;+type EvidenceReadFailure =+ | EvidenceFileError+ | EvidenceIntegrityError+ | AnalysisResourceConstraintError;type EvidenceWriteFailure =| EvidenceFileError| EvidenceIntegrityErrordiff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.tsindex cdf80d7e8..8132255da 100644--- a/src/application/JsonFiles.ts+++ b/src/application/JsonFiles.ts@@ -12,8 +12,12 @@ import {import { basename, dirname, resolve } from "node:path";import { EvidenceFileError } from "../domain/evidenceErrors.js";-import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";+import {+ AnalysisCancelledError,+ AnalysisResourceConstraintError,+} from "../domain/analysisErrorCore.js";import { err, ok, type Result } from "../domain/result.js";+import { parseUtf8Json } from "./Utf8JsonInput.js";/** Request control and its owning operation for an interruptible atomic write. */export interface TextWriteCancellation {@@ -27,7 +31,9 @@ type TextWriteResult<Failure = EvidenceFileError | AnalysisCancelledError> =/** Read JSON data from a regular file at the caller-supplied path. */export const readJsonFile = async (path: string,-): Promise<Result<unknown, EvidenceFileError>> => {+): Promise<+ Result<unknown, EvidenceFileError | AnalysisResourceConstraintError>+> => {const requestedPath = resolve(path);try {const canonicalPath = await realpath(requestedPath);@@ -37,23 +43,18 @@ export const readJsonFile = async (new EvidenceFileError("read", "not-file", { path: requestedPath }),);const encoded = await readFile(canonicalPath);- let decoded: unknown;- try {- decoded = JSON.parse(- new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(- encoded,- ),- );- } catch (cause: unknown) {+ const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath);+ if (!decoded.ok) {return err(new EvidenceFileError("read", "invalid-json", {- cause,+ cause: decoded.cause,path: requestedPath,}),);}- return ok(decoded);+ return ok(decoded.value);} catch (cause: unknown) {+ if (cause instanceof AnalysisResourceConstraintError) return err(cause);return err(new EvidenceFileError("read", missingOrIo(cause), {cause,diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.tsnew file mode 100644index 000000000..4b9ecf10b--- /dev/null+++ b/src/application/Utf8JsonInput.ts@@ -0,0 +1,78 @@+import { constants } from "node:buffer";++import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";+import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";++/** Decode JSON bytes without confusing a runtime string limit with malformed input. */+export const parseUtf8Json = (+ bytes: Uint8Array,+ operation: string,+ path: string,+ boundary = "json-input",+): SafeJsonParseResult => {+ const stringLimitFailure = (+ cause?: unknown,+ decodedUnits?: number,+ ): AnalysisResourceConstraintError =>+ new AnalysisResourceConstraintError(+ operation,+ "memory",+ "JSON input exceeds the runtime's maximum decoded string length",+ {+ boundary,+ input_path: path,+ input_bytes: bytes.byteLength,+ max_string_code_units: constants.MAX_STRING_LENGTH,+ ...(decodedUnits === undefined+ ? {}+ : { decoded_string_code_units_at_least: decodedUnits }),+ },+ {+ ...(cause === undefined ? {} : { cause }),+ remediationAction:+ "Provide a smaller JSON document. For Evidence-based workflows, re-analyze a smaller selection of the original target and use its Evidence; splitting JSON text alone does not produce a valid workflow input.",+ },+ );+ let text: string;+ try {+ const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });+ if (bytes.byteLength <= constants.MAX_STRING_LENGTH) {+ text = decoder.decode(bytes);+ } else {+ // Node 22's UTF-8 fast path bounds encoded bytes before decoding. Decode+ // large inputs in chunks so valid multibyte text is bounded by its actual+ // UTF-16 length. Streaming also preserves characters spanning chunks.+ const parts: string[] = [];+ let decodedUnits = 0;+ const append = (part: string): void => {+ decodedUnits += part.length;+ if (decodedUnits > constants.MAX_STRING_LENGTH)+ throw stringLimitFailure(undefined, decodedUnits);+ parts.push(part);+ };+ const chunkBytes = 1024 * 1024;+ for (let offset = 0; offset < bytes.byteLength; offset += chunkBytes)+ append(+ decoder.decode(bytes.subarray(offset, offset + chunkBytes), {+ stream: true,+ }),+ );+ append(decoder.decode());+ text = parts.join("");+ }+ } catch (cause: unknown) {+ if (cause instanceof AnalysisResourceConstraintError) throw cause;+ const code =+ typeof cause === "object" && cause !== null && "code" in cause+ ? cause.code+ : undefined;+ if (code === "ERR_STRING_TOO_LONG") throw stringLimitFailure(cause);+ if (code !== "ERR_ENCODING_INVALID_ENCODED_DATA") throw cause;+ return {+ ok: false,+ error: "JSON input is not valid UTF-8",+ cause,+ };+ }+ return safeParseJson(text);+};diff --git a/src/application/binary/AnalysisSnapshotFiles.ts b/src/application/binary/AnalysisSnapshotFiles.tsindex 2841e1dbf..87d910b5d 100644--- a/src/application/binary/AnalysisSnapshotFiles.ts+++ b/src/application/binary/AnalysisSnapshotFiles.ts@@ -8,6 +8,7 @@ import {EvidenceIntegrityError,} from "../../domain/evidenceErrors.js";import { err, ok, type Result } from "../../domain/result.js";+import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";import { readJsonFile, writeTextFile } from "../JsonFiles.js";type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;@@ -15,7 +16,9 @@ type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;/** Read and validate an analysis snapshot from the caller's path. */export const readAnalysisSnapshot = async (path: string,-): Promise<Result<AnalysisSnapshot, SnapshotFailure>> => {+): Promise<+ Result<AnalysisSnapshot, SnapshotFailure | AnalysisResourceConstraintError>+> => {const loaded = await readJsonFile(path);if (!loaded.ok) return loaded;try {diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.tsindex 74dd30bc9..a47d1e5f5 100644--- a/src/cliJsonInput.ts+++ b/src/cliJsonInput.ts@@ -5,6 +5,7 @@ import {NonRegularFileReadError,readRegularFile,} from "./application/RegularFileRead.js";+import { parseUtf8Json } from "./application/Utf8JsonInput.js";import {AnalysisInputError,AnalysisResourceConstraintError,@@ -21,18 +22,33 @@ export const parseCliJsonInput = async (| { readonly ok: true; readonly value: unknown }| { readonly ok: false; readonly error: JsonValue }> => {- const inline = parseJson(value);- if (inline.kind === "valid") return { ok: true, value: inline.value };- if (inline.kind === "too-large")+ const inline = safeParseJson(value);+ if (inline.ok) return { ok: true, value: inline.value };+ if (isStringLengthLimit(inline.cause))return jsonTooLargeError(undefined, operation);try {- // Read raw bytes so invalid UTF-8 is rejected by parseJson instead of+ // Read raw bytes so invalid UTF-8 is rejected instead of// being silently replaced by lossy "utf8" decoding.- const parsed = parseJson(await readRegularFile(value));- if (parsed.kind === "valid") return { ok: true, value: parsed.value };- if (parsed.kind === "too-large") return jsonTooLargeError(value, operation);- return jsonFileError(value, operation, "invalid-json");+ const parsed = parseUtf8Json(+ await readRegularFile(value),+ operation,+ value,+ "cli-json-input",+ );+ return parsed.ok+ ? { ok: true, value: parsed.value }+ : jsonFileError(value, operation, "invalid-json");} catch (cause: unknown) {+ if (cause instanceof AnalysisResourceConstraintError)+ return {+ ok: false,+ error: {+ error: "Application workflow failed",+ ...projectAnalysisError(cause),+ input_path: value,+ input_reason: "too-large",+ },+ };if (["{", "["].includes(value.trimStart()[0] ?? "") &&cannotBeAnExistingFile(cause) &&@@ -93,39 +109,12 @@ const cannotBeAnExistingFile = (cause: unknown): boolean =>const hasExplicitJsonFileExtension = (value: string): boolean =>value.toLowerCase().endsWith(".json");-type JsonParseResult =- | { readonly kind: "valid"; readonly value: unknown }- | { readonly kind: "invalid" }- | { readonly kind: "too-large" };--const parseJson = (value: string | Uint8Array): JsonParseResult => {- let text: string;- if (typeof value === "string") {- text = value;- } else {- try {- text = new TextDecoder("utf-8", {- fatal: true,- ignoreBOM: true,- }).decode(value);- } catch (cause: unknown) {- return isStringLengthLimit(cause)- ? { kind: "too-large" }- : { kind: "invalid" };- }- }- const parsed = safeParseJson(text);- if (parsed.ok) return { kind: "valid", value: parsed.value };- return isStringLengthLimit(parsed.cause)- ? { kind: "too-large" }- : { kind: "invalid" };-};-const isStringLengthLimit = (cause: unknown): boolean =>cause instanceof Error &&- /cannot create a string longer than|invalid string length/i.test(- cause.message,- );+ (("code" in cause && cause.code === "ERR_STRING_TOO_LONG") ||+ /cannot create a string longer than|invalid string length/i.test(+ cause.message,+ ));const inputError = (operation: string): JsonValue => ({error: "Application workflow failed",@@ -181,29 +170,26 @@ const readFailureIssue = (path: string | undefined, cause: unknown) => {const jsonTooLargeError = (path: string | undefined,operation: string,-): { readonly ok: false; readonly error: JsonValue } => {- const projection = projectAnalysisError(- new AnalysisResourceConstraintError(- operation,- "memory",- `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`,- {- boundary: "cli-json-input",- max_string_code_units: bufferConstants.MAX_STRING_LENGTH,- },- {- remediationAction:- "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.",- },+): { readonly ok: false; readonly error: JsonValue } => ({+ ok: false,+ error: {+ error: "Application workflow failed",+ ...projectAnalysisError(+ new AnalysisResourceConstraintError(+ operation,+ "memory",+ `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`,+ {+ boundary: "cli-json-input",+ max_string_code_units: bufferConstants.MAX_STRING_LENGTH,+ },+ {+ remediationAction:+ "Provide a smaller JSON document or rerun its producer on a smaller selection. Splitting JSON text alone does not produce a valid workflow input.",+ },+ ),),- );- return {- ok: false,- error: {- error: "Application workflow failed",- ...projection,- ...(path === undefined ? {} : { input_path: path }),- input_reason: "too-large",- },- };-};+ ...(path === undefined ? {} : { input_path: path }),+ input_reason: "too-large",+ },+});diff --git a/src/cliJsonInputStatus.test.ts b/src/cliJsonInputStatus.test.tsindex 8f9238e00..f4bf8854d 100644--- a/src/cliJsonInputStatus.test.ts+++ b/src/cliJsonInputStatus.test.ts@@ -18,6 +18,7 @@ describe("CLI JSON input failure recognition", () => {{ input_reason: "invalid-json" },{ input_path: "input.json", input_reason: "invalid-json" },{ input_path: "missing.json", input_reason: "read-failed" },+ { input_path: "large.json", input_reason: "too-large" },{ input_path: "", input_reason: "read-failed" },])("recognizes a complete envelope with metadata %j", (metadata) => {const value = { ...envelope, ...metadata };diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.tsnew file mode 100644index 000000000..fce9412f2--- /dev/null+++ b/tests/boundary/cli/jsonInputSize.test.ts@@ -0,0 +1,69 @@+import { constants } from "node:buffer";+import { open } from "node:fs/promises";+import { join } from "node:path";++import { expect } from "vitest";++import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";+import { cliTest } from "../../support/cli/cliFixture.js";++cliTest(+ "preserves the runtime string limit through CLI and Evidence file inputs",+ async ({ cli, processes }) => {+ const root = await createTestTempDirectory("rea-json-string-limit-");+ const input = join(root, "oversized.json");+ const file = await open(input, "wx");+ try {+ // Sparse zero bytes are valid UTF-8. Decoded length exceeds the runtime+ // string limit before JSON syntax can be inspected; no fixture is retained.+ await file.truncate(constants.MAX_STRING_LENGTH + 1);+ } finally {+ await file.close();+ }+ const limits = {+ input_path: input,+ input_bytes: constants.MAX_STRING_LENGTH + 1,+ max_string_code_units: constants.MAX_STRING_LENGTH,+ };+ // Run readers sequentially in owned child processes, releasing each large+ // read buffer when its process exits instead of retaining it in Vitest.+ const result = await cli.run({+ arguments: ["trace-application-feature", input, "--json"],+ });+ expect(result.json).toMatchObject({+ code: "resource_constraint",+ category: "resource_constraint",+ input_path: input,+ input_reason: "too-large",+ retryable: false,+ remediation: { action: expect.stringContaining("re-analyze") },+ details: {+ operation: "trace-application-feature",+ resource: "memory",+ reported_limits: limits,+ },+ });+ expect(result.exitCode).toBe(1);+ expect(result.stderr).toBe("");++ const evidence = await processes.run(process.execPath, [+ "--input-type=module",+ "-e",+ `import { readJsonFile } from "./dist/application/JsonFiles.js";+ import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js";+ const result = await readJsonFile(process.argv[1]);+ console.log(JSON.stringify(result.ok ? result : projectAnalysisError(result.error)));`,+ input,+ ]);+ expect(JSON.parse(evidence.stdout)).toMatchObject({+ code: "resource_constraint",+ details: {+ operation: "read_evidence_file",+ resource: "memory",+ reported_limits: limits,+ },+ });+ expect(evidence.stderr).toBe("");+ expect(evidence.exitCode).toBe(0);+ },+);diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.tsindex 273168fdb..e550f88b8 100644--- a/tests/boundary/filesystem/cliJsonInput.test.ts+++ b/tests/boundary/filesystem/cliJsonInput.test.ts@@ -1,6 +1,7 @@import { execFile } from "node:child_process";import { constants as bufferConstants } from "node:buffer";import {+ appendFile,chmod,open,symlink,@@ -119,8 +120,7 @@ describe("CLI JSON input", () => {},},remediation: {- action:- "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.",+ action: expect.stringContaining("re-analyze a smaller selection"),},},});@@ -137,6 +137,13 @@ describe("CLI JSON input", () => {if (typeof multibyteResult.value !== "string")throw new Error("Expected parsed JSON string");expect(multibyteResult.value.length).toBe(characterCount);++ // Large-input streaming must flush and reject an incomplete UTF-8 suffix.+ await appendFile(path, Buffer.from([0xc3]));+ expect(await parseCliJsonInput(path, "test-input")).toMatchObject({+ ok: false,+ error: { input_path: path, input_reason: "invalid-json" },+ });}, 20_000);it("distinguishes malformed inline text from files and preserves bracket-prefixed paths", async () => {