Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(json): preserve runtime limits across JSON file readers (#1350)

Ysopwnable9c4a68bc19a0
diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts
index 487679070..4c7e310c7 100644
--- a/src/application/EvidenceBundleFiles.ts
+++ b/src/application/EvidenceBundleFiles.ts
@@ -9,7 +9,10 @@ import {
EvidenceIntegrityError,
} from "../domain/evidenceErrors.js";
import { err, ok, type Result } from "../domain/result.js";
-import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
+import {
+ AnalysisCancelledError,
+ type AnalysisResourceConstraintError,
+} from "../domain/analysisErrorCore.js";
import { parseProcessCapture } from "../domain/process/processCaptureParsing.js";
import {
bufferedJsonParts,
@@ -17,7 +20,10 @@ import {
} from "../domain/jsonSerialization.js";
import { readJsonFile, writeTextParts } from "./JsonFiles.js";
-type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError;
+type EvidenceReadFailure =
+ | EvidenceFileError
+ | EvidenceIntegrityError
+ | AnalysisResourceConstraintError;
type EvidenceWriteFailure =
| EvidenceFileError
| EvidenceIntegrityError
diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts
index cdf80d7e8..8132255da 100644
--- a/src/application/JsonFiles.ts
+++ b/src/application/JsonFiles.ts
@@ -12,8 +12,12 @@ import {
import { basename, dirname, resolve } from "node:path";
import { EvidenceFileError } from "../domain/evidenceErrors.js";
-import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
+import {
+ AnalysisCancelledError,
+ AnalysisResourceConstraintError,
+} from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
+import { parseUtf8Json } from "./Utf8JsonInput.js";
/** Request control and its owning operation for an interruptible atomic write. */
export interface TextWriteCancellation {
@@ -27,7 +31,9 @@ type TextWriteResult<Failure = EvidenceFileError | AnalysisCancelledError> =
/** Read JSON data from a regular file at the caller-supplied path. */
export const readJsonFile = async (
path: string,
-): Promise<Result<unknown, EvidenceFileError>> => {
+): Promise<
+ Result<unknown, EvidenceFileError | AnalysisResourceConstraintError>
+> => {
const requestedPath = resolve(path);
try {
const canonicalPath = await realpath(requestedPath);
@@ -37,23 +43,18 @@ export const readJsonFile = async (
new EvidenceFileError("read", "not-file", { path: requestedPath }),
);
const encoded = await readFile(canonicalPath);
- let decoded: unknown;
- try {
- decoded = JSON.parse(
- new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(
- encoded,
- ),
- );
- } catch (cause: unknown) {
+ const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath);
+ if (!decoded.ok) {
return err(
new EvidenceFileError("read", "invalid-json", {
- cause,
+ cause: decoded.cause,
path: requestedPath,
}),
);
}
- return ok(decoded);
+ return ok(decoded.value);
} catch (cause: unknown) {
+ if (cause instanceof AnalysisResourceConstraintError) return err(cause);
return err(
new EvidenceFileError("read", missingOrIo(cause), {
cause,
diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts
new file mode 100644
index 000000000..4b9ecf10b
--- /dev/null
+++ b/src/application/Utf8JsonInput.ts
@@ -0,0 +1,78 @@
+import { constants } from "node:buffer";
+
+import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js";
+import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";
+
+/** Decode JSON bytes without confusing a runtime string limit with malformed input. */
+export const parseUtf8Json = (
+ bytes: Uint8Array,
+ operation: string,
+ path: string,
+ boundary = "json-input",
+): SafeJsonParseResult => {
+ const stringLimitFailure = (
+ cause?: unknown,
+ decodedUnits?: number,
+ ): AnalysisResourceConstraintError =>
+ new AnalysisResourceConstraintError(
+ operation,
+ "memory",
+ "JSON input exceeds the runtime's maximum decoded string length",
+ {
+ boundary,
+ input_path: path,
+ input_bytes: bytes.byteLength,
+ max_string_code_units: constants.MAX_STRING_LENGTH,
+ ...(decodedUnits === undefined
+ ? {}
+ : { decoded_string_code_units_at_least: decodedUnits }),
+ },
+ {
+ ...(cause === undefined ? {} : { cause }),
+ remediationAction:
+ "Provide a smaller JSON document. For Evidence-based workflows, re-analyze a smaller selection of the original target and use its Evidence; splitting JSON text alone does not produce a valid workflow input.",
+ },
+ );
+ let text: string;
+ try {
+ const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+ if (bytes.byteLength <= constants.MAX_STRING_LENGTH) {
+ text = decoder.decode(bytes);
+ } else {
+ // Node 22's UTF-8 fast path bounds encoded bytes before decoding. Decode
+ // large inputs in chunks so valid multibyte text is bounded by its actual
+ // UTF-16 length. Streaming also preserves characters spanning chunks.
+ const parts: string[] = [];
+ let decodedUnits = 0;
+ const append = (part: string): void => {
+ decodedUnits += part.length;
+ if (decodedUnits > constants.MAX_STRING_LENGTH)
+ throw stringLimitFailure(undefined, decodedUnits);
+ parts.push(part);
+ };
+ const chunkBytes = 1024 * 1024;
+ for (let offset = 0; offset < bytes.byteLength; offset += chunkBytes)
+ append(
+ decoder.decode(bytes.subarray(offset, offset + chunkBytes), {
+ stream: true,
+ }),
+ );
+ append(decoder.decode());
+ text = parts.join("");
+ }
+ } catch (cause: unknown) {
+ if (cause instanceof AnalysisResourceConstraintError) throw cause;
+ const code =
+ typeof cause === "object" && cause !== null && "code" in cause
+ ? cause.code
+ : undefined;
+ if (code === "ERR_STRING_TOO_LONG") throw stringLimitFailure(cause);
+ if (code !== "ERR_ENCODING_INVALID_ENCODED_DATA") throw cause;
+ return {
+ ok: false,
+ error: "JSON input is not valid UTF-8",
+ cause,
+ };
+ }
+ return safeParseJson(text);
+};
diff --git a/src/application/binary/AnalysisSnapshotFiles.ts b/src/application/binary/AnalysisSnapshotFiles.ts
index 2841e1dbf..87d910b5d 100644
--- a/src/application/binary/AnalysisSnapshotFiles.ts
+++ b/src/application/binary/AnalysisSnapshotFiles.ts
@@ -8,6 +8,7 @@ import {
EvidenceIntegrityError,
} from "../../domain/evidenceErrors.js";
import { err, ok, type Result } from "../../domain/result.js";
+import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
import { readJsonFile, writeTextFile } from "../JsonFiles.js";
type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;
@@ -15,7 +16,9 @@ type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;
/** Read and validate an analysis snapshot from the caller's path. */
export const readAnalysisSnapshot = async (
path: string,
-): Promise<Result<AnalysisSnapshot, SnapshotFailure>> => {
+): Promise<
+ Result<AnalysisSnapshot, SnapshotFailure | AnalysisResourceConstraintError>
+> => {
const loaded = await readJsonFile(path);
if (!loaded.ok) return loaded;
try {
diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts
index 74dd30bc9..a47d1e5f5 100644
--- a/src/cliJsonInput.ts
+++ b/src/cliJsonInput.ts
@@ -5,6 +5,7 @@ import {
NonRegularFileReadError,
readRegularFile,
} from "./application/RegularFileRead.js";
+import { parseUtf8Json } from "./application/Utf8JsonInput.js";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
@@ -21,18 +22,33 @@ export const parseCliJsonInput = async (
| { readonly ok: true; readonly value: unknown }
| { readonly ok: false; readonly error: JsonValue }
> => {
- const inline = parseJson(value);
- if (inline.kind === "valid") return { ok: true, value: inline.value };
- if (inline.kind === "too-large")
+ const inline = safeParseJson(value);
+ if (inline.ok) return { ok: true, value: inline.value };
+ if (isStringLengthLimit(inline.cause))
return jsonTooLargeError(undefined, operation);
try {
- // Read raw bytes so invalid UTF-8 is rejected by parseJson instead of
+ // Read raw bytes so invalid UTF-8 is rejected instead of
// being silently replaced by lossy "utf8" decoding.
- const parsed = parseJson(await readRegularFile(value));
- if (parsed.kind === "valid") return { ok: true, value: parsed.value };
- if (parsed.kind === "too-large") return jsonTooLargeError(value, operation);
- return jsonFileError(value, operation, "invalid-json");
+ const parsed = parseUtf8Json(
+ await readRegularFile(value),
+ operation,
+ value,
+ "cli-json-input",
+ );
+ return parsed.ok
+ ? { ok: true, value: parsed.value }
+ : jsonFileError(value, operation, "invalid-json");
} catch (cause: unknown) {
+ if (cause instanceof AnalysisResourceConstraintError)
+ return {
+ ok: false,
+ error: {
+ error: "Application workflow failed",
+ ...projectAnalysisError(cause),
+ input_path: value,
+ input_reason: "too-large",
+ },
+ };
if (
["{", "["].includes(value.trimStart()[0] ?? "") &&
cannotBeAnExistingFile(cause) &&
@@ -93,39 +109,12 @@ const cannotBeAnExistingFile = (cause: unknown): boolean =>
const hasExplicitJsonFileExtension = (value: string): boolean =>
value.toLowerCase().endsWith(".json");
-type JsonParseResult =
- | { readonly kind: "valid"; readonly value: unknown }
- | { readonly kind: "invalid" }
- | { readonly kind: "too-large" };
-
-const parseJson = (value: string | Uint8Array): JsonParseResult => {
- let text: string;
- if (typeof value === "string") {
- text = value;
- } else {
- try {
- text = new TextDecoder("utf-8", {
- fatal: true,
- ignoreBOM: true,
- }).decode(value);
- } catch (cause: unknown) {
- return isStringLengthLimit(cause)
- ? { kind: "too-large" }
- : { kind: "invalid" };
- }
- }
- const parsed = safeParseJson(text);
- if (parsed.ok) return { kind: "valid", value: parsed.value };
- return isStringLengthLimit(parsed.cause)
- ? { kind: "too-large" }
- : { kind: "invalid" };
-};
-
const isStringLengthLimit = (cause: unknown): boolean =>
cause instanceof Error &&
- /cannot create a string longer than|invalid string length/i.test(
- cause.message,
- );
+ (("code" in cause && cause.code === "ERR_STRING_TOO_LONG") ||
+ /cannot create a string longer than|invalid string length/i.test(
+ cause.message,
+ ));
const inputError = (operation: string): JsonValue => ({
error: "Application workflow failed",
@@ -181,29 +170,26 @@ const readFailureIssue = (path: string | undefined, cause: unknown) => {
const jsonTooLargeError = (
path: string | undefined,
operation: string,
-): { readonly ok: false; readonly error: JsonValue } => {
- const projection = projectAnalysisError(
- new AnalysisResourceConstraintError(
- operation,
- "memory",
- `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`,
- {
- boundary: "cli-json-input",
- max_string_code_units: bufferConstants.MAX_STRING_LENGTH,
- },
- {
- remediationAction:
- "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.",
- },
+): { readonly ok: false; readonly error: JsonValue } => ({
+ ok: false,
+ error: {
+ error: "Application workflow failed",
+ ...projectAnalysisError(
+ new AnalysisResourceConstraintError(
+ operation,
+ "memory",
+ `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`,
+ {
+ boundary: "cli-json-input",
+ max_string_code_units: bufferConstants.MAX_STRING_LENGTH,
+ },
+ {
+ remediationAction:
+ "Provide a smaller JSON document or rerun its producer on a smaller selection. Splitting JSON text alone does not produce a valid workflow input.",
+ },
+ ),
),
- );
- return {
- ok: false,
- error: {
- error: "Application workflow failed",
- ...projection,
- ...(path === undefined ? {} : { input_path: path }),
- input_reason: "too-large",
- },
- };
-};
+ ...(path === undefined ? {} : { input_path: path }),
+ input_reason: "too-large",
+ },
+});
diff --git a/src/cliJsonInputStatus.test.ts b/src/cliJsonInputStatus.test.ts
index 8f9238e00..f4bf8854d 100644
--- a/src/cliJsonInputStatus.test.ts
+++ b/src/cliJsonInputStatus.test.ts
@@ -18,6 +18,7 @@ describe("CLI JSON input failure recognition", () => {
{ input_reason: "invalid-json" },
{ input_path: "input.json", input_reason: "invalid-json" },
{ input_path: "missing.json", input_reason: "read-failed" },
+ { input_path: "large.json", input_reason: "too-large" },
{ input_path: "", input_reason: "read-failed" },
])("recognizes a complete envelope with metadata %j", (metadata) => {
const value = { ...envelope, ...metadata };
diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.ts
new file mode 100644
index 000000000..fce9412f2
--- /dev/null
+++ b/tests/boundary/cli/jsonInputSize.test.ts
@@ -0,0 +1,69 @@
+import { constants } from "node:buffer";
+import { open } from "node:fs/promises";
+import { join } from "node:path";
+
+import { expect } from "vitest";
+
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+cliTest(
+ "preserves the runtime string limit through CLI and Evidence file inputs",
+ async ({ cli, processes }) => {
+ const root = await createTestTempDirectory("rea-json-string-limit-");
+ const input = join(root, "oversized.json");
+ const file = await open(input, "wx");
+ try {
+ // Sparse zero bytes are valid UTF-8. Decoded length exceeds the runtime
+ // string limit before JSON syntax can be inspected; no fixture is retained.
+ await file.truncate(constants.MAX_STRING_LENGTH + 1);
+ } finally {
+ await file.close();
+ }
+ const limits = {
+ input_path: input,
+ input_bytes: constants.MAX_STRING_LENGTH + 1,
+ max_string_code_units: constants.MAX_STRING_LENGTH,
+ };
+ // Run readers sequentially in owned child processes, releasing each large
+ // read buffer when its process exits instead of retaining it in Vitest.
+ const result = await cli.run({
+ arguments: ["trace-application-feature", input, "--json"],
+ });
+ expect(result.json).toMatchObject({
+ code: "resource_constraint",
+ category: "resource_constraint",
+ input_path: input,
+ input_reason: "too-large",
+ retryable: false,
+ remediation: { action: expect.stringContaining("re-analyze") },
+ details: {
+ operation: "trace-application-feature",
+ resource: "memory",
+ reported_limits: limits,
+ },
+ });
+ expect(result.exitCode).toBe(1);
+ expect(result.stderr).toBe("");
+
+ const evidence = await processes.run(process.execPath, [
+ "--input-type=module",
+ "-e",
+ `import { readJsonFile } from "./dist/application/JsonFiles.js";
+ import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js";
+ const result = await readJsonFile(process.argv[1]);
+ console.log(JSON.stringify(result.ok ? result : projectAnalysisError(result.error)));`,
+ input,
+ ]);
+ expect(JSON.parse(evidence.stdout)).toMatchObject({
+ code: "resource_constraint",
+ details: {
+ operation: "read_evidence_file",
+ resource: "memory",
+ reported_limits: limits,
+ },
+ });
+ expect(evidence.stderr).toBe("");
+ expect(evidence.exitCode).toBe(0);
+ },
+);
diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts
index 273168fdb..e550f88b8 100644
--- a/tests/boundary/filesystem/cliJsonInput.test.ts
+++ b/tests/boundary/filesystem/cliJsonInput.test.ts
@@ -1,6 +1,7 @@
import { execFile } from "node:child_process";
import { constants as bufferConstants } from "node:buffer";
import {
+ appendFile,
chmod,
open,
symlink,
@@ -119,8 +120,7 @@ describe("CLI JSON input", () => {
},
},
remediation: {
- action:
- "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.",
+ action: expect.stringContaining("re-analyze a smaller selection"),
},
},
});
@@ -137,6 +137,13 @@ describe("CLI JSON input", () => {
if (typeof multibyteResult.value !== "string")
throw new Error("Expected parsed JSON string");
expect(multibyteResult.value.length).toBe(characterCount);
+
+ // Large-input streaming must flush and reject an incomplete UTF-8 suffix.
+ await appendFile(path, Buffer.from([0xc3]));
+ expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
+ ok: false,
+ error: { input_path: path, input_reason: "invalid-json" },
+ });
}, 20_000);
it("distinguishes malformed inline text from files and preserves bracket-prefixed paths", async () => {