Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(javascript): hash graph and evidence JSON incrementally (#646)

N0zoM1z09f42495d1110
diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md
index a08862934..97d6cd41a 100644
--- a/docs/javascript-artifact-reconstruction.md
+++ b/docs/javascript-artifact-reconstruction.md
@@ -52,6 +52,23 @@ manifest and graph commitments, JavaScript Application Graph, static
Electron summary, reconstruction statistics, and explicit limitations. It does
not require a live Hopper, Ghidra, browser, or Electron process.
+## Large results
+
+Graph and Evidence identifiers hash canonical JSON incrementally, without
+assembling a single string for the whole graph. The canonical bytes and existing
+identifiers remain unchanged. The opt-in regression check is
+`npm run verify:javascript:digests`;
+it hashes a value larger than the running Node engine's single-string limit.
+
+Output formatters and MCP transport serialization still assemble whole strings.
+For a large CLI result, select the needed fields
+before formatting, for example `--format json --filter-output
+evidence_id,normalized_result.statistics`. Complete serialization of a result
+beyond the engine's string limit is not established by this digest check.
+Client framing limits also apply: the pinned Node MCP SDK's stdio transport
+defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate
+the complete response, including text and structured Evidence projections.
+
## What is reconstructed
The projector reuses the content-addressed artifact inventory and safe artifact
diff --git a/package.json b/package.json
index f9d6de563..5caf04138 100644
--- a/package.json
+++ b/package.json
@@ -128,6 +128,7 @@
"verify:electron": "npm run build:cached && node scripts/verify-real-electron.mjs",
"verify:inspector": "npm run build:cached && node scripts/verify-real-javascript-runtime-observation.mjs",
"verify:javascript-runtime": "npm run build:cached && node scripts/verify-local-javascript-runtime.mjs",
+ "verify:javascript:digests": "npm run build:cached && node scripts/verify-large-javascript-digests.mjs",
"verify:application-workflows": "npm run build:cached && node scripts/verify-local-application-workflows.mjs",
"verify:managed": "npm run build:cached && node scripts/verify-managed-conformance.mjs",
"verify:agent": "npm run build:cached && node scripts/verify-agent-experience.mjs",
diff --git a/scripts/verify-large-javascript-digests.mjs b/scripts/verify-large-javascript-digests.mjs
new file mode 100644
index 000000000..14fa5977e
--- /dev/null
+++ b/scripts/verify-large-javascript-digests.mjs
@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+import assert from "node:assert/strict";
+import { constants } from "node:buffer";
+import { createHash } from "node:crypto";
+
+import { canonicalDigest } from "../dist/domain/comparisonSemantics.js";
+import { createEvidence, parseEvidence } from "../dist/domain/evidence.js";
+
+// This opt-in check crosses the actual engine string limit; keep it outside
+// routine source tests. Build first, then run this script with the local Node.
+const leaf = "x".repeat(1024 * 1024);
+const encodedLeaf = JSON.stringify(leaf);
+const count = Math.ceil(constants.MAX_STRING_LENGTH / encodedLeaf.length) + 1;
+const value = { payload: Array.from({ length: count }, () => leaf) };
+const expected = createHash("sha256");
+expected.update('{"payload":[');
+for (let index = 0; index < count; index += 1) {
+ if (index > 0) expected.update(",");
+ expected.update(encodedLeaf);
+}
+expected.update("]}");
+assert.equal(canonicalDigest(value), expected.digest("hex"));
+
+const evidence = createEvidence(
+ undefined,
+ { id: "large-json-fixture", name: "Large JSON fixture", version: "1" },
+ { operation: "inspect_fixture", parameters: {}, result: value },
+);
+assert.equal(parseEvidence(evidence).evidence_id, evidence.evidence_id);
+process.stdout.write(
+ `${JSON.stringify({
+ verified: true,
+ serialized_value_bytes: count * encodedLeaf.length + count - 1 + 14,
+ engine_string_limit: constants.MAX_STRING_LENGTH,
+ evidence_id: evidence.evidence_id,
+ })}\n`,
+);
diff --git a/src/application/JavaScriptApplicationService.test.ts b/src/application/JavaScriptApplicationService.test.ts
new file mode 100644
index 000000000..f43d3ce19
--- /dev/null
+++ b/src/application/JavaScriptApplicationService.test.ts
@@ -0,0 +1,87 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { describe, expect, it } from "vitest";
+
+import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
+import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
+import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
+
+describe("JavaScript application failure diagnostics", () => {
+ it("retains unexpected failures and their input identity in caller-visible diagnostics", async () => {
+ const inputPath = await createTestTempDirectory("rea-js-failure-");
+ await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
+ const cause = new RangeError("Invalid string length");
+ const result = await analyzeJavaScriptApplication(
+ { input_path: inputPath, format: "directory" },
+ {
+ progress: {
+ report: async (event) => {
+ if (event.terminal) throw cause;
+ },
+ },
+ },
+ );
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected analysis failure");
+ expect(result.error.cause).toBe(cause);
+ expect(projectAnalysisError(result.error)).toMatchObject({
+ code: "execution_failure",
+ details: {
+ provider_id: "rea-javascript-application",
+ operation: "analyze_javascript_application",
+ diagnostics: {
+ input_path: inputPath,
+ error_name: "RangeError",
+ error_message: "Invalid string length",
+ },
+ },
+ });
+ });
+
+ it("keeps filesystem failures distinct from engine failures", async () => {
+ const root = await createTestTempDirectory("rea-js-missing-");
+ const result = await analyzeJavaScriptApplication({
+ input_path: join(root, "missing"),
+ format: "directory",
+ });
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected missing artifact failure");
+ expect(projectAnalysisError(result.error)).toMatchObject({
+ code: "artifact_operation_failed",
+ details: { operation: "analyze_javascript_application", reason: "io" },
+ });
+ });
+
+ it.each(["reported failure", null])(
+ "retains non-Error failures (%s)",
+ async (cause) => {
+ const inputPath = await createTestTempDirectory("rea-js-non-error-");
+ await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
+ const result = await analyzeJavaScriptApplication(
+ { input_path: inputPath, format: "directory" },
+ {
+ progress: {
+ report: async (event) => {
+ if (event.terminal) throw cause;
+ },
+ },
+ },
+ );
+ expect(result.ok).toBe(false);
+ if (result.ok) throw new Error("Expected analysis failure");
+ expect(projectAnalysisError(result.error)).toMatchObject({
+ code: "execution_failure",
+ details: {
+ diagnostics: {
+ error_name: "UnknownError",
+ error_message:
+ typeof cause === "string"
+ ? cause
+ : "JavaScript analysis failed with a non-Error value",
+ },
+ },
+ });
+ },
+ );
+});
diff --git a/src/application/JavaScriptApplicationService.ts b/src/application/JavaScriptApplicationService.ts
index 8e51a6d0f..b2389cb68 100644
--- a/src/application/JavaScriptApplicationService.ts
+++ b/src/application/JavaScriptApplicationService.ts
@@ -14,9 +14,11 @@ import { type AnalysisError } from "../domain/analysisErrorBase.js";
import type { Evidence } from "../domain/evidence.js";
import { projectInputIssues } from "../domain/inputIssueProjection.js";
import { err, ok, type Result } from "../domain/result.js";
+import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import type { ExecutionOptions } from "./AnalysisProvider.js";
import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";
import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js";
+import { JAVASCRIPT_APPLICATION_PROVIDER } from "./InvestigationProviders.js";
const OPERATION = "analyze_javascript_application" as const;
@@ -82,6 +84,42 @@ export const analyzeJavaScriptApplicationValidated = async (
{ cause },
),
);
- return err(new ArtifactOperationError(OPERATION, "io"));
+ if (
+ cause instanceof Error &&
+ "code" in cause &&
+ typeof cause.code === "string" &&
+ FILESYSTEM_ERROR_CODES.has(cause.code)
+ )
+ return err(new ArtifactOperationError(OPERATION, "io"));
+ return err(
+ new ProviderAdapterError(JAVASCRIPT_APPLICATION_PROVIDER.id, OPERATION, {
+ cause,
+ diagnostics: {
+ input_path: input.input_path,
+ error_name: cause instanceof Error ? cause.name : "UnknownError",
+ error_message:
+ cause instanceof Error
+ ? cause.message
+ : typeof cause === "string"
+ ? cause
+ : "JavaScript analysis failed with a non-Error value",
+ },
+ }),
+ );
}
};
+
+const FILESYSTEM_ERROR_CODES = new Set([
+ "ENOENT",
+ "EACCES",
+ "EPERM",
+ "ENOTDIR",
+ "EISDIR",
+ "EIO",
+ "ELOOP",
+ "ENAMETOOLONG",
+ "EMFILE",
+ "ENFILE",
+ "ENOSPC",
+ "EROFS",
+]);
diff --git a/src/domain/canonicalDigest.test.ts b/src/domain/canonicalDigest.test.ts
new file mode 100644
index 000000000..02a641905
--- /dev/null
+++ b/src/domain/canonicalDigest.test.ts
@@ -0,0 +1,83 @@
+import { createHash } from "node:crypto";
+
+import { fc, it } from "@fast-check/vitest";
+import canonicalize from "canonicalize";
+import { describe, expect } from "vitest";
+
+import { digestCanonicalValue } from "./canonicalDigest.js";
+
+const legacyDigest = (value: unknown): string => {
+ const encoded = canonicalize(value);
+ if (encoded === undefined) throw new TypeError("Unserializable fixture");
+ return createHash("sha256").update(encoded).digest("hex");
+};
+
+describe("incremental canonical digest", () => {
+ it.prop([fc.jsonValue()])(
+ "preserves canonicalize digests for JSON values",
+ (value) => {
+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
+ },
+ );
+
+ it("preserves Unicode ordering, escaping, numeric forms and non-JSON normalization", () => {
+ const shared = { key: "value" };
+ const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined];
+ sparse.length = 6;
+ const fixtures: unknown[] = [
+ {
+ "€": "\ud800",
+ "😀": "\udc00",
+ "\r": '\n\t"\\',
+ number: -0,
+ exponent: 1e30,
+ },
+ { omitted: undefined, symbol: Symbol("omit"), fn: () => undefined },
+ sparse,
+ [shared, shared],
+ new Date("2026-10-06T00:00:00Z"),
+ { toJSON: () => ({ b: 2, a: [1, null] }) },
+ { nested: { toJSON: () => undefined } },
+ ];
+ for (const value of fixtures)
+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
+ });
+
+ it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => {
+ for (const value of [undefined, Symbol("root"), () => undefined])
+ expect(() => digestCanonicalValue(value, "Fixture")).toThrow(
+ "Fixture could not canonicalize data",
+ );
+ const cyclic: Record<string, unknown> = {};
+ cyclic.self = cyclic;
+ const recursiveJson = { toJSON: (): unknown => recursiveJson };
+ for (const value of [NaN, Infinity, -Infinity, 1n, cyclic, recursiveJson])
+ expect(() => digestCanonicalValue(value)).toThrow();
+ });
+
+ it("hashes repeated large leaves with the same byte sequence", () => {
+ const value = {
+ payload: Array.from({ length: 64 }, () => "x".repeat(16_384)),
+ };
+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
+ });
+
+ it("preserves canonicalize property access and toJSON cycle behavior", () => {
+ const createAccessor = () => {
+ let reads = 0;
+ return {
+ get value() {
+ reads += 1;
+ return reads;
+ },
+ };
+ };
+ expect(digestCanonicalValue(createAccessor())).toBe(
+ legacyDigest(createAccessor()),
+ );
+ const shared = { toJSON: () => ({ answer: 42 }) };
+ expect(digestCanonicalValue([shared, shared])).toBe(
+ legacyDigest([shared, shared]),
+ );
+ });
+});
diff --git a/src/domain/canonicalDigest.ts b/src/domain/canonicalDigest.ts
new file mode 100644
index 000000000..c9f1c6260
--- /dev/null
+++ b/src/domain/canonicalDigest.ts
@@ -0,0 +1,75 @@
+import { createHash } from "node:crypto";
+
+/** Hash canonicalize-compatible JSON without assembling an aggregate string. */
+export const digestCanonicalValue = (
+ value: unknown,
+ context = "Comparison",
+): string => {
+ const hash = createHash("sha256");
+ const encoded = emitCanonical(value, (part) => hash.update(part), new Set());
+ if (!encoded) throw new TypeError(`${context} could not canonicalize data`);
+ return hash.digest("hex");
+};
+
+type Emit = (part: string) => void;
+
+const emitCanonical = (
+ value: unknown,
+ emit: Emit,
+ ancestors: Set<object>,
+): boolean => {
+ if (typeof value === "number" && !Number.isFinite(value))
+ throw new Error(
+ Number.isNaN(value) ? "NaN is not allowed" : "Infinity is not allowed",
+ );
+ if (value === null || typeof value !== "object") {
+ const encoded = JSON.stringify(value);
+ if (encoded === undefined) return false;
+ emit(encoded);
+ return true;
+ }
+ if (ancestors.has(value)) throw new Error("Circular reference detected");
+ ancestors.add(value);
+ try {
+ if ("toJSON" in value && typeof value.toJSON === "function") {
+ const normalized: unknown = value.toJSON();
+ return emitCanonical(normalized, emit, ancestors);
+ }
+ if (Array.isArray(value)) {
+ emit("[");
+ const length = value.length;
+ for (let index = 0; index < length; index += 1) {
+ if (index > 0) emit(",");
+ // Preserve canonicalize's sparse-array and unsupported-value behavior.
+ if (!(index in value)) continue;
+ const item: unknown = value[index];
+ emitCanonical(
+ item === undefined || typeof item === "symbol" ? null : item,
+ emit,
+ ancestors,
+ );
+ }
+ emit("]");
+ } else {
+ emit("{");
+ let first = true;
+ for (const key of Object.keys(value).sort()) {
+ if (
+ Reflect.get(value, key) === undefined ||
+ typeof Reflect.get(value, key) === "symbol"
+ )
+ continue;
+ const item: unknown = Reflect.get(value, key);
+ if (!first) emit(",");
+ first = false;
+ emit(JSON.stringify(key));
+ emit(":");
+ if (!emitCanonical(item, emit, ancestors)) emit("undefined");
+ }
+ emit("}");
+ }
+ return true;
+ } finally {
+ ancestors.delete(value);
+ }
+};
diff --git a/src/domain/comparisonSemantics.ts b/src/domain/comparisonSemantics.ts
index 804a446ba..9c005d124 100644
--- a/src/domain/comparisonSemantics.ts
+++ b/src/domain/comparisonSemantics.ts
@@ -1,7 +1,7 @@
-import { createHash } from "node:crypto";
-
import canonicalize from "canonicalize";
+import { digestCanonicalValue } from "./canonicalDigest.js";
+
/** Whether an inventory can support a claim that an item is absent. */
export const absenceClaimable = (coverage: {
readonly status: string;
@@ -17,9 +17,7 @@ export const canonicalDigest = (
value: unknown,
context = "Comparison",
): string => {
- return createHash("sha256")
- .update(canonicalJson(value, context))
- .digest("hex");
+ return digestCanonicalValue(value, context);
};
/** Canonical JSON used for deterministic tie breaking and stable digests. */
diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts
index c7ee5b885..c4ebc7187 100644
--- a/src/domain/evidence.ts
+++ b/src/domain/evidence.ts
@@ -1,8 +1,7 @@
-import { createHash } from "node:crypto";
-
-import canonicalize from "canonicalize";
import { z } from "zod";
+import { digestCanonicalValue } from "./canonicalDigest.js";
+
import {
analysisProfileSchema,
type AnalysisProfileCommitment,
@@ -173,22 +172,11 @@ export interface EvidenceObservation {
readonly evidenceLinks?: readonly string[];
}
-const sha256 = (value: string): string =>
- createHash("sha256").update(value).digest("hex");
-
type WithoutEvidenceId<Record_> = Record_ extends unknown
? Omit<Record_, "evidence_id">
: never;
type EvidenceWithoutId = WithoutEvidenceId<Evidence>;
-/** Serialize JSON according to RFC 8785 JSON Canonicalization Scheme. */
-const canonicalJson = (value: JsonValue): string => {
- const serialized = canonicalize(value);
- if (serialized === undefined)
- throw new TypeError("RFC 8785 canonicalization rejected a JSON value");
- return serialized;
-};
-
const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
subject:
evidence.subject === null
@@ -217,7 +205,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
/** Recompute the semantic identifier, excluding paths and raw payload bytes. */
const computeEvidenceId = (evidence: EvidenceWithoutId): string =>
- `ev_${sha256(canonicalJson(semanticProjection(evidence)))}`;
+ `ev_${digestCanonicalValue(semanticProjection(evidence), "Evidence")}`;
/** Parse evidence and reject a syntactically valid but tampered semantic ID. */
export const parseEvidence = (input: unknown): Evidence => {