Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(javascript): hash graph and evidence JSON incrementally (#646)
N0zoM1z09f42495d1110
diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.mdindex a08862934..97d6cd41a 100644--- a/docs/javascript-artifact-reconstruction.md+++ b/docs/javascript-artifact-reconstruction.md@@ -52,6 +52,23 @@ manifest and graph commitments, JavaScript Application Graph, staticElectron summary, reconstruction statistics, and explicit limitations. It doesnot require a live Hopper, Ghidra, browser, or Electron process.+## Large results++Graph and Evidence identifiers hash canonical JSON incrementally, without+assembling a single string for the whole graph. The canonical bytes and existing+identifiers remain unchanged. The opt-in regression check is+`npm run verify:javascript:digests`;+it hashes a value larger than the running Node engine's single-string limit.++Output formatters and MCP transport serialization still assemble whole strings.+For a large CLI result, select the needed fields+before formatting, for example `--format json --filter-output+evidence_id,normalized_result.statistics`. Complete serialization of a result+beyond the engine's string limit is not established by this digest check.+Client framing limits also apply: the pinned Node MCP SDK's stdio transport+defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate+the complete response, including text and structured Evidence projections.+## What is reconstructedThe projector reuses the content-addressed artifact inventory and safe artifactdiff --git a/package.json b/package.jsonindex f9d6de563..5caf04138 100644--- a/package.json+++ b/package.json@@ -128,6 +128,7 @@"verify:electron": "npm run build:cached && node scripts/verify-real-electron.mjs","verify:inspector": "npm run build:cached && node scripts/verify-real-javascript-runtime-observation.mjs","verify:javascript-runtime": "npm run build:cached && node scripts/verify-local-javascript-runtime.mjs",+ "verify:javascript:digests": "npm run build:cached && node scripts/verify-large-javascript-digests.mjs","verify:application-workflows": "npm run build:cached && node scripts/verify-local-application-workflows.mjs","verify:managed": "npm run build:cached && node scripts/verify-managed-conformance.mjs","verify:agent": "npm run build:cached && node scripts/verify-agent-experience.mjs",diff --git a/scripts/verify-large-javascript-digests.mjs b/scripts/verify-large-javascript-digests.mjsnew file mode 100644index 000000000..14fa5977e--- /dev/null+++ b/scripts/verify-large-javascript-digests.mjs@@ -0,0 +1,38 @@+#!/usr/bin/env node++import assert from "node:assert/strict";+import { constants } from "node:buffer";+import { createHash } from "node:crypto";++import { canonicalDigest } from "../dist/domain/comparisonSemantics.js";+import { createEvidence, parseEvidence } from "../dist/domain/evidence.js";++// This opt-in check crosses the actual engine string limit; keep it outside+// routine source tests. Build first, then run this script with the local Node.+const leaf = "x".repeat(1024 * 1024);+const encodedLeaf = JSON.stringify(leaf);+const count = Math.ceil(constants.MAX_STRING_LENGTH / encodedLeaf.length) + 1;+const value = { payload: Array.from({ length: count }, () => leaf) };+const expected = createHash("sha256");+expected.update('{"payload":[');+for (let index = 0; index < count; index += 1) {+ if (index > 0) expected.update(",");+ expected.update(encodedLeaf);+}+expected.update("]}");+assert.equal(canonicalDigest(value), expected.digest("hex"));++const evidence = createEvidence(+ undefined,+ { id: "large-json-fixture", name: "Large JSON fixture", version: "1" },+ { operation: "inspect_fixture", parameters: {}, result: value },+);+assert.equal(parseEvidence(evidence).evidence_id, evidence.evidence_id);+process.stdout.write(+ `${JSON.stringify({+ verified: true,+ serialized_value_bytes: count * encodedLeaf.length + count - 1 + 14,+ engine_string_limit: constants.MAX_STRING_LENGTH,+ evidence_id: evidence.evidence_id,+ })}\n`,+);diff --git a/src/application/JavaScriptApplicationService.test.ts b/src/application/JavaScriptApplicationService.test.tsnew file mode 100644index 000000000..f43d3ce19--- /dev/null+++ b/src/application/JavaScriptApplicationService.test.ts@@ -0,0 +1,87 @@+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { describe, expect, it } from "vitest";++import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";+import { projectAnalysisError } from "../domain/analysisErrorProjection.js";+import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";++describe("JavaScript application failure diagnostics", () => {+ it("retains unexpected failures and their input identity in caller-visible diagnostics", async () => {+ const inputPath = await createTestTempDirectory("rea-js-failure-");+ await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");+ const cause = new RangeError("Invalid string length");+ const result = await analyzeJavaScriptApplication(+ { input_path: inputPath, format: "directory" },+ {+ progress: {+ report: async (event) => {+ if (event.terminal) throw cause;+ },+ },+ },+ );+ expect(result.ok).toBe(false);+ if (result.ok) throw new Error("Expected analysis failure");+ expect(result.error.cause).toBe(cause);+ expect(projectAnalysisError(result.error)).toMatchObject({+ code: "execution_failure",+ details: {+ provider_id: "rea-javascript-application",+ operation: "analyze_javascript_application",+ diagnostics: {+ input_path: inputPath,+ error_name: "RangeError",+ error_message: "Invalid string length",+ },+ },+ });+ });++ it("keeps filesystem failures distinct from engine failures", async () => {+ const root = await createTestTempDirectory("rea-js-missing-");+ const result = await analyzeJavaScriptApplication({+ input_path: join(root, "missing"),+ format: "directory",+ });+ expect(result.ok).toBe(false);+ if (result.ok) throw new Error("Expected missing artifact failure");+ expect(projectAnalysisError(result.error)).toMatchObject({+ code: "artifact_operation_failed",+ details: { operation: "analyze_javascript_application", reason: "io" },+ });+ });++ it.each(["reported failure", null])(+ "retains non-Error failures (%s)",+ async (cause) => {+ const inputPath = await createTestTempDirectory("rea-js-non-error-");+ await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");+ const result = await analyzeJavaScriptApplication(+ { input_path: inputPath, format: "directory" },+ {+ progress: {+ report: async (event) => {+ if (event.terminal) throw cause;+ },+ },+ },+ );+ expect(result.ok).toBe(false);+ if (result.ok) throw new Error("Expected analysis failure");+ expect(projectAnalysisError(result.error)).toMatchObject({+ code: "execution_failure",+ details: {+ diagnostics: {+ error_name: "UnknownError",+ error_message:+ typeof cause === "string"+ ? cause+ : "JavaScript analysis failed with a non-Error value",+ },+ },+ });+ },+ );+});diff --git a/src/application/JavaScriptApplicationService.ts b/src/application/JavaScriptApplicationService.tsindex 8e51a6d0f..b2389cb68 100644--- a/src/application/JavaScriptApplicationService.ts+++ b/src/application/JavaScriptApplicationService.ts@@ -14,9 +14,11 @@ import { type AnalysisError } from "../domain/analysisErrorBase.js";import type { Evidence } from "../domain/evidence.js";import { projectInputIssues } from "../domain/inputIssueProjection.js";import { err, ok, type Result } from "../domain/result.js";+import { ProviderAdapterError } from "../domain/providerAdapterError.js";import type { ExecutionOptions } from "./AnalysisProvider.js";import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js";+import { JAVASCRIPT_APPLICATION_PROVIDER } from "./InvestigationProviders.js";const OPERATION = "analyze_javascript_application" as const;@@ -82,6 +84,42 @@ export const analyzeJavaScriptApplicationValidated = async ({ cause },),);- return err(new ArtifactOperationError(OPERATION, "io"));+ if (+ cause instanceof Error &&+ "code" in cause &&+ typeof cause.code === "string" &&+ FILESYSTEM_ERROR_CODES.has(cause.code)+ )+ return err(new ArtifactOperationError(OPERATION, "io"));+ return err(+ new ProviderAdapterError(JAVASCRIPT_APPLICATION_PROVIDER.id, OPERATION, {+ cause,+ diagnostics: {+ input_path: input.input_path,+ error_name: cause instanceof Error ? cause.name : "UnknownError",+ error_message:+ cause instanceof Error+ ? cause.message+ : typeof cause === "string"+ ? cause+ : "JavaScript analysis failed with a non-Error value",+ },+ }),+ );}};++const FILESYSTEM_ERROR_CODES = new Set([+ "ENOENT",+ "EACCES",+ "EPERM",+ "ENOTDIR",+ "EISDIR",+ "EIO",+ "ELOOP",+ "ENAMETOOLONG",+ "EMFILE",+ "ENFILE",+ "ENOSPC",+ "EROFS",+]);diff --git a/src/domain/canonicalDigest.test.ts b/src/domain/canonicalDigest.test.tsnew file mode 100644index 000000000..02a641905--- /dev/null+++ b/src/domain/canonicalDigest.test.ts@@ -0,0 +1,83 @@+import { createHash } from "node:crypto";++import { fc, it } from "@fast-check/vitest";+import canonicalize from "canonicalize";+import { describe, expect } from "vitest";++import { digestCanonicalValue } from "./canonicalDigest.js";++const legacyDigest = (value: unknown): string => {+ const encoded = canonicalize(value);+ if (encoded === undefined) throw new TypeError("Unserializable fixture");+ return createHash("sha256").update(encoded).digest("hex");+};++describe("incremental canonical digest", () => {+ it.prop([fc.jsonValue()])(+ "preserves canonicalize digests for JSON values",+ (value) => {+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));+ },+ );++ it("preserves Unicode ordering, escaping, numeric forms and non-JSON normalization", () => {+ const shared = { key: "value" };+ const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined];+ sparse.length = 6;+ const fixtures: unknown[] = [+ {+ "€": "\ud800",+ "😀": "\udc00",+ "\r": '\n\t"\\',+ number: -0,+ exponent: 1e30,+ },+ { omitted: undefined, symbol: Symbol("omit"), fn: () => undefined },+ sparse,+ [shared, shared],+ new Date("2026-10-06T00:00:00Z"),+ { toJSON: () => ({ b: 2, a: [1, null] }) },+ { nested: { toJSON: () => undefined } },+ ];+ for (const value of fixtures)+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));+ });++ it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => {+ for (const value of [undefined, Symbol("root"), () => undefined])+ expect(() => digestCanonicalValue(value, "Fixture")).toThrow(+ "Fixture could not canonicalize data",+ );+ const cyclic: Record<string, unknown> = {};+ cyclic.self = cyclic;+ const recursiveJson = { toJSON: (): unknown => recursiveJson };+ for (const value of [NaN, Infinity, -Infinity, 1n, cyclic, recursiveJson])+ expect(() => digestCanonicalValue(value)).toThrow();+ });++ it("hashes repeated large leaves with the same byte sequence", () => {+ const value = {+ payload: Array.from({ length: 64 }, () => "x".repeat(16_384)),+ };+ expect(digestCanonicalValue(value)).toBe(legacyDigest(value));+ });++ it("preserves canonicalize property access and toJSON cycle behavior", () => {+ const createAccessor = () => {+ let reads = 0;+ return {+ get value() {+ reads += 1;+ return reads;+ },+ };+ };+ expect(digestCanonicalValue(createAccessor())).toBe(+ legacyDigest(createAccessor()),+ );+ const shared = { toJSON: () => ({ answer: 42 }) };+ expect(digestCanonicalValue([shared, shared])).toBe(+ legacyDigest([shared, shared]),+ );+ });+});diff --git a/src/domain/canonicalDigest.ts b/src/domain/canonicalDigest.tsnew file mode 100644index 000000000..c9f1c6260--- /dev/null+++ b/src/domain/canonicalDigest.ts@@ -0,0 +1,75 @@+import { createHash } from "node:crypto";++/** Hash canonicalize-compatible JSON without assembling an aggregate string. */+export const digestCanonicalValue = (+ value: unknown,+ context = "Comparison",+): string => {+ const hash = createHash("sha256");+ const encoded = emitCanonical(value, (part) => hash.update(part), new Set());+ if (!encoded) throw new TypeError(`${context} could not canonicalize data`);+ return hash.digest("hex");+};++type Emit = (part: string) => void;++const emitCanonical = (+ value: unknown,+ emit: Emit,+ ancestors: Set<object>,+): boolean => {+ if (typeof value === "number" && !Number.isFinite(value))+ throw new Error(+ Number.isNaN(value) ? "NaN is not allowed" : "Infinity is not allowed",+ );+ if (value === null || typeof value !== "object") {+ const encoded = JSON.stringify(value);+ if (encoded === undefined) return false;+ emit(encoded);+ return true;+ }+ if (ancestors.has(value)) throw new Error("Circular reference detected");+ ancestors.add(value);+ try {+ if ("toJSON" in value && typeof value.toJSON === "function") {+ const normalized: unknown = value.toJSON();+ return emitCanonical(normalized, emit, ancestors);+ }+ if (Array.isArray(value)) {+ emit("[");+ const length = value.length;+ for (let index = 0; index < length; index += 1) {+ if (index > 0) emit(",");+ // Preserve canonicalize's sparse-array and unsupported-value behavior.+ if (!(index in value)) continue;+ const item: unknown = value[index];+ emitCanonical(+ item === undefined || typeof item === "symbol" ? null : item,+ emit,+ ancestors,+ );+ }+ emit("]");+ } else {+ emit("{");+ let first = true;+ for (const key of Object.keys(value).sort()) {+ if (+ Reflect.get(value, key) === undefined ||+ typeof Reflect.get(value, key) === "symbol"+ )+ continue;+ const item: unknown = Reflect.get(value, key);+ if (!first) emit(",");+ first = false;+ emit(JSON.stringify(key));+ emit(":");+ if (!emitCanonical(item, emit, ancestors)) emit("undefined");+ }+ emit("}");+ }+ return true;+ } finally {+ ancestors.delete(value);+ }+};diff --git a/src/domain/comparisonSemantics.ts b/src/domain/comparisonSemantics.tsindex 804a446ba..9c005d124 100644--- a/src/domain/comparisonSemantics.ts+++ b/src/domain/comparisonSemantics.ts@@ -1,7 +1,7 @@-import { createHash } from "node:crypto";-import canonicalize from "canonicalize";+import { digestCanonicalValue } from "./canonicalDigest.js";+/** Whether an inventory can support a claim that an item is absent. */export const absenceClaimable = (coverage: {readonly status: string;@@ -17,9 +17,7 @@ export const canonicalDigest = (value: unknown,context = "Comparison",): string => {- return createHash("sha256")- .update(canonicalJson(value, context))- .digest("hex");+ return digestCanonicalValue(value, context);};/** Canonical JSON used for deterministic tie breaking and stable digests. */diff --git a/src/domain/evidence.ts b/src/domain/evidence.tsindex c7ee5b885..c4ebc7187 100644--- a/src/domain/evidence.ts+++ b/src/domain/evidence.ts@@ -1,8 +1,7 @@-import { createHash } from "node:crypto";--import canonicalize from "canonicalize";import { z } from "zod";+import { digestCanonicalValue } from "./canonicalDigest.js";+import {analysisProfileSchema,type AnalysisProfileCommitment,@@ -173,22 +172,11 @@ export interface EvidenceObservation {readonly evidenceLinks?: readonly string[];}-const sha256 = (value: string): string =>- createHash("sha256").update(value).digest("hex");-type WithoutEvidenceId<Record_> = Record_ extends unknown? Omit<Record_, "evidence_id">: never;type EvidenceWithoutId = WithoutEvidenceId<Evidence>;-/** Serialize JSON according to RFC 8785 JSON Canonicalization Scheme. */-const canonicalJson = (value: JsonValue): string => {- const serialized = canonicalize(value);- if (serialized === undefined)- throw new TypeError("RFC 8785 canonicalization rejected a JSON value");- return serialized;-};-const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({subject:evidence.subject === null@@ -217,7 +205,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({/** Recompute the semantic identifier, excluding paths and raw payload bytes. */const computeEvidenceId = (evidence: EvidenceWithoutId): string =>- `ev_${sha256(canonicalJson(semanticProjection(evidence)))}`;+ `ev_${digestCanonicalValue(semanticProjection(evidence), "Evidence")}`;/** Parse evidence and reject a syntactically valid but tampered semantic ID. */export const parseEvidence = (input: unknown): Evidence => {