Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(process): name the reason two captures cannot be compared (#1080)
Kaoru0822eb69763d27e2
diff --git a/docs/process-capture.md b/docs/process-capture.mdindex 78d84bf91..20195bdc4 100644--- a/docs/process-capture.md+++ b/docs/process-capture.md@@ -130,7 +130,12 @@ causal evidence. Trace comparison returns `unknown` when a capture lacks therequired complete event journal or contains relevant residual unknowns.Each capture carries commitments for the selected scenario, executable,-comparison contract, and normalization rules. Comparison rejects captures-whose comparison contracts differ. Captures are local Evidence files; keep+comparison contract, and normalization rules. The comparison contract covers+the working directory, explicit environment, filesystem observation paths,+terminal size, scripted events, timeouts, limits, and normalization; the+executable and its arguments may differ. Comparison rejects captures whose+comparison contracts differ and names the differing fields, so run both+scenarios from the same absolute working directory and observation paths.+Captures are local Evidence files; keeptheir source artifacts and invocation context available when interpreting adifference.diff --git a/src/application/process/ProcessCli.ts b/src/application/process/ProcessCli.tsindex 3d0e53749..7c60613ef 100644--- a/src/application/process/ProcessCli.ts+++ b/src/application/process/ProcessCli.ts@@ -5,6 +5,7 @@ import { AnalysisError } from "../../domain/analysisErrorBase.js";import { AnalysisInputError } from "../../domain/analysisErrorCore.js";import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";import { createEvidence, parseEvidence } from "../../domain/evidence.js";+import { describeValidationFailure } from "../../domain/evidenceBundle.js";import { jsonValueSchema } from "../../domain/jsonValue.js";import { projectInputIssues } from "../../domain/inputIssueProjection.js";import { processTraceSpecificationSchema } from "../../domain/process/processTraceComparison.js";@@ -120,7 +121,7 @@ const parseCaptureEvidence = (input: unknown) => {{ cause },projectInputIssues(cause.issues, input),);- throw invalidCaptureEvidence();+ throw invalidCaptureEvidence(cause);}if (evidence.operation !== "capture_process_scenario" ||@@ -140,24 +141,28 @@ const parseCaptureEvidence = (input: unknown) => {locations: evidence.locations,};} catch (cause: unknown) {- throw invalidCaptureEvidence();+ throw invalidCaptureEvidence(cause);}};const parseTraceSpecification = (input: unknown) => {const parsed = processTraceSpecificationSchema.safeParse(input);if (parsed.success) return parsed.data;+ // Match the MCP boundary: the operation name and a trace_spec-rooted path.throw new AnalysisInputError(- "compare-process-captures",+ "compare_process_captures",{ cause: parsed.error },- projectInputIssues(parsed.error.issues, input),+ projectInputIssues(parsed.error.issues, input).map((issue) => ({+ ...issue,+ path: ["trace_spec", ...issue.path],+ })),);};-const invalidCaptureEvidence = (): ProcessCliFailure =>+const invalidCaptureEvidence = (cause: unknown): ProcessCliFailure =>new ProcessCliFailure("invalid_input",- "Capture evidence is malformed. Create new capture evidence, then try again.",+ `Capture evidence is malformed (${describeValidationFailure(cause)}). Create new capture evidence, then try again.`,);const readJson = async (path: string): Promise<unknown> => {diff --git a/src/domain/process/processCapture.validation.test.ts b/src/domain/process/processCapture.validation.test.tsindex 912b7c16d..558d4743c 100644--- a/src/domain/process/processCapture.validation.test.ts+++ b/src/domain/process/processCapture.validation.test.ts@@ -241,11 +241,33 @@ it("requires compatible contracts and enforces capture age through a clock seam"}),},}),- ).toThrow("incompatible comparison contracts");+ ).toThrow(+ expect.objectContaining({+ issues: [+ expect.objectContaining({+ path: ["right"],+ message: expect.stringContaining(+ "incompatible comparison contracts; these scenario fields differ: changed",+ ),+ expected: ["changed"],+ }),+ ],+ }),+ );expect(() =>compareProcessCaptures(capture, capture, {maxCaptureAgeMs: 1,now: () => Date.parse("2026-01-01T00:00:01.000Z"),}),- ).toThrow("max_capture_age_ms");+ ).toThrow(+ expect.objectContaining({+ issues: [+ expect.objectContaining({+ path: ["max_capture_age_ms"],+ reason: "out_of_range",+ message: expect.stringContaining("left completed at"),+ }),+ ],+ }),+ );});diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.tsindex b6e5a8cc0..1aa052bc4 100644--- a/src/domain/process/processComparison.ts+++ b/src/domain/process/processComparison.ts@@ -2,6 +2,7 @@ import canonicalize from "canonicalize";import { z } from "zod";import type { ProcessCapture } from "./processCapture.js";+import { AnalysisInputError } from "../analysisErrorCore.js";import { jsonValueSchema } from "../jsonValue.js";import {compareProcessTraces,@@ -14,6 +15,8 @@ import {traceCoversObservedDimension,} from "./processTraceDimensionProjection.js";+const OPERATION = "compare_process_captures";+/** Comparison classification that never equates incomplete evidence. */export const comparisonStatusSchema = z.enum(["unchanged",@@ -228,22 +231,60 @@ const assertComparable = (if (left.manifest.comparison_contract_sha256 !==right.manifest.comparison_contract_sha256- )- throw new TypeError(- "Process captures have incompatible comparison contracts",+ ) {+ const fields = differingContractFields(+ left.manifest.comparison_contract,+ right.manifest.comparison_contract,);+ throw new AnalysisInputError(OPERATION, undefined, [+ {+ path: ["right"],+ reason: "invalid_value",+ message: `Process captures have incompatible comparison contracts${fields.length === 0 ? "" : `; these scenario fields differ: ${fields.join(", ")}`}. Capture both scenarios with the same values for these fields, then compare them.`,+ expected: fields,+ },+ ]);+ }if (options.maxCaptureAgeMs === undefined) return;const maxCaptureAgeMs = options.maxCaptureAgeMs;const now = (options.now ?? Date.now)();- if (- [left, right].some(- ({ manifest }) =>- now - Date.parse(manifest.completed_at) > maxCaptureAgeMs,- )- )- throw new TypeError("Process capture exceeds max_capture_age_ms");+ const stale = (+ [+ ["left", left],+ ["right", right],+ ] as const+ ).flatMap(([side, { manifest }]) => {+ const age = now - Date.parse(manifest.completed_at);+ return age > maxCaptureAgeMs+ ? [+ `${side} completed at ${manifest.completed_at} (${String(age)} ms ago)`,+ ]+ : [];+ });+ if (stale.length > 0)+ throw new AnalysisInputError(OPERATION, undefined, [+ {+ path: ["max_capture_age_ms"],+ reason: "out_of_range",+ message: `Process capture exceeds max_capture_age_ms ${String(maxCaptureAgeMs)}: ${stale.join("; ")}. Recapture the stale scenario or raise max_capture_age_ms.`,+ },+ ]);};+/** Name top-level contract fields whose canonical values differ. */+const differingContractFields = (+ left: Readonly<Record<string, unknown>>,+ right: Readonly<Record<string, unknown>>,+): string[] =>+ [...new Set([...Object.keys(left), ...Object.keys(right)])]+ .filter(+ (field) =>+ !Object.hasOwn(left, field) ||+ !Object.hasOwn(right, field) ||+ canonicalize(left[field]) !== canonicalize(right[field]),+ )+ .sort();+const truncatedComparison = (): ProcessCaptureComparison => ({status: "truncated",terminal: "truncated",diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.tsindex 3b47d9c42..bff0492cb 100644--- a/src/server/registerProcessComparisonTool.ts+++ b/src/server/registerProcessComparisonTool.ts@@ -2,6 +2,9 @@ import type { McpServer } from "@modelcontextprotocol/server";import type { BinarySessionPort } from "../application/binary/BinarySession.js";import { toolContract } from "../contracts/toolContracts.js";+import { AnalysisInputError } from "../domain/analysisErrorCore.js";+import { type AnalysisError } from "../domain/analysisErrorBase.js";+import { describeValidationFailure } from "../domain/evidenceBundle.js";import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";import {createEvidence,@@ -13,9 +16,10 @@ import { jsonValueSchema } from "../domain/jsonValue.js";import {compareProcessCaptures,parseProcessCapture,+ type ProcessCapture,} from "../domain/process/processCapture.js";import type { RecordUnknownInput } from "../domain/residualUnknown.js";-import { err } from "../domain/result.js";+import { err, ok, type Result } from "../domain/result.js";import { recordDerivedEvidence } from "./recordDerivedEvidence.js";import { recordSessionEvidenceSources } from "./sessionEvidence.js";import { runDerivedOperation } from "./runDerivedOperation.js";@@ -33,6 +37,57 @@ const sourceLocations = (right: readonly EvidenceLocation[] | undefined,): readonly EvidenceLocation[] => [...(left ?? []), ...(right ?? [])];+/**+ * Parse one side and name the failed constraint: tampered Evidence, Evidence+ * from another workflow, or an invalid capture result.+ */+const parseCaptureSide = (+ operation: string,+ side: "left" | "right",+ value: unknown,+): Result<+ { readonly record: Evidence; readonly capture: ProcessCapture },+ AnalysisError+> => {+ let record: Evidence;+ try {+ record = parseEvidence(value);+ } catch (cause: unknown) {+ return err(+ new EvidenceIntegrityError(`${side} Evidence failed validation`, {+ cause,+ userMessage: `The ${side} Evidence failed validation (${describeValidationFailure(cause)}). Supply unmodified capture Evidence from capture_process_scenario.`,+ }),+ );+ }+ if (+ record.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||+ record.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate+ )+ return err(+ new AnalysisInputError(operation, undefined, [+ {+ path: [side],+ reason: "invalid_value",+ message: `Expected process capture Evidence (operation ${PROCESS_CAPTURE_EVIDENCE.operation}, predicate ${PROCESS_CAPTURE_EVIDENCE.predicate}); the ${side} Evidence has operation ${record.operation} and predicate ${record.predicate_type}.`,+ },+ ]),+ );+ try {+ return ok({+ record,+ capture: parseProcessCapture(record.normalized_result),+ });+ } catch (cause: unknown) {+ return err(+ new EvidenceIntegrityError(`${side} process capture failed validation`, {+ cause,+ userMessage: `The ${side} Evidence has an invalid process capture result (${describeValidationFailure(cause)}). Recreate the capture with capture_process_scenario.`,+ }),+ );+ }+};+/** Register deterministic process-capture comparison and contradiction tracking. */export const registerProcessComparisonTool = (server: McpServer,@@ -44,59 +99,25 @@ export const registerProcessComparisonTool = (contract.name,toolRegistrationOptions(contract),async (input, context) => {- let leftRecord: Evidence;- let rightRecord: Evidence;- try {- leftRecord = parseEvidence(input.left);- rightRecord = parseEvidence(input.right);- if (- leftRecord.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||- rightRecord.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||- leftRecord.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate ||- rightRecord.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate- )- throw new TypeError("Expected process capture Evidence");- } catch (cause: unknown) {- return toCallToolResult(- err(- new EvidenceIntegrityError(- cause instanceof Error ? cause.message : "Invalid Evidence",- ),- ),- contract,- );- }- let comparison: ReturnType<typeof compareProcessCaptures>;- let leftCapture: ReturnType<typeof parseProcessCapture>;- let rightCapture: ReturnType<typeof parseProcessCapture>;- try {- leftCapture = parseProcessCapture(leftRecord.normalized_result);- rightCapture = parseProcessCapture(rightRecord.normalized_result);- const computed = await runDerivedOperation(context, contract.name, () =>- compareProcessCaptures(leftCapture, rightCapture, {- ...(input.max_capture_age_ms === undefined- ? {}- : { maxCaptureAgeMs: input.max_capture_age_ms }),- ...(input.trace_spec === undefined- ? {}- : { traceSpecification: input.trace_spec }),- now,- }),- );- if (!computed.ok) return toCallToolResult(computed, contract);- comparison = computed.value;- } catch (cause: unknown) {- return toCallToolResult(- err(- new EvidenceIntegrityError(- cause instanceof Error- ? cause.message- : "Invalid process capture",- ),- ),- contract,- );- }+ const left = parseCaptureSide(contract.name, "left", input.left);+ if (!left.ok) return toCallToolResult(left, contract);+ const right = parseCaptureSide(contract.name, "right", input.right);+ if (!right.ok) return toCallToolResult(right, contract);+ const { record: leftRecord, capture: leftCapture } = left.value;+ const { record: rightRecord, capture: rightCapture } = right.value;+ const computed = await runDerivedOperation(context, contract.name, () =>+ compareProcessCaptures(leftCapture, rightCapture, {+ ...(input.max_capture_age_ms === undefined+ ? {}+ : { maxCaptureAgeMs: input.max_capture_age_ms }),+ ...(input.trace_spec === undefined+ ? {}+ : { traceSpecification: input.trace_spec }),+ now,+ }),+ );+ if (!computed.ok) return toCallToolResult(computed, contract);+ const comparison = computed.value;const evidence = createEvidence(undefined, PROCESS_PROVIDER, {predicateType: "rea.process-comparison",operation: contract.name,diff --git a/src/server/runDerivedOperation.ts b/src/server/runDerivedOperation.tsindex 1fbdf39ba..0a7a55da6 100644--- a/src/server/runDerivedOperation.ts+++ b/src/server/runDerivedOperation.ts@@ -33,8 +33,10 @@ export const runDerivedOperation = async <Value>(try {value = compute();} catch (cause: unknown) {+ // Typed causes already name the failed constraint; keep their issues.return err(- cause instanceof AnalysisCancelledError+ cause instanceof AnalysisCancelledError ||+ cause instanceof AnalysisInputError? cause: new AnalysisInputError(operation, { cause }),);diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.tsnew file mode 100644index 000000000..81722a384--- /dev/null+++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts@@ -0,0 +1,210 @@+import { writeFile } from "node:fs/promises";+import { join } from "node:path";++import { Client, InMemoryTransport } from "@modelcontextprotocol/client";+import { expect, it, onTestFinished } from "vitest";++import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";+import { PROCESS_PROVIDER } from "../../../src/application/process/ProcessEvidence.js";+import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/contracts/process/processCaptureExample.js";+import { createEvidence } from "../../../src/domain/evidence.js";+import { jsonValueSchema } from "../../../src/domain/jsonValue.js";+import { digestProcessCommitment } from "../../../src/domain/process/processCapture.js";+import { createServer } from "../../../src/server/createServer.js";+import { createTestBinarySession } from "../../fixtures/binarySession.js";+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";++const captureEvidence = (+ side: "left" | "right",+ comparisonContract: Readonly<Record<string, string>> = {},+) =>+ createEvidence(undefined, PROCESS_PROVIDER, {+ predicateType: "rea.process-capture",+ operation: "capture_process_scenario",+ parameters: { side },+ result: jsonValueSchema.parse({+ ...EMPTY_PROCESS_CAPTURE_EXAMPLE,+ manifest: {+ ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,+ comparison_contract: comparisonContract,+ comparison_contract_sha256: digestProcessCommitment(comparisonContract),+ },+ }),+ confidence: "observed",+ authority: "controlled-replay",+ });++const left = captureEvidence("left", { working_directory: "/work/reference" });+const right = captureEvidence("right", {+ working_directory: "/work/reconstruction",+});++const connect = async () => {+ const session = createTestBinarySession(() => {+ throw new Error("Process comparison must not launch a provider");+ });+ const server = createServer(session, session);+ const client = new Client({ name: "process-comparison-test", version: "1" });+ onTestFinished(async () => {+ await Promise.allSettled([client.close(), server.close(), session.close()]);+ });+ const [clientTransport, serverTransport] =+ InMemoryTransport.createLinkedPair();+ await server.connect(serverTransport);+ await client.connect(clientTransport);+ return client;+};++it("names differing comparison contract fields through MCP and the CLI", async () => {+ const client = await connect();+ const response = await client.callTool({+ name: "compare_process_captures",+ arguments: { left, right },+ });+ expect(response.isError).toBe(true);+ const issue = {+ path: ["right"],+ reason: "invalid_value",+ message: expect.stringContaining(+ "these scenario fields differ: working_directory",+ ),+ expected: ["working_directory"],+ };+ expect(response.structuredContent).toMatchObject({+ error: { code: "invalid_request", details: { issues: [issue] } },+ });++ const root = await createTestTempDirectory("rea-process-contract-");+ const leftPath = join(root, "left.json");+ const rightPath = join(root, "right.json");+ await writeFile(leftPath, JSON.stringify(left));+ await writeFile(rightPath, JSON.stringify(right));+ expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({+ error: "Process command failed",+ code: "invalid_request",+ details: { operation: "compare_process_captures", issues: [issue] },+ });+});++it("names the stale capture that exceeds max_capture_age_ms", async () => {+ const client = await connect();+ const response = await client.callTool({+ name: "compare_process_captures",+ arguments: {+ left: captureEvidence("left"),+ right: captureEvidence("right"),+ max_capture_age_ms: 1000,+ },+ });+ expect(response.isError).toBe(true);+ expect(response.structuredContent).toMatchObject({+ error: {+ code: "invalid_request",+ details: {+ issues: [+ {+ path: ["max_capture_age_ms"],+ reason: "out_of_range",+ message: expect.stringContaining(+ `left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,+ ),+ },+ ],+ },+ },+ });+});++it("reports CLI trace specification issues like the MCP trace_spec input", async () => {+ const root = await createTestTempDirectory("rea-process-trace-spec-");+ const capturePath = join(root, "capture.json");+ const specPath = join(root, "spec.json");+ await writeFile(capturePath, JSON.stringify(captureEvidence("left")));+ await writeFile(specPath, JSON.stringify({ language: {} }));+ expect(+ await compareProcessEvidenceFiles(capturePath, capturePath, specPath),+ ).toMatchObject({+ code: "invalid_request",+ details: {+ operation: "compare_process_captures",+ issues: expect.arrayContaining([+ expect.objectContaining({ path: ["trace_spec", "events"] }),+ ]),+ },+ });+});++it("names the side and constraint when Evidence is not a usable capture", async () => {+ const client = await connect();+ const capture = captureEvidence("right");+ const unrelated = createEvidence(undefined, PROCESS_PROVIDER, {+ operation: "other",+ parameters: {},+ result: {},+ });+ const wrongKind = await client.callTool({+ name: "compare_process_captures",+ arguments: { left: unrelated, right: capture },+ });+ expect(wrongKind.structuredContent).toMatchObject({+ error: {+ code: "invalid_request",+ details: {+ issues: [+ {+ path: ["left"],+ reason: "invalid_value",+ message: expect.stringContaining(+ "the left Evidence has operation other",+ ),+ },+ ],+ },+ },+ });++ const tampered = await client.callTool({+ name: "compare_process_captures",+ arguments: {+ left: capture,+ right: { ...capture, parameters: { side: "changed" } },+ },+ });+ expect(tampered.structuredContent).toMatchObject({+ error: {+ code: "evidence_integrity_mismatch",+ message: expect.stringContaining(+ "The right Evidence failed validation (Evidence semantic identifier does not match its record)",+ ),+ },+ });++ const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, {+ predicateType: "rea.process-capture",+ operation: "capture_process_scenario",+ parameters: {},+ result: { manifest: {} },+ });+ const malformed = await client.callTool({+ name: "compare_process_captures",+ arguments: { left: capture, right: invalidResult },+ });+ expect(malformed.structuredContent).toMatchObject({+ error: {+ code: "evidence_integrity_mismatch",+ message: expect.stringContaining(+ "The right Evidence has an invalid process capture result (at ",+ ),+ },+ });++ const root = await createTestTempDirectory("rea-process-malformed-");+ const leftPath = join(root, "left.json");+ const rightPath = join(root, "right.json");+ await writeFile(leftPath, JSON.stringify(capture));+ await writeFile(rightPath, JSON.stringify(invalidResult));+ expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({+ category: "invalid_input",+ message: expect.stringContaining("Capture evidence is malformed (at "),+ });+});