Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process): name the reason two captures cannot be compared (#1080)

Kaoru0822eb69763d27e2
diff --git a/docs/process-capture.md b/docs/process-capture.md
index 78d84bf91..20195bdc4 100644
--- a/docs/process-capture.md
+++ b/docs/process-capture.md
@@ -130,7 +130,12 @@ causal evidence. Trace comparison returns `unknown` when a capture lacks the
required complete event journal or contains relevant residual unknowns.
Each capture carries commitments for the selected scenario, executable,
-comparison contract, and normalization rules. Comparison rejects captures
-whose comparison contracts differ. Captures are local Evidence files; keep
+comparison contract, and normalization rules. The comparison contract covers
+the working directory, explicit environment, filesystem observation paths,
+terminal size, scripted events, timeouts, limits, and normalization; the
+executable and its arguments may differ. Comparison rejects captures whose
+comparison contracts differ and names the differing fields, so run both
+scenarios from the same absolute working directory and observation paths.
+Captures are local Evidence files; keep
their source artifacts and invocation context available when interpreting a
difference.
diff --git a/src/application/process/ProcessCli.ts b/src/application/process/ProcessCli.ts
index 3d0e53749..7c60613ef 100644
--- a/src/application/process/ProcessCli.ts
+++ b/src/application/process/ProcessCli.ts
@@ -5,6 +5,7 @@ import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
import { createEvidence, parseEvidence } from "../../domain/evidence.js";
+import { describeValidationFailure } from "../../domain/evidenceBundle.js";
import { jsonValueSchema } from "../../domain/jsonValue.js";
import { projectInputIssues } from "../../domain/inputIssueProjection.js";
import { processTraceSpecificationSchema } from "../../domain/process/processTraceComparison.js";
@@ -120,7 +121,7 @@ const parseCaptureEvidence = (input: unknown) => {
{ cause },
projectInputIssues(cause.issues, input),
);
- throw invalidCaptureEvidence();
+ throw invalidCaptureEvidence(cause);
}
if (
evidence.operation !== "capture_process_scenario" ||
@@ -140,24 +141,28 @@ const parseCaptureEvidence = (input: unknown) => {
locations: evidence.locations,
};
} catch (cause: unknown) {
- throw invalidCaptureEvidence();
+ throw invalidCaptureEvidence(cause);
}
};
const parseTraceSpecification = (input: unknown) => {
const parsed = processTraceSpecificationSchema.safeParse(input);
if (parsed.success) return parsed.data;
+ // Match the MCP boundary: the operation name and a trace_spec-rooted path.
throw new AnalysisInputError(
- "compare-process-captures",
+ "compare_process_captures",
{ cause: parsed.error },
- projectInputIssues(parsed.error.issues, input),
+ projectInputIssues(parsed.error.issues, input).map((issue) => ({
+ ...issue,
+ path: ["trace_spec", ...issue.path],
+ })),
);
};
-const invalidCaptureEvidence = (): ProcessCliFailure =>
+const invalidCaptureEvidence = (cause: unknown): ProcessCliFailure =>
new ProcessCliFailure(
"invalid_input",
- "Capture evidence is malformed. Create new capture evidence, then try again.",
+ `Capture evidence is malformed (${describeValidationFailure(cause)}). Create new capture evidence, then try again.`,
);
const readJson = async (path: string): Promise<unknown> => {
diff --git a/src/domain/process/processCapture.validation.test.ts b/src/domain/process/processCapture.validation.test.ts
index 912b7c16d..558d4743c 100644
--- a/src/domain/process/processCapture.validation.test.ts
+++ b/src/domain/process/processCapture.validation.test.ts
@@ -241,11 +241,33 @@ it("requires compatible contracts and enforces capture age through a clock seam"
}),
},
}),
- ).toThrow("incompatible comparison contracts");
+ ).toThrow(
+ expect.objectContaining({
+ issues: [
+ expect.objectContaining({
+ path: ["right"],
+ message: expect.stringContaining(
+ "incompatible comparison contracts; these scenario fields differ: changed",
+ ),
+ expected: ["changed"],
+ }),
+ ],
+ }),
+ );
expect(() =>
compareProcessCaptures(capture, capture, {
maxCaptureAgeMs: 1,
now: () => Date.parse("2026-01-01T00:00:01.000Z"),
}),
- ).toThrow("max_capture_age_ms");
+ ).toThrow(
+ expect.objectContaining({
+ issues: [
+ expect.objectContaining({
+ path: ["max_capture_age_ms"],
+ reason: "out_of_range",
+ message: expect.stringContaining("left completed at"),
+ }),
+ ],
+ }),
+ );
});
diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts
index b6e5a8cc0..1aa052bc4 100644
--- a/src/domain/process/processComparison.ts
+++ b/src/domain/process/processComparison.ts
@@ -2,6 +2,7 @@ import canonicalize from "canonicalize";
import { z } from "zod";
import type { ProcessCapture } from "./processCapture.js";
+import { AnalysisInputError } from "../analysisErrorCore.js";
import { jsonValueSchema } from "../jsonValue.js";
import {
compareProcessTraces,
@@ -14,6 +15,8 @@ import {
traceCoversObservedDimension,
} from "./processTraceDimensionProjection.js";
+const OPERATION = "compare_process_captures";
+
/** Comparison classification that never equates incomplete evidence. */
export const comparisonStatusSchema = z.enum([
"unchanged",
@@ -228,22 +231,60 @@ const assertComparable = (
if (
left.manifest.comparison_contract_sha256 !==
right.manifest.comparison_contract_sha256
- )
- throw new TypeError(
- "Process captures have incompatible comparison contracts",
+ ) {
+ const fields = differingContractFields(
+ left.manifest.comparison_contract,
+ right.manifest.comparison_contract,
);
+ throw new AnalysisInputError(OPERATION, undefined, [
+ {
+ path: ["right"],
+ reason: "invalid_value",
+ message: `Process captures have incompatible comparison contracts${fields.length === 0 ? "" : `; these scenario fields differ: ${fields.join(", ")}`}. Capture both scenarios with the same values for these fields, then compare them.`,
+ expected: fields,
+ },
+ ]);
+ }
if (options.maxCaptureAgeMs === undefined) return;
const maxCaptureAgeMs = options.maxCaptureAgeMs;
const now = (options.now ?? Date.now)();
- if (
- [left, right].some(
- ({ manifest }) =>
- now - Date.parse(manifest.completed_at) > maxCaptureAgeMs,
- )
- )
- throw new TypeError("Process capture exceeds max_capture_age_ms");
+ const stale = (
+ [
+ ["left", left],
+ ["right", right],
+ ] as const
+ ).flatMap(([side, { manifest }]) => {
+ const age = now - Date.parse(manifest.completed_at);
+ return age > maxCaptureAgeMs
+ ? [
+ `${side} completed at ${manifest.completed_at} (${String(age)} ms ago)`,
+ ]
+ : [];
+ });
+ if (stale.length > 0)
+ throw new AnalysisInputError(OPERATION, undefined, [
+ {
+ path: ["max_capture_age_ms"],
+ reason: "out_of_range",
+ message: `Process capture exceeds max_capture_age_ms ${String(maxCaptureAgeMs)}: ${stale.join("; ")}. Recapture the stale scenario or raise max_capture_age_ms.`,
+ },
+ ]);
};
+/** Name top-level contract fields whose canonical values differ. */
+const differingContractFields = (
+ left: Readonly<Record<string, unknown>>,
+ right: Readonly<Record<string, unknown>>,
+): string[] =>
+ [...new Set([...Object.keys(left), ...Object.keys(right)])]
+ .filter(
+ (field) =>
+ !Object.hasOwn(left, field) ||
+ !Object.hasOwn(right, field) ||
+ canonicalize(left[field]) !== canonicalize(right[field]),
+ )
+ .sort();
+
const truncatedComparison = (): ProcessCaptureComparison => ({
status: "truncated",
terminal: "truncated",
diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts
index 3b47d9c42..bff0492cb 100644
--- a/src/server/registerProcessComparisonTool.ts
+++ b/src/server/registerProcessComparisonTool.ts
@@ -2,6 +2,9 @@ import type { McpServer } from "@modelcontextprotocol/server";
import type { BinarySessionPort } from "../application/binary/BinarySession.js";
import { toolContract } from "../contracts/toolContracts.js";
+import { AnalysisInputError } from "../domain/analysisErrorCore.js";
+import { type AnalysisError } from "../domain/analysisErrorBase.js";
+import { describeValidationFailure } from "../domain/evidenceBundle.js";
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
import {
createEvidence,
@@ -13,9 +16,10 @@ import { jsonValueSchema } from "../domain/jsonValue.js";
import {
compareProcessCaptures,
parseProcessCapture,
+ type ProcessCapture,
} from "../domain/process/processCapture.js";
import type { RecordUnknownInput } from "../domain/residualUnknown.js";
-import { err } from "../domain/result.js";
+import { err, ok, type Result } from "../domain/result.js";
import { recordDerivedEvidence } from "./recordDerivedEvidence.js";
import { recordSessionEvidenceSources } from "./sessionEvidence.js";
import { runDerivedOperation } from "./runDerivedOperation.js";
@@ -33,6 +37,57 @@ const sourceLocations = (
right: readonly EvidenceLocation[] | undefined,
): readonly EvidenceLocation[] => [...(left ?? []), ...(right ?? [])];
+/**
+ * Parse one side and name the failed constraint: tampered Evidence, Evidence
+ * from another workflow, or an invalid capture result.
+ */
+const parseCaptureSide = (
+ operation: string,
+ side: "left" | "right",
+ value: unknown,
+): Result<
+ { readonly record: Evidence; readonly capture: ProcessCapture },
+ AnalysisError
+> => {
+ let record: Evidence;
+ try {
+ record = parseEvidence(value);
+ } catch (cause: unknown) {
+ return err(
+ new EvidenceIntegrityError(`${side} Evidence failed validation`, {
+ cause,
+ userMessage: `The ${side} Evidence failed validation (${describeValidationFailure(cause)}). Supply unmodified capture Evidence from capture_process_scenario.`,
+ }),
+ );
+ }
+ if (
+ record.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||
+ record.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate
+ )
+ return err(
+ new AnalysisInputError(operation, undefined, [
+ {
+ path: [side],
+ reason: "invalid_value",
+ message: `Expected process capture Evidence (operation ${PROCESS_CAPTURE_EVIDENCE.operation}, predicate ${PROCESS_CAPTURE_EVIDENCE.predicate}); the ${side} Evidence has operation ${record.operation} and predicate ${record.predicate_type}.`,
+ },
+ ]),
+ );
+ try {
+ return ok({
+ record,
+ capture: parseProcessCapture(record.normalized_result),
+ });
+ } catch (cause: unknown) {
+ return err(
+ new EvidenceIntegrityError(`${side} process capture failed validation`, {
+ cause,
+ userMessage: `The ${side} Evidence has an invalid process capture result (${describeValidationFailure(cause)}). Recreate the capture with capture_process_scenario.`,
+ }),
+ );
+ }
+};
+
/** Register deterministic process-capture comparison and contradiction tracking. */
export const registerProcessComparisonTool = (
server: McpServer,
@@ -44,59 +99,25 @@ export const registerProcessComparisonTool = (
contract.name,
toolRegistrationOptions(contract),
async (input, context) => {
- let leftRecord: Evidence;
- let rightRecord: Evidence;
- try {
- leftRecord = parseEvidence(input.left);
- rightRecord = parseEvidence(input.right);
- if (
- leftRecord.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||
- rightRecord.operation !== PROCESS_CAPTURE_EVIDENCE.operation ||
- leftRecord.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate ||
- rightRecord.predicate_type !== PROCESS_CAPTURE_EVIDENCE.predicate
- )
- throw new TypeError("Expected process capture Evidence");
- } catch (cause: unknown) {
- return toCallToolResult(
- err(
- new EvidenceIntegrityError(
- cause instanceof Error ? cause.message : "Invalid Evidence",
- ),
- ),
- contract,
- );
- }
- let comparison: ReturnType<typeof compareProcessCaptures>;
- let leftCapture: ReturnType<typeof parseProcessCapture>;
- let rightCapture: ReturnType<typeof parseProcessCapture>;
- try {
- leftCapture = parseProcessCapture(leftRecord.normalized_result);
- rightCapture = parseProcessCapture(rightRecord.normalized_result);
- const computed = await runDerivedOperation(context, contract.name, () =>
- compareProcessCaptures(leftCapture, rightCapture, {
- ...(input.max_capture_age_ms === undefined
- ? {}
- : { maxCaptureAgeMs: input.max_capture_age_ms }),
- ...(input.trace_spec === undefined
- ? {}
- : { traceSpecification: input.trace_spec }),
- now,
- }),
- );
- if (!computed.ok) return toCallToolResult(computed, contract);
- comparison = computed.value;
- } catch (cause: unknown) {
- return toCallToolResult(
- err(
- new EvidenceIntegrityError(
- cause instanceof Error
- ? cause.message
- : "Invalid process capture",
- ),
- ),
- contract,
- );
- }
+ const left = parseCaptureSide(contract.name, "left", input.left);
+ if (!left.ok) return toCallToolResult(left, contract);
+ const right = parseCaptureSide(contract.name, "right", input.right);
+ if (!right.ok) return toCallToolResult(right, contract);
+ const { record: leftRecord, capture: leftCapture } = left.value;
+ const { record: rightRecord, capture: rightCapture } = right.value;
+ const computed = await runDerivedOperation(context, contract.name, () =>
+ compareProcessCaptures(leftCapture, rightCapture, {
+ ...(input.max_capture_age_ms === undefined
+ ? {}
+ : { maxCaptureAgeMs: input.max_capture_age_ms }),
+ ...(input.trace_spec === undefined
+ ? {}
+ : { traceSpecification: input.trace_spec }),
+ now,
+ }),
+ );
+ if (!computed.ok) return toCallToolResult(computed, contract);
+ const comparison = computed.value;
const evidence = createEvidence(undefined, PROCESS_PROVIDER, {
predicateType: "rea.process-comparison",
operation: contract.name,
diff --git a/src/server/runDerivedOperation.ts b/src/server/runDerivedOperation.ts
index 1fbdf39ba..0a7a55da6 100644
--- a/src/server/runDerivedOperation.ts
+++ b/src/server/runDerivedOperation.ts
@@ -33,8 +33,10 @@ export const runDerivedOperation = async <Value>(
try {
value = compute();
} catch (cause: unknown) {
+ // Typed causes already name the failed constraint; keep their issues.
return err(
- cause instanceof AnalysisCancelledError
+ cause instanceof AnalysisCancelledError ||
+ cause instanceof AnalysisInputError
? cause
: new AnalysisInputError(operation, { cause }),
);
diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
new file mode 100644
index 000000000..81722a384
--- /dev/null
+++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
@@ -0,0 +1,210 @@
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
+import { expect, it, onTestFinished } from "vitest";
+
+import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
+import { PROCESS_PROVIDER } from "../../../src/application/process/ProcessEvidence.js";
+import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/contracts/process/processCaptureExample.js";
+import { createEvidence } from "../../../src/domain/evidence.js";
+import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
+import { digestProcessCommitment } from "../../../src/domain/process/processCapture.js";
+import { createServer } from "../../../src/server/createServer.js";
+import { createTestBinarySession } from "../../fixtures/binarySession.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const captureEvidence = (
+ side: "left" | "right",
+ comparisonContract: Readonly<Record<string, string>> = {},
+) =>
+ createEvidence(undefined, PROCESS_PROVIDER, {
+ predicateType: "rea.process-capture",
+ operation: "capture_process_scenario",
+ parameters: { side },
+ result: jsonValueSchema.parse({
+ ...EMPTY_PROCESS_CAPTURE_EXAMPLE,
+ manifest: {
+ ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
+ comparison_contract: comparisonContract,
+ comparison_contract_sha256: digestProcessCommitment(comparisonContract),
+ },
+ }),
+ confidence: "observed",
+ authority: "controlled-replay",
+ });
+
+const left = captureEvidence("left", { working_directory: "/work/reference" });
+const right = captureEvidence("right", {
+ working_directory: "/work/reconstruction",
+});
+
+const connect = async () => {
+ const session = createTestBinarySession(() => {
+ throw new Error("Process comparison must not launch a provider");
+ });
+ const server = createServer(session, session);
+ const client = new Client({ name: "process-comparison-test", version: "1" });
+ onTestFinished(async () => {
+ await Promise.allSettled([client.close(), server.close(), session.close()]);
+ });
+ const [clientTransport, serverTransport] =
+ InMemoryTransport.createLinkedPair();
+ await server.connect(serverTransport);
+ await client.connect(clientTransport);
+ return client;
+};
+
+it("names differing comparison contract fields through MCP and the CLI", async () => {
+ const client = await connect();
+ const response = await client.callTool({
+ name: "compare_process_captures",
+ arguments: { left, right },
+ });
+ expect(response.isError).toBe(true);
+ const issue = {
+ path: ["right"],
+ reason: "invalid_value",
+ message: expect.stringContaining(
+ "these scenario fields differ: working_directory",
+ ),
+ expected: ["working_directory"],
+ };
+ expect(response.structuredContent).toMatchObject({
+ error: { code: "invalid_request", details: { issues: [issue] } },
+ });
+
+ const root = await createTestTempDirectory("rea-process-contract-");
+ const leftPath = join(root, "left.json");
+ const rightPath = join(root, "right.json");
+ await writeFile(leftPath, JSON.stringify(left));
+ await writeFile(rightPath, JSON.stringify(right));
+ expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
+ error: "Process command failed",
+ code: "invalid_request",
+ details: { operation: "compare_process_captures", issues: [issue] },
+ });
+});
+
+it("names the stale capture that exceeds max_capture_age_ms", async () => {
+ const client = await connect();
+ const response = await client.callTool({
+ name: "compare_process_captures",
+ arguments: {
+ left: captureEvidence("left"),
+ right: captureEvidence("right"),
+ max_capture_age_ms: 1000,
+ },
+ });
+ expect(response.isError).toBe(true);
+ expect(response.structuredContent).toMatchObject({
+ error: {
+ code: "invalid_request",
+ details: {
+ issues: [
+ {
+ path: ["max_capture_age_ms"],
+ reason: "out_of_range",
+ message: expect.stringContaining(
+ `left completed at ${EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest.completed_at}`,
+ ),
+ },
+ ],
+ },
+ },
+ });
+});
+
+it("reports CLI trace specification issues like the MCP trace_spec input", async () => {
+ const root = await createTestTempDirectory("rea-process-trace-spec-");
+ const capturePath = join(root, "capture.json");
+ const specPath = join(root, "spec.json");
+ await writeFile(capturePath, JSON.stringify(captureEvidence("left")));
+ await writeFile(specPath, JSON.stringify({ language: {} }));
+ expect(
+ await compareProcessEvidenceFiles(capturePath, capturePath, specPath),
+ ).toMatchObject({
+ code: "invalid_request",
+ details: {
+ operation: "compare_process_captures",
+ issues: expect.arrayContaining([
+ expect.objectContaining({ path: ["trace_spec", "events"] }),
+ ]),
+ },
+ });
+});
+
+it("names the side and constraint when Evidence is not a usable capture", async () => {
+ const client = await connect();
+ const capture = captureEvidence("right");
+ const unrelated = createEvidence(undefined, PROCESS_PROVIDER, {
+ operation: "other",
+ parameters: {},
+ result: {},
+ });
+ const wrongKind = await client.callTool({
+ name: "compare_process_captures",
+ arguments: { left: unrelated, right: capture },
+ });
+ expect(wrongKind.structuredContent).toMatchObject({
+ error: {
+ code: "invalid_request",
+ details: {
+ issues: [
+ {
+ path: ["left"],
+ reason: "invalid_value",
+ message: expect.stringContaining(
+ "the left Evidence has operation other",
+ ),
+ },
+ ],
+ },
+ },
+ });
+
+ const tampered = await client.callTool({
+ name: "compare_process_captures",
+ arguments: {
+ left: capture,
+ right: { ...capture, parameters: { side: "changed" } },
+ },
+ });
+ expect(tampered.structuredContent).toMatchObject({
+ error: {
+ code: "evidence_integrity_mismatch",
+ message: expect.stringContaining(
+ "The right Evidence failed validation (Evidence semantic identifier does not match its record)",
+ ),
+ },
+ });
+
+ const invalidResult = createEvidence(undefined, PROCESS_PROVIDER, {
+ predicateType: "rea.process-capture",
+ operation: "capture_process_scenario",
+ parameters: {},
+ result: { manifest: {} },
+ });
+ const malformed = await client.callTool({
+ name: "compare_process_captures",
+ arguments: { left: capture, right: invalidResult },
+ });
+ expect(malformed.structuredContent).toMatchObject({
+ error: {
+ code: "evidence_integrity_mismatch",
+ message: expect.stringContaining(
+ "The right Evidence has an invalid process capture result (at ",
+ ),
+ },
+ });
+
+ const root = await createTestTempDirectory("rea-process-malformed-");
+ const leftPath = join(root, "left.json");
+ const rightPath = join(root, "right.json");
+ await writeFile(leftPath, JSON.stringify(capture));
+ await writeFile(rightPath, JSON.stringify(invalidResult));
+ expect(await compareProcessEvidenceFiles(leftPath, rightPath)).toMatchObject({
+ category: "invalid_input",
+ message: expect.stringContaining("Capture evidence is malformed (at "),
+ });
+});