Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process): preserve unrelated trace comparison uncertainty (#4178)

Ysopwnable7b86d5caf63a
diff --git a/docs/process-capture.md b/docs/process-capture.md
index 2bf0d06da..17e69f311 100644
--- a/docs/process-capture.md
+++ b/docs/process-capture.md
@@ -425,7 +425,9 @@ results for unaffected dimensions and mark affected dimensions unknown; an
incomplete capture cannot locate the first divergence across all dimensions.
Trace assertions use coverage for the sources they select, so an assertion
about complete raw terminal output need not fail because rendered snapshots
-were omitted. Captures missing `truncation_details` are rejected on import;
+were omitted. A trace difference applies to the dimensions identified by its
+diagnostic; unrelated dimensions retain their own comparison, including unknown
+coverage. Captures missing `truncation_details` are rejected on import;
preserve them as historical files and recapture for current comparisons.
`limits.file_bytes` is already the cumulative whole-file hashing budget,
independent of `limits.output_bytes`, which bounds retained terminal content.
diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts
index 06e134061..71f3b9952 100644
--- a/src/domain/process/processComparison.ts
+++ b/src/domain/process/processComparison.ts
@@ -330,7 +330,8 @@ const applyTraceVerdict = (
const sources = new Set(specification.events.map(({ source }) => source));
const covered = dimensionsForTraceSources(sources);
// A different verdict names the failing events and observations; attribute
- // it only to their dimensions so equal observations keep their exact status.
+ // it only to their dimensions; unrelated comparisons keep their own status,
+ // including unknown coverage and independently observed additions/removals.
const diagnosticSources = new Set<ProcessObservationSource>([
...specification.events
.filter(({ id }) => trace.diagnostic?.event_ids.includes(id) === true)
@@ -350,8 +351,7 @@ const applyTraceVerdict = (
if (!covered.has(dimension)) return exact;
if (trace.verdict === "nonconforming") return exact;
if (trace.verdict === "different")
- return traceShowsDifference &&
- (exact !== "unchanged" || attributed.has(dimension))
+ return traceShowsDifference && attributed.has(dimension)
? "changed"
: exact;
if (
diff --git a/src/domain/process/processTraceComparison.test.ts b/src/domain/process/processTraceComparison.test.ts
index 5966e4ef1..da3767aed 100644
--- a/src/domain/process/processTraceComparison.test.ts
+++ b/src/domain/process/processTraceComparison.test.ts
@@ -360,6 +360,50 @@ describe("unknown process trace evidence", () => {
});
describe("process trace dimension attribution", () => {
+ it.each([false, true])(
+ "preserves an independently observed terminal addition/removal (reversed: %s)",
+ (reversed) => {
+ const absent = capture({
+ frames: [],
+ process_samples: [],
+ event_journal: [],
+ });
+ const present = capture(values(["terminal", "process"]));
+ const specification = partialSpecification();
+ const result = compareProcessCaptures(
+ reversed ? present : absent,
+ reversed ? absent : present,
+ {
+ traceSpecification: {
+ ...specification,
+ events: specification.events.map((event) =>
+ event.id === "ready"
+ ? { ...event, cardinality: { kind: "optional" } }
+ : event,
+ ),
+ language: {
+ kind: "partial_order",
+ happens_before: [],
+ not_before: [],
+ unordered_groups: [{ events: ["ready", "worker"] }],
+ prefix: [],
+ suffix: [],
+ },
+ },
+ },
+ );
+ expect(result).toMatchObject({
+ terminal: reversed ? "removed" : "added",
+ process: "changed",
+ status: "changed",
+ trace: {
+ verdict: "different",
+ diagnostic: { event_ids: ["worker"], locations: [] },
+ },
+ });
+ },
+ );
+
it("applies passing filesystem trace timestamp omissions to exact comparison", () => {
const base = capture(values(["terminal", "process"]));
const left = base;
diff --git a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
index bda4672de..a67373e0c 100644
--- a/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
+++ b/tests/boundary/mcp/processComparisonDiagnosticsMcp.test.ts
@@ -8,6 +8,17 @@ import { expect, it, onTestFinished } from "vitest";
import { compareProcessEvidenceFiles } from "../../../src/application/process/ProcessCli.js";
import { PROCESS_PROVIDER } from "../../../src/domain/process/processEvidenceProvider.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../../src/domain/process/processCaptureExample.js";
+import {
+ capture,
+ partialSpecification,
+ processStarted,
+ terminal,
+} from "../../../src/domain/process/processTraceComparison.fixture.js";
+import {
+ parseProcessCapture,
+ type ProcessCapture,
+} from "../../../src/domain/process/processCaptureParsing.js";
+import type { ProcessTraceSpecification } from "../../../src/domain/process/processTraceSpecification.js";
import {
createEvidence,
parseEvidence,
@@ -23,15 +34,18 @@ const captureEvidence = (
side: "left" | "right",
comparisonContract: Readonly<Record<string, string>> = {},
provider: EvidenceProvider = PROCESS_PROVIDER,
+ observation: ProcessCapture = parseProcessCapture(
+ EMPTY_PROCESS_CAPTURE_EXAMPLE,
+ ),
) =>
createEvidence(undefined, provider, {
predicateType: "rea.process-capture",
operation: "capture_process_scenario",
parameters: { side },
result: jsonValueSchema.parse({
- ...EMPTY_PROCESS_CAPTURE_EXAMPLE,
+ ...observation,
manifest: {
- ...EMPTY_PROCESS_CAPTURE_EXAMPLE.manifest,
+ ...observation.manifest,
comparison_contract: comparisonContract,
comparison_contract_sha256: digestProcessCommitment(comparisonContract),
},
@@ -64,6 +78,7 @@ const connect = async () => {
const compareViaCli = async (
left: ReturnType<typeof captureEvidence>,
right: ReturnType<typeof captureEvidence>,
+ specification?: ProcessTraceSpecification,
) => {
const root = await createTestTempDirectory("rea-process-provider-");
const leftPath = join(root, "left.json");
@@ -72,9 +87,100 @@ const compareViaCli = async (
writeFile(leftPath, JSON.stringify(left)),
writeFile(rightPath, JSON.stringify(right)),
]);
- return compareProcessEvidenceFiles(leftPath, rightPath);
+ const tracePath = join(root, "trace.json");
+ if (specification !== undefined)
+ await writeFile(tracePath, JSON.stringify(specification));
+ return compareProcessEvidenceFiles(
+ leftPath,
+ rightPath,
+ specification === undefined ? undefined : tracePath,
+ );
};
+it.each(
+ (["left", "right"] as const).flatMap((missingWorker) =>
+ (["left", "right", "both", "neither"] as const).map((omittedRendered) => ({
+ missingWorker,
+ omittedRendered,
+ })),
+ ),
+)(
+ "preserves terminal coverage when $missingWorker lacks a worker and $omittedRendered omits rendered output",
+ async ({ missingWorker, omittedRendered }) => {
+ const observation = (side: "left" | "right") => {
+ const hasWorker = side !== missingWorker;
+ const complete = capture({
+ frames: [terminal],
+ process_samples: hasWorker ? [processStarted] : [],
+ event_journal: [
+ { collection: "frames", index: 0 },
+ ...(hasWorker
+ ? [{ collection: "process_samples" as const, index: 0 }]
+ : []),
+ ],
+ });
+ const omitted = omittedRendered === "both" || omittedRendered === side;
+ return captureEvidence(
+ side,
+ {},
+ PROCESS_PROVIDER,
+ omitted
+ ? parseProcessCapture({
+ ...complete,
+ truncated: true,
+ truncation_details: {
+ ...complete.truncation_details,
+ rendered_terminal: {
+ budget_bytes: 5,
+ observed_bytes: 0,
+ retained_bytes: 0,
+ observed_frames: 1,
+ retained_frames: 0,
+ unknown_size_frames: 1,
+ },
+ },
+ })
+ : complete,
+ );
+ };
+ const left = observation("left");
+ const right = observation("right");
+ const terminalStatus =
+ omittedRendered === "neither" ? "unchanged" : "unknown";
+ const plain = parseEvidence(await compareViaCli(left, right));
+ expect(plain.normalized_result).toMatchObject({
+ terminal: terminalStatus,
+ process: missingWorker === "left" ? "added" : "removed",
+ status: "changed",
+ });
+
+ const specification = partialSpecification();
+ const cli = parseEvidence(await compareViaCli(left, right, specification));
+ const client = await connect();
+ const response = await client.callTool({
+ name: "compare_process_captures",
+ arguments: { left, right, trace_spec: specification },
+ });
+ expect(response.isError).not.toBe(true);
+ expect(parseEvidence(response.structuredContent)).toEqual(cli);
+ expect(cli.evidence_links).toEqual([left.evidence_id, right.evidence_id]);
+ expect(cli.normalized_result).toMatchObject({
+ terminal: terminalStatus,
+ process: "changed",
+ status: "changed",
+ trace: {
+ verdict: "different",
+ diagnostic: {
+ kind: "cardinality",
+ side: missingWorker,
+ event_ids: ["worker"],
+ locations: [],
+ },
+ },
+ });
+ },
+);
+
it.each(
(["left", "right"] as const).flatMap((side) =>
[