Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

fix(process): bound trace comparison semantics

morlutob4df565b779c
diff --git a/docs/process-capture.md b/docs/process-capture.md
index cf6004ce9..58eba6c32 100644
--- a/docs/process-capture.md
+++ b/docs/process-capture.md
@@ -198,6 +198,9 @@ each side's raw journal locations and identifies either the matched finite
variant or the complete satisfied constraint set. A failure returns the first
declaration-ordered predicate, cardinality, edge, prefix, suffix, or language
violation with its relevant event locations.
+If both captures have the same invalid declared trace, the nested verdict is
+`nonconforming` while the top-level capture comparison remains unchanged. Side
+statuses and the diagnostic distinguish conformance from pairwise difference.
The result classifies terminal, interaction, exit, filesystem, protocol,
process, and shim behavior. `first_divergence` points to the earliest observed
diff --git a/docs/product-catalog.json b/docs/product-catalog.json
index db39b51a7..9526ab20e 100644
--- a/docs/product-catalog.json
+++ b/docs/product-catalog.json
@@ -579,10 +579,10 @@
"mcp_resource_templates": 9
},
"digests": {
- "tools_sha256": "cca8f06f796f8a95d2f2e46c0fb4176e0ae442081fe7bf992ee4526071fb0aa5",
+ "tools_sha256": "073b2d48443034a05ba56b649057f9e5f0ab21afcb39be2c380e5a05bc0cecae",
"prompts_sha256": "f6da3ad38a1000c11364d1710f122ac4990551f04efd8c53ffb2e819b5cf5707",
"resources_sha256": "94bebbab1a1b3cf47644c38d3de6469de568caa7bd7b842f529ead8ad320a912",
- "combined_sha256": "3ce063a71e1e53c18094aca45df31840eff75f1eaded7cbd84a7ef48db279817"
+ "combined_sha256": "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f"
}
}
}
diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json
index 7b4bc6cd7..2ea26ef25 100644
--- a/docs/verification/managed-conformance-manifest.json
+++ b/docs/verification/managed-conformance-manifest.json
@@ -1,6 +1,6 @@
{
"schema_version": 1,
- "manifest_id": "ecm_8ecf7db7dddcf68278e18291d8d5a4d352b807a55f8b10cd20ad893b3bf9a2ea",
+ "manifest_id": "ecm_a77e4a40338b12af5108a59c7323c22a0f2367ff2cd925c8c3fb4213b5eba480",
"verifier": {
"id": "rea-managed-conformance",
"version": "1"
@@ -14,7 +14,7 @@
"skill_digests": [
{
"skill_id": "reverse-engineer-anything",
- "sha256": "718b929b39749da6416a45645db6ed4d0775548cfaa1443509066b5c8201015f"
+ "sha256": "6308c7735c8b1f16f58f9d7020102fdf0a2492d3f78d979f4537d49e8e83fdc9"
}
],
"claims": [
diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md
index 268f0c446..04ccbda24 100644
--- a/skills/reverse-engineer-anything/SKILL.md
+++ b/skills/reverse-engineer-anything/SKILL.md
@@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, an
metadata:
version: "23"
tool_count: 97
- catalog_digest: "3ce063a71e1e53c18094aca45df31840eff75f1eaded7cbd84a7ef48db279817"
+ catalog_digest: "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f"
---
# REA
diff --git a/src/domain/processTraceComparison.ts b/src/domain/processTraceComparison.ts
index 1b30bee6a..c545295f3 100644
--- a/src/domain/processTraceComparison.ts
+++ b/src/domain/processTraceComparison.ts
@@ -1,4 +1,5 @@
import type { ProcessCapture } from "./processCapture.js";
+import { validateProcessCapture } from "./processCaptureValidation.js";
import {
evaluateProcessTraceSide,
processTraceComparisonResultSchema,
@@ -27,6 +28,14 @@ export const compareProcessTraces = (
right: ProcessCapture,
specificationInput: ProcessTraceSpecification,
): ProcessTraceComparisonResult => {
+ const captureIssues = [
+ ...validateProcessCapture(left),
+ ...validateProcessCapture(right),
+ ];
+ if (captureIssues.length > 0)
+ throw new TypeError(
+ `Invalid Process Capture v4: ${captureIssues[0]!.path}`,
+ );
const specification =
processTraceSpecificationSchema.parse(specificationInput);
const leftEvaluation = evaluateProcessTraceSide(left, specification);
@@ -37,6 +46,14 @@ export const compareProcessTraces = (
? null
: { side: "right" as const, ...rightEvaluation.diagnostic }
: { side: "left" as const, ...leftEvaluation.diagnostic };
+ const outcomesDiffer =
+ leftEvaluation.result.status !== rightEvaluation.result.status ||
+ JSON.stringify(
+ leftEvaluation.result.raw_trace.map(({ event_id }) => event_id),
+ ) !==
+ JSON.stringify(
+ rightEvaluation.result.raw_trace.map(({ event_id }) => event_id),
+ );
return processTraceComparisonResultSchema.parse({
verdict:
leftEvaluation.result.status === "unknown" ||
@@ -45,7 +62,9 @@ export const compareProcessTraces = (
: leftEvaluation.result.status === "pass" &&
rightEvaluation.result.status === "pass"
? "equivalent"
- : "different",
+ : outcomesDiffer
+ ? "different"
+ : "nonconforming",
left: leftEvaluation.result,
right: rightEvaluation.result,
diagnostic,
diff --git a/src/domain/processTraceEvaluation.ts b/src/domain/processTraceEvaluation.ts
index b181b722a..10539e83b 100644
--- a/src/domain/processTraceEvaluation.ts
+++ b/src/domain/processTraceEvaluation.ts
@@ -73,7 +73,7 @@ const diagnosticSchema = z.strictObject({
/** Structured verdict for one declared process trace language. */
export const processTraceComparisonResultSchema = z.strictObject({
- verdict: z.enum(["equivalent", "different", "unknown"]),
+ verdict: z.enum(["equivalent", "different", "nonconforming", "unknown"]),
left: sideResultSchema,
right: sideResultSchema,
diagnostic: diagnosticSchema.nullable(),
@@ -257,31 +257,41 @@ const matchRecords = (
): MatchedTrace | EvaluatedSide => {
const rawTrace: z.infer<typeof matchedEventSchema>[] = [];
const locationsById = new Map<string, ProcessTraceLocation[]>();
+ const ignoredFieldsBySource = new Map<
+ ProcessTraceSource,
+ readonly string[]
+ >();
+ const eventsByKey = new Map<
+ string,
+ ProcessTraceSpecification["events"][number]
+ >();
+ for (const event of specification.events) {
+ const ignoredFields = event.ignore_fields ?? [];
+ ignoredFieldsBySource.set(event.source, ignoredFields);
+ eventsByKey.set(
+ `${event.source}\0${canonicalTraceJson(
+ comparableTracePayload(event.exact, ignoredFields),
+ )}`,
+ event,
+ );
+ }
for (const record of records) {
- const matches = specification.events.filter(
- (event) =>
- event.source === record.source &&
- canonicalTraceJson(
- comparableTracePayload(event.exact, event.ignore_fields),
- ) ===
- canonicalTraceJson(
- comparableTracePayload(record.payload, event.ignore_fields),
- ),
+ const match = eventsByKey.get(
+ `${record.source}\0${canonicalTraceJson(
+ comparableTracePayload(
+ record.payload,
+ ignoredFieldsBySource.get(record.source),
+ ),
+ )}`,
);
- if (matches.length !== 1)
+ if (match === undefined)
return failure({
kind: "predicate",
- message:
- matches.length === 0
- ? "Observed event does not match any declared exact predicate."
- : "Observed event matches multiple predicates.",
- eventIds: matches.map(({ id }) => id),
+ message: "Observed event does not match any declared exact predicate.",
+ eventIds: [],
locations: [record.location],
rawTrace,
});
- const match = matches[0];
- if (match === undefined)
- throw new TypeError("Unique trace match is missing");
rawTrace.push({ event_id: match.id, location: record.location });
const locations = locationsById.get(match.id) ?? [];
locations.push(record.location);
diff --git a/src/domain/processTraceSpecification.ts b/src/domain/processTraceSpecification.ts
index b597c62dc..b15b8a017 100644
--- a/src/domain/processTraceSpecification.ts
+++ b/src/domain/processTraceSpecification.ts
@@ -197,6 +197,16 @@ const buildAdjacency = (
});
adjacency.get(edge.before)?.add(edge.after);
}
+ return adjacency;
+};
+
+const validateNotBefore = (
+ language: PartialOrderLanguage,
+ context: z.RefinementCtx,
+ known: ReadonlySet<string>,
+ reachable: ReadonlyMap<string, ReadonlySet<string>>,
+): ReadonlySet<string> => {
+ const pairs = new Set<string>();
for (const [index, constraint] of language.not_before.entries()) {
const path = ["language", "not_before", index];
addReferenceIssue(context, known, constraint.event, [...path, "event"]);
@@ -207,9 +217,22 @@ const buildAdjacency = (
message: "not-before constraints must reference distinct events",
path,
});
- adjacency.get(constraint.anchor)?.add(constraint.event);
+ const key = pairKey(constraint.event, constraint.anchor);
+ if (pairs.has(key))
+ context.addIssue({
+ code: "custom",
+ message: "not-before event pairs must be declared once",
+ path,
+ });
+ pairs.add(key);
+ if (reachable.get(constraint.event)?.has(constraint.anchor) === true)
+ context.addIssue({
+ code: "custom",
+ message: "not-before constraint conflicts with happens-before order",
+ path,
+ });
}
- return adjacency;
+ return pairs;
};
const graphHasCycle = (
@@ -257,11 +280,16 @@ const isOrdered = (
reachable.get(left)?.has(right) === true ||
reachable.get(right)?.has(left) === true;
+type OrderingRelations = {
+ readonly reachable: ReadonlyMap<string, ReadonlySet<string>>;
+ readonly notBeforePairs: ReadonlySet<string>;
+};
+
const validateUnorderedGroups = (
language: PartialOrderLanguage,
context: z.RefinementCtx,
known: ReadonlySet<string>,
- reachable: ReadonlyMap<string, ReadonlySet<string>>,
+ relations: OrderingRelations,
): ReadonlySet<string> => {
const pairs = new Set<string>();
for (const [groupIndex, group] of language.unordered_groups.entries()) {
@@ -284,7 +312,10 @@ const validateUnorderedGroups = (
path,
});
pairs.add(key);
- if (isOrdered(reachable, leftId, rightId))
+ if (
+ isOrdered(relations.reachable, leftId, rightId) ||
+ relations.notBeforePairs.has(key)
+ )
context.addIssue({
code: "custom",
message: "ordered events cannot also be declared unordered",
@@ -298,14 +329,15 @@ const validateUnorderedGroups = (
const validateCompletePairDeclarations = (
ids: readonly string[],
- reachable: ReadonlyMap<string, ReadonlySet<string>>,
+ relations: OrderingRelations,
unorderedPairs: ReadonlySet<string>,
context: z.RefinementCtx,
): void => {
for (const [leftIndex, leftId] of ids.entries())
for (const rightId of ids.slice(leftIndex + 1))
if (
- !isOrdered(reachable, leftId, rightId) &&
+ !isOrdered(relations.reachable, leftId, rightId) &&
+ !relations.notBeforePairs.has(pairKey(leftId, rightId)) &&
!unorderedPairs.has(pairKey(leftId, rightId))
)
context.addIssue({
@@ -330,13 +362,20 @@ const validatePartialOrder = (
path: ["language"],
});
const reachable = computeReachability(ids, adjacency);
- const unorderedPairs = validateUnorderedGroups(
+ const notBeforePairs = validateNotBefore(
specification.language,
context,
known,
reachable,
);
- validateCompletePairDeclarations(ids, reachable, unorderedPairs, context);
+ const relations = { reachable, notBeforePairs };
+ const unorderedPairs = validateUnorderedGroups(
+ specification.language,
+ context,
+ known,
+ relations,
+ );
+ validateCompletePairDeclarations(ids, relations, unorderedPairs, context);
for (const [kind, values] of [
["prefix", specification.language.prefix],
["suffix", specification.language.suffix],
diff --git a/tests/processTraceAdapters.test.ts b/tests/processTraceAdapters.test.ts
index 019ea8335..37d1c8176 100644
--- a/tests/processTraceAdapters.test.ts
+++ b/tests/processTraceAdapters.test.ts
@@ -290,7 +290,7 @@ describe("declared trace comparison adapters", () => {
}),
).toMatchObject({
status: "unchanged",
- trace: { verdict: "different" },
+ trace: { verdict: "nonconforming" },
});
});
diff --git a/tests/processTraceComparison.test.ts b/tests/processTraceComparison.test.ts
index 8099e0d78..d4b577555 100644
--- a/tests/processTraceComparison.test.ts
+++ b/tests/processTraceComparison.test.ts
@@ -1,12 +1,18 @@
import { describe, expect, it } from "vitest";
-import type { ProcessCapture } from "../src/domain/processCapture.js";
+import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../src/contracts/processCaptureExample.js";
+import {
+ processCaptureSchema,
+ type ProcessCapture,
+} from "../src/domain/processCapture.js";
import {
compareProcessTraces,
processTraceSpecificationSchema,
type ProcessTraceSpecification,
} from "../src/domain/processTraceComparison.js";
+const emptyCapture = processCaptureSchema.parse(EMPTY_PROCESS_CAPTURE_EXAMPLE);
+
const capture = (
values: Pick<
ProcessCapture,
@@ -23,58 +29,40 @@ const capture = (
readonly residualUnknowns?: ProcessCapture["residual_unknowns"];
} = {},
): ProcessCapture => ({
- schema_version: 4,
- manifest: {
- rea_version: "test",
- provider_version: "3",
- platform: "linux",
- architecture: "x64",
- pty_backend: "node-pty",
- started_at: "2026-01-01T00:00:00.000Z",
- completed_at: "2026-01-01T00:00:01.000Z",
- scenario: {},
- comparison_contract: {},
- shim_plan: [],
- replay_plan: {},
- full_scenario_sha256: "0".repeat(64),
- comparison_contract_sha256: "1".repeat(64),
- executable_sha256: "2".repeat(64),
- normalization_sha256: "3".repeat(64),
- shim_plan_sha256: "4".repeat(64),
- replay_plan_sha256: "5".repeat(64),
- },
- normalization: {
- paths: true,
- pids: true,
- ports: true,
- time_bucket_ms: 10,
- patterns: [],
- },
+ ...emptyCapture,
frames: values.frames,
- rendered_frames: [],
- interaction_events: [],
- exit: { code: 0, signal: null, reason: "exited" },
- settlement: {
- state: "quiesced",
- elapsed_ms: 10,
- cleanup_outcome: "not_required",
- },
process_samples: values.process_samples,
- filesystem_checkpoints: values.filesystem_checkpoints,
+ filesystem_checkpoints: emptyCapture.filesystem_checkpoints,
shim_events: values.shim_events,
protocol_events: values.protocol_events,
- replay_transitions: [],
- event_journal: values.event_journal,
- files_before: [],
- files_after: [],
- filesystem_effects: [],
+ event_journal: [
+ {
+ capture_order: 0,
+ collection: "filesystem_checkpoints",
+ index: 0,
+ },
+ ...values.event_journal.map((entry) => ({
+ ...entry,
+ capture_order: entry.capture_order + 1,
+ })),
+ {
+ capture_order: values.event_journal.length + 1,
+ collection: "lifecycle",
+ index: 0,
+ },
+ {
+ capture_order: values.event_journal.length + 2,
+ collection: "lifecycle",
+ index: 1,
+ },
+ {
+ capture_order: values.event_journal.length + 3,
+ collection: "filesystem_checkpoints",
+ index: 1,
+ },
+ ],
truncated: options.truncated ?? false,
- limitations: [],
residual_unknowns: options.residualUnknowns ?? [],
- cleanup: {
- owned_process_group: "verified",
- temporary_root: "removed",
- },
});
const terminal = { sequence: 0, at_ms: 900, data: "Ready" };
@@ -94,7 +82,7 @@ const http = {
method: "GET",
path: "/status",
data: "",
- outcome: "matched" as const,
+ outcome: "unmatched" as const,
};
const websocket = {
sequence: 1,
@@ -186,6 +174,18 @@ describe("process trace specification", () => {
},
}).success,
).toBe(false);
+ expect(
+ processTraceSpecificationSchema.safeParse({
+ version: 1,
+ events: base.events.slice(0, 3),
+ language: {
+ kind: "partial_order",
+ happens_before: [{ before: "ready", after: "status" }],
+ not_before: [{ event: "worker", anchor: "ready" }],
+ unordered_groups: [],
+ },
+ }).success,
+ ).toBe(false);
expect(
processTraceSpecificationSchema.safeParse({
version: 1,
@@ -315,7 +315,7 @@ describe("process trace comparison", () => {
kind: "edge",
side: "right",
event_ids: ["ready", "worker"],
- locations: [{ capture_order: 1 }, { capture_order: 0 }],
+ locations: [{ capture_order: 2 }, { capture_order: 1 }],
},
});
});
@@ -417,6 +417,80 @@ describe("process trace comparison", () => {
});
});
+ it("enforces exact and range cardinality for one repeated predicate", () => {
+ const repeated = (count: number) =>
+ capture({
+ frames: Array.from({ length: count }, (_, sequence) => ({
+ sequence,
+ at_ms: sequence,
+ data: "tick",
+ })),
+ process_samples: [],
+ filesystem_checkpoints: [],
+ protocol_events: [],
+ shim_events: [],
+ event_journal: Array.from({ length: count }, (_, index) => ({
+ capture_order: index,
+ collection: "frames" as const,
+ index,
+ })),
+ });
+ const event: Omit<
+ ProcessTraceSpecification["events"][number],
+ "cardinality"
+ > = {
+ id: "tick",
+ source: "terminal_raw",
+ exact: { data: "tick" },
+ ignore_fields: ["sequence", "at_ms"],
+ };
+ const rangeSpecification: ProcessTraceSpecification = {
+ version: 1,
+ events: [{ ...event, cardinality: { kind: "range", min: 2, max: 3 } }],
+ language: {
+ kind: "partial_order",
+ happens_before: [],
+ not_before: [],
+ unordered_groups: [],
+ prefix: [],
+ suffix: [],
+ },
+ };
+ expect(
+ compareProcessTraces(repeated(2), repeated(3), rangeSpecification),
+ ).toMatchObject({ verdict: "equivalent" });
+ expect(
+ compareProcessTraces(repeated(1), repeated(2), rangeSpecification),
+ ).toMatchObject({
+ verdict: "different",
+ diagnostic: { kind: "cardinality", side: "left" },
+ });
+ expect(
+ compareProcessTraces(repeated(2), repeated(4), rangeSpecification),
+ ).toMatchObject({
+ verdict: "different",
+ diagnostic: { kind: "cardinality", side: "right" },
+ });
+
+ const exactSpecification: ProcessTraceSpecification = {
+ version: 1,
+ events: [{ ...event, cardinality: { kind: "exact", count: 2 } }],
+ language: {
+ kind: "finite_traces",
+ variants: [{ id: "two", trace: ["tick", "tick"] }],
+ },
+ };
+ expect(
+ compareProcessTraces(repeated(2), repeated(2), exactSpecification),
+ ).toMatchObject({ verdict: "equivalent" });
+ expect(
+ compareProcessTraces(repeated(3), repeated(2), exactSpecification),
+ ).toMatchObject({
+ verdict: "different",
+ diagnostic: { kind: "cardinality", side: "left" },
+ });
+ });
+
it("never proves equivalence from truncated, unknown, or journal-free evidence", () => {
const complete = capture(
values(["terminal", "process", "http", "websocket"]),
@@ -437,7 +511,7 @@ describe("process trace comparison", () => {
compareProcessTraces(complete, unknown, partialSpecification()).verdict,
).toBe("unknown");
- const noJournal = capture({ ...values([]), event_journal: [] });
+ const noJournal = { ...complete, event_journal: [] };
const comparison = compareProcessTraces(
complete,
noJournal,
@@ -447,5 +521,13 @@ describe("process trace comparison", () => {
verdict: "unknown",
diagnostic: { kind: "journal", side: "right" },
});
+
+ const incompleteJournal = {
+ ...complete,
+ event_journal: (complete.event_journal ?? []).slice(1),
+ };
+ expect(() =>
+ compareProcessTraces(complete, incompleteJournal, partialSpecification()),
+ ).toThrow("Invalid Process Capture v4: event_journal.0.capture_order");
});
});