Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
fix(process): attribute finite trace omissions and cleanup states
morlutofc7e41ff3b1d
diff --git a/src/domain/process/processTraceComparison.test.ts b/src/domain/process/processTraceComparison.test.tsindex 07113bfad..5966e4ef1 100644--- a/src/domain/process/processTraceComparison.test.ts+++ b/src/domain/process/processTraceComparison.test.ts@@ -1,10 +1,12 @@import { describe, expect, it } from "vitest";+import { jsonValueSchema } from "../jsonValue.js";import { parseProcessCapture } from "./processCaptureParsing.js";import type { ProcessTraceSpecification } from "./processTraceSpecification.js";import { processTraceSpecificationSchema as specificationSchema } from "./processTraceSpecification.js";import { processTraceComparisonResultSchema } from "./processTraceEvaluation.js";import {capture,+ emptyCapture,partialSpecification,processStarted,terminal,@@ -13,6 +15,35 @@ import {import { compareProcessCaptures } from "./processComparison.js";import { compareProcessTraces } from "./processTraceComparison.js";++const traced = (+ left: ReturnType<typeof capture>,+ right: ReturnType<typeof capture>,+) =>+ compareProcessCaptures(left, right, {+ traceSpecification: partialSpecification(),+ });++const withPayloads = (data: string, command: string) =>+ capture({+ ...values(["terminal", "process"]),+ frames: [{ ...terminal, data }],+ process_samples: [{ ...processStarted, command }],+ });++type FiniteTraceLanguage = Extract<+ ProcessTraceSpecification["language"],+ { kind: "finite_traces" }+>;++const finiteTraceSpecification = (+ events: ProcessTraceSpecification["events"],+ variants: FiniteTraceLanguage["variants"],+): ProcessTraceSpecification => ({+ events,+ language: { kind: "finite_traces", variants },+});+describe("generic capture trace comparison", () => {it("requires explicit partial ordering and never infers causality from timestamps", () => {const specification = partialSpecification();@@ -73,8 +104,8 @@ describe("generic capture trace comparison", () => {{ collection: "frames", index: 1 },],});- const specification: ProcessTraceSpecification = {- events: [+ const specification = finiteTraceSpecification(+ [{id: "tick1",source: "terminal_raw",@@ -88,14 +119,11 @@ describe("generic capture trace comparison", () => {cardinality: { kind: "optional" },},],- language: {- kind: "finite_traces",- variants: [- { id: "one", trace: ["tick1"] },- { id: "two", trace: ["tick1", "tick2"] },- ],- },- };+ [+ { id: "one", trace: ["tick1"] },+ { id: "two", trace: ["tick1", "tick2"] },+ ],+ );expect(compareProcessTraces(one, two, specification)).toMatchObject({verdict: "equivalent",left: { matched_variant: "one" },@@ -108,8 +136,8 @@ describe("generic capture trace comparison", () => {// echoes those identifiers verbatim, so both must accept the same values// rather than handing the caller a raw schema error for its own input.const eventId = `e${"0".repeat(64)}`;- const specification: ProcessTraceSpecification = {- events: [+ const specification = finiteTraceSpecification(+ [{id: eventId,source: "terminal_raw",@@ -117,11 +145,8 @@ describe("generic capture trace comparison", () => {cardinality: { kind: "required" },},],- language: {- kind: "finite_traces",- variants: [{ id: "only", trace: [eventId] }],- },- };+ [{ id: "only", trace: [eventId] }],+ );expect(specificationSchema.safeParse(specification).success).toBe(true);const left = capture(values(["terminal", "process"]));expect(compareProcessTraces(left, left, specification)).toMatchObject({@@ -131,6 +156,53 @@ describe("generic capture trace comparison", () => {});});+describe("finite trace variant mismatch attribution", () => {+ it("reports tied best-prefix omissions across variants", () => {+ const observed = capture({+ frames: [terminal],+ process_samples: [],+ event_journal: [{ collection: "frames", index: 0 }],+ });+ const specification = finiteTraceSpecification(+ [+ {+ id: "ready",+ source: "terminal_raw",+ exact: terminal,+ cardinality: { kind: "required" },+ },+ {+ id: "worker",+ source: "process",+ exact: processStarted,+ cardinality: { kind: "optional" },+ },+ {+ id: "worker_alt",+ source: "process",+ exact: { ...processStarted, command: "alternate-worker" },+ cardinality: { kind: "optional" },+ },+ ],+ [+ { id: "ready_worker", trace: ["ready", "worker"] },+ { id: "ready_worker_alt", trace: ["ready", "worker_alt"] },+ ],+ );++ expect(+ compareProcessTraces(observed, observed, specification),+ ).toMatchObject({+ verdict: "nonconforming",+ diagnostic: {+ kind: "trace",+ event_ids: ["worker", "worker_alt"],+ locations: [],+ },+ });+ });+});+describe("process trace constraints and evidence sufficiency", () => {it("enforces exact and range cardinality for a repeated terminal observation", () => {const repeated = (count: number) =>@@ -288,38 +360,255 @@ describe("unknown process trace evidence", () => {});describe("process trace dimension attribution", () => {- const traced = (- left: ReturnType<typeof capture>,- right: ReturnType<typeof capture>,- ) =>- compareProcessCaptures(left, right, {- traceSpecification: partialSpecification(),- });- const make = (data: string, command: string) =>- capture({- ...values(["terminal", "process"]),- frames: [{ ...terminal, data }],- process_samples: [{ ...processStarted, command }],+ it("applies passing filesystem trace timestamp omissions to exact comparison", () => {+ const base = capture(values(["terminal", "process"]));+ const left = base;+ const right = parseProcessCapture({+ ...base,+ filesystem_checkpoints: base.filesystem_checkpoints.map((checkpoint) =>+ checkpoint.name === "after_settlement"+ ? { ...checkpoint, at_ms: checkpoint.at_ms + 10 }+ : checkpoint,+ ),+ });+ const specification = finiteTraceSpecification(+ emptyCapture.filesystem_checkpoints.map(+ ({ at_ms: _atMs, ...checkpoint }) => ({+ id: checkpoint.name,+ source: "filesystem" as const,+ exact: jsonValueSchema.parse(checkpoint),+ ignore_fields: ["at_ms"],+ cardinality: { kind: "required" as const },+ }),+ ),+ [{ id: "checkpoints", trace: ["before", "after_settlement"] }],+ );++ expect(compareProcessCaptures(left, right)).toMatchObject({+ filesystem: "changed",+ first_divergence: { status: "found", dimension: "filesystem" },+ });+ expect(+ compareProcessCaptures(left, right, {+ traceSpecification: specification,+ }),+ ).toMatchObject({+ trace: { verdict: "equivalent" },+ filesystem: "unchanged",+ first_divergence: { status: "none" },+ });++ const finalFile = {+ path: "root_0:created.txt",+ type: "file" as const,+ mode: 0o644,+ size: 4,+ sha256: "a".repeat(64),+ symlink_target: null,+ };+ const created = {+ path: finalFile.path,+ status: "created" as const,+ before: null,+ after: finalFile,+ };+ const changedRight = parseProcessCapture({+ ...right,+ filesystem_checkpoints: right.filesystem_checkpoints.map((checkpoint) =>+ checkpoint.name === "after_settlement"+ ? { ...checkpoint, files: [finalFile], effects: [created] }+ : checkpoint,+ ),+ files_after: [finalFile],+ filesystem_effects: [created],+ truncation_details: {+ ...right.truncation_details,+ filesystem_after: {+ ...right.truncation_details.filesystem_after,+ hashed_bytes: finalFile.size,+ },+ },+ });+ expect(+ compareProcessCaptures(left, changedRight, {+ traceSpecification: specification,+ }),+ ).toMatchObject({+ trace: { verdict: "different" },+ filesystem: "changed",+ first_divergence: { status: "found", dimension: "filesystem" },});+ });it("keeps equal dimensions unchanged when one side fails a predicate", () => {expect(- traced(make("Ready", "worker"), make("Different", "worker")),+ traced(+ withPayloads("Ready", "worker"),+ withPayloads("Different", "worker"),+ ),).toMatchObject({trace: { verdict: "different" },terminal: "changed",process: "unchanged",});expect(- traced(make("Ready", "worker"), make("Ready", "other-worker")),+ traced(+ withPayloads("Ready", "worker"),+ withPayloads("Ready", "other-worker"),+ ),).toMatchObject({trace: { verdict: "different" },terminal: "unchanged",process: "changed",});});+});++describe("finite trace dimension attribution", () => {+ it("attributes a finite-trace omission to the omitted event dimension", () => {+ const left = capture({+ frames: [terminal],+ process_samples: [],+ event_journal: [{ collection: "frames", index: 0 }],+ });+ const right = capture(values(["terminal", "process"]));+ const specification = finiteTraceSpecification(+ [+ {+ id: "ready",+ source: "terminal_raw",+ exact: terminal,+ cardinality: { kind: "required" },+ },+ {+ id: "worker",+ source: "process",+ exact: processStarted,+ cardinality: { kind: "optional" },+ },+ ],+ [{ id: "ready_worker", trace: ["ready", "worker"] }],+ );++ expect(+ compareProcessCaptures(left, right, {+ traceSpecification: specification,+ }),+ ).toMatchObject({+ status: "changed",+ process: "changed",+ terminal: "unchanged",+ trace: {+ verdict: "different",+ diagnostic: { kind: "trace", event_ids: ["worker"], locations: [] },+ },+ first_divergence: {+ status: "found",+ dimension: "process",+ index: 0,+ },+ });+ });+});++describe("unordered finite trace mismatch attribution", () => {+ it("attributes only the missing member of an unordered group", () => {+ const db = { sequence: 1, at_ms: 901, data: "DB" };+ const alert = { ...processStarted, pid: 2, command: "other-worker" };+ const left = capture({+ frames: [terminal],+ process_samples: [processStarted],+ event_journal: [+ { collection: "frames", index: 0 },+ { collection: "process_samples", index: 0 },+ ],+ });+ const right = capture({+ frames: [terminal, db],+ process_samples: [processStarted],+ event_journal: [+ { collection: "frames", index: 0 },+ { collection: "process_samples", index: 0 },+ { collection: "frames", index: 1 },+ ],+ });+ const specification = finiteTraceSpecification(+ [+ {+ id: "ready",+ source: "terminal_raw",+ exact: terminal,+ cardinality: { kind: "required" },+ },+ {+ id: "worker",+ source: "process",+ exact: processStarted,+ cardinality: { kind: "optional" },+ },+ {+ id: "db",+ source: "terminal_raw",+ exact: db,+ cardinality: { kind: "optional" },+ },+ {+ id: "alert",+ source: "process",+ exact: alert,+ cardinality: { kind: "optional" },+ },+ ],+ [+ {+ id: "ready_then_work",+ trace: ["ready", { unordered: ["worker", "db"] }],+ },+ ],+ );++ expect(+ compareProcessCaptures(left, right, {+ traceSpecification: specification,+ }),+ ).toMatchObject({+ trace: {+ verdict: "different",+ diagnostic: {+ side: "left",+ event_ids: ["db"],+ locations: [],+ },+ },+ terminal: "changed",+ process: "unchanged",+ first_divergence: { status: "found", dimension: "terminal", index: 1 },+ });++ const unexpected = capture({+ frames: [terminal],+ process_samples: [processStarted, alert],+ event_journal: [+ { collection: "frames", index: 0 },+ { collection: "process_samples", index: 0 },+ { collection: "process_samples", index: 1 },+ ],+ });+ expect(+ compareProcessTraces(unexpected, unexpected, specification),+ ).toMatchObject({+ verdict: "nonconforming",+ diagnostic: {+ kind: "trace",+ event_ids: ["db", "alert"],+ locations: [{ collection: "process_samples", index: 1 }],+ },+ });+ });+});- it("still attributes an ordering difference to both ordered dimensions", () => {+describe("partial-order dimension attribution", () => {+ it("attributes an ordering difference to both ordered dimensions", () => {expect(traced(capture(values(["terminal", "process"])),diff --git a/src/domain/process/processTraceDimensionProjection.ts b/src/domain/process/processTraceDimensionProjection.tsindex 8e8208d0a..ed909acd8 100644--- a/src/domain/process/processTraceDimensionProjection.ts+++ b/src/domain/process/processTraceDimensionProjection.ts@@ -46,7 +46,7 @@ export const traceCoversObservedDimension = (case "exit":return sources.has("lifecycle");case "filesystem":- return false;+ return sources.has("filesystem");case "process":return sources.has("process");}diff --git a/src/domain/process/processTraceEvaluation.ts b/src/domain/process/processTraceEvaluation.tsindex 4cf42d8b7..cf80a791d 100644--- a/src/domain/process/processTraceEvaluation.ts+++ b/src/domain/process/processTraceEvaluation.ts@@ -123,29 +123,92 @@ const failure = (input: FailureInput): EvaluatedSide => ({},});+type FiniteTraceEvaluation =+ | { readonly status: "pass"; readonly variant: string }+ | {+ readonly status: "fail";+ readonly eventIds: readonly string[];+ readonly locationIndexes: readonly number[];+ };++/** Match variants once, retaining the best-prefix mismatch for attribution. */const evaluateFiniteTrace = (ids: readonly string[],- specification: ProcessTraceSpecification,-): string | null => {- if (specification.language.kind !== "finite_traces") return null;- for (const variant of specification.language.variants) {+ language: Extract<+ ProcessTraceSpecification["language"],+ { kind: "finite_traces" }+ >,+): FiniteTraceEvaluation => {+ let bestPrefix = -1;+ const bestEventIds = new Set<string>();+ const bestLocationIndexes = new Set<number>();++ const recordMismatch = (+ offset: number,+ missing: readonly string[],+ unexpected: readonly { readonly id: string; readonly index: number }[],+ ): void => {+ if (offset > bestPrefix) {+ bestPrefix = offset;+ bestEventIds.clear();+ bestLocationIndexes.clear();+ }+ if (offset !== bestPrefix) return;+ for (const eventId of missing) bestEventIds.add(eventId);+ for (const { id, index } of unexpected) {+ bestEventIds.add(id);+ bestLocationIndexes.add(index);+ }+ };++ for (const variant of language.variants) {let offset = 0;- let accepted = true;+ let mismatched = false;for (const token of variant.trace) {- if (typeof token === "string") {- if (ids[offset] !== token) accepted = false;- offset += 1;- continue;+ const expected = typeof token === "string" ? [token] : token.unordered;+ const actual = ids.slice(offset, offset + expected.length);+ const remaining = new Map<string, number>();+ for (const eventId of expected)+ remaining.set(eventId, (remaining.get(eventId) ?? 0) + 1);+ const unexpected: Array<{ readonly id: string; readonly index: number }> =+ [];+ for (const [index, eventId] of actual.entries()) {+ const count = remaining.get(eventId) ?? 0;+ if (count === 0)+ unexpected.push({ id: eventId, index: offset + index });+ else if (count === 1) remaining.delete(eventId);+ else remaining.set(eventId, count - 1);}- const actual = ids.slice(offset, offset + token.unordered.length).sort();- const expected = [...token.unordered].sort();- if (canonicalTraceJson(actual) !== canonicalTraceJson(expected))- accepted = false;- offset += token.unordered.length;+ const missing = [...remaining].flatMap(([eventId, count]) =>+ Array.from({ length: count }, () => eventId),+ );+ if (+ missing.length > 0 ||+ unexpected.length > 0 ||+ actual.length < expected.length+ ) {+ recordMismatch(offset, missing, unexpected);+ mismatched = true;+ break;+ }+ offset += expected.length;}- if (accepted && offset === ids.length) return variant.id;++ if (mismatched) continue;+ if (offset === ids.length) return { status: "pass", variant: variant.id };+ const unexpected = ids[offset];+ recordMismatch(+ offset,+ [],+ unexpected === undefined ? [] : [{ id: unexpected, index: offset }],+ );}- return null;++ return {+ status: "fail",+ eventIds: [...bestEventIds],+ locationIndexes: [...bestLocationIndexes],+ };};type MatchedTrace = {@@ -391,20 +454,23 @@ const evaluateMatchedTrace = (if (specification.language.kind === "partial_order")return evaluatePartialOrder(trace, specification.language);const ids = trace.rawTrace.map(({ event_id }) => event_id);- const variant = evaluateFiniteTrace(ids, specification);- return variant === null+ const evaluation = evaluateFiniteTrace(ids, specification.language);+ return evaluation.status === "fail"? failure({kind: "trace",message: "Observed events do not match any finite trace variant.",- eventIds: ids,- locations: trace.rawTrace.map(({ location }) => location),+ eventIds: evaluation.eventIds,+ locations: evaluation.locationIndexes.flatMap((index) => {+ const location = trace.rawTrace[index]?.location;+ return location === undefined ? [] : [location];+ }),rawTrace: trace.rawTrace,}): {result: {status: "pass",- matched_variant: variant,- satisfied_constraints: [`variant:${variant}`],+ matched_variant: evaluation.variant,+ satisfied_constraints: [`variant:${evaluation.variant}`],raw_trace: trace.rawTrace,},diagnostic: null,diff --git a/src/process/capture/ProcessCaptureProgress.test.ts b/src/process/capture/ProcessCaptureProgress.test.tsindex 69349bef9..d3cc98b2c 100644--- a/src/process/capture/ProcessCaptureProgress.test.ts+++ b/src/process/capture/ProcessCaptureProgress.test.ts@@ -110,4 +110,31 @@ it("still delivers terminal status after an observer rejects an update", async (completed: 2,terminal: true,});+ expect(delivered.at(-1)?.message).toContain(+ "owned_process_group=unverified terminal_renderer=unverified temporary_root=unverified",+ );+});++it("preserves known cleanup states in terminal progress", async () => {+ const delivered: Parameters<ProcessCaptureProgress["report"]>[0][] = [];+ const tracker = createProcessCaptureProgressTracker({+ async report(update) {+ delivered.push(update);+ },+ });+ await tracker.report({+ phase: "cleanup",+ counts: { frames: 0, samples: 0, interactions: 0 },+ disposition: "failed",+ terminal: true,+ cleanup: {+ owned_process_group: { state: "not_required", reason: null },+ terminal_renderer: { state: "not_required", reason: null },+ temporary_root: { state: "cleaned", reason: null },+ },+ });++ expect(delivered[0]?.message).toContain(+ "owned_process_group=not_required terminal_renderer=not_required temporary_root=cleaned",+ );});diff --git a/src/process/capture/ProcessCaptureProgress.ts b/src/process/capture/ProcessCaptureProgress.tsindex 205334db7..5cbe34cdc 100644--- a/src/process/capture/ProcessCaptureProgress.ts+++ b/src/process/capture/ProcessCaptureProgress.ts@@ -53,9 +53,11 @@ const formatProgressMessage = (const dispositionText =disposition === undefined ? "" : ` disposition=${disposition}`;const cleanupText =- cleanup === undefined- ? ""- : ` owned_process_group=${cleanup.owned_process_group.state} terminal_renderer=${cleanup.terminal_renderer.state} temporary_root=${cleanup.temporary_root.state}`;+ cleanup !== undefined+ ? ` owned_process_group=${cleanup.owned_process_group.state} terminal_renderer=${cleanup.terminal_renderer.state} temporary_root=${cleanup.temporary_root.state}`+ : disposition === undefined+ ? ""+ : " owned_process_group=unverified terminal_renderer=unverified temporary_root=unverified";return `${status}${dispositionText}${cleanupText}`;};diff --git a/src/process/capture/ProcessHarness.ts b/src/process/capture/ProcessHarness.tsindex ef3c84408..f411718aa 100644--- a/src/process/capture/ProcessHarness.ts+++ b/src/process/capture/ProcessHarness.ts@@ -672,6 +672,10 @@ const runProcessScenario = async (counts: progressCounts(),terminal: true,disposition: signal?.aborted === true ? "cancelled" : "failed",+ ...(cause instanceof ProcessCaptureError &&+ cause.cleanupReport !== undefined+ ? { cleanup: cause.cleanupReport }+ : {}),});throw cause;}diff --git a/tests/boundary/process/processCaptureProgress.test.ts b/tests/boundary/process/processCaptureProgress.test.tsindex 3ffaafd12..c30d3cce6 100644--- a/tests/boundary/process/processCaptureProgress.test.ts+++ b/tests/boundary/process/processCaptureProgress.test.ts@@ -99,12 +99,20 @@ const writeScenario = async (const runCaptureCli = async (scenarioPath: string,- options: { readonly stopOnRunning?: boolean } = {},+ options: {+ readonly stopOnRunning?: boolean;+ readonly env?: NodeJS.ProcessEnv;+ } = {},): Promise<CliCapture> => {const child = spawn(process.execPath,["scripts/rea.mjs", "capture-process", scenarioPath, "--json"],- { stdio: ["ignore", "pipe", "pipe"] },+ {+ stdio: ["ignore", "pipe", "pipe"],+ ...(options.env === undefined+ ? {}+ : { env: { ...process.env, ...options.env } }),+ },);let stdout = "";let stderr = "";@@ -274,6 +282,44 @@ itWithCaptureCapability(30_000,);+itWithCaptureCapability(+ "reports unknown resource cleanup when temporary-root preparation fails",+ async () => {+ const scenario = await writeScenario(+ "rea-process-progress-prepare-failed-",+ {+ executable: process.execPath,+ arguments: childScript("process.stdout.write('ready\\n')"),+ timeout_ms: 5_000,+ },+ );+ const missingTempRoot = join(+ await createTestTempDirectory("rea-process-progress-missing-tmpdir-"),+ "missing",+ );+ const capture = await runCaptureCli(scenario, {+ env: {+ TMPDIR: missingTempRoot,+ TMP: missingTempRoot,+ TEMP: missingTempRoot,+ },+ });++ expect(capture.exitCode).toBe(1);+ expect(capture.lines.at(-1)?.update).toMatchObject({+ phase: "cleanup",+ terminal: true,+ details: {+ disposition: "failed",+ owned_process_group: "unverified",+ terminal_renderer: "unverified",+ temporary_root: "unverified",+ },+ });+ },+ 30_000,+);+itWithCaptureCapability("bounds stderr progress while a busy child emits many output chunks",async () => {