Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

refactor(process): remove unused paired experiment

morlutoe1d755f7cd57
diff --git a/src/application/ProcessPairedExperiment.test.ts b/src/application/ProcessPairedExperiment.test.ts
deleted file mode 100644
index 03f2da707..000000000
--- a/src/application/ProcessPairedExperiment.test.ts
+++ /dev/null
@@ -1,260 +0,0 @@
-import { describe, expect, it } from "vitest";
-
-import {
- runPairedProcessExperiment,
- type ProcessCapturePort,
-} from "./ProcessPairedExperiment.js";
-import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../domain/processCapture.fixture.js";
-import {
- parseProcessCapture,
- type EnabledProcessExecutionPolicy,
-} from "../domain/processCapture.js";
-import {
- analyzeProcessRepeatability,
- bindProcessScenario,
- processPairedExperimentSchema,
-} from "../domain/processPairedExperiment.js";
-import { ok } from "../domain/result.js";
-
-const experiment = () =>
- processPairedExperimentSchema.parse({
- shared_scenario: {
- executable: "/bin/template",
- arguments: ["shared"],
- working_directory: "/tmp",
- environment: { SHARED: "yes" },
- },
- authority: {
- id: "authority",
- executable: "/bin/authority",
- environment: { SIDE: "authority" },
- },
- candidate: {
- id: "candidate",
- executable: "/bin/candidate",
- arguments: ["candidate"],
- environment: { SIDE: "candidate" },
- },
- repeat_count: 3,
- freshness_policy: { max_capture_age_ms: 60_000 },
- required_dimensions: [
- "terminal",
- "interaction",
- "exit",
- "filesystem",
- "protocol",
- "process",
- "shim",
- ],
- });
-
-const policy: EnabledProcessExecutionPolicy = {
- status: "enabled",
- executableRoots: ["/bin"],
- workingRoots: ["/tmp"],
- allowedEnvironment: ["SHARED", "SIDE"],
- networkAccess: "none",
-};
-
-const cloneCapture = (
- capture: ReturnType<typeof parseProcessCapture>,
-): ReturnType<typeof parseProcessCapture> =>
- parseProcessCapture(structuredClone(capture));
-
-describe("paired process experiments", () => {
- it("binds only declared side inputs onto the shared observation contract", () => {
- const parsed = experiment();
- const authority = bindProcessScenario(
- parsed.shared_scenario,
- parsed.authority,
- );
- const candidate = bindProcessScenario(
- parsed.shared_scenario,
- parsed.candidate,
- );
- expect(authority).toMatchObject({
- executable: "/bin/authority",
- arguments: ["shared"],
- environment: { SHARED: "yes", SIDE: "authority" },
- });
- expect(candidate).toMatchObject({
- executable: "/bin/candidate",
- arguments: ["candidate"],
- environment: { SHARED: "yes", SIDE: "candidate" },
- });
- });
-
- it("runs same-side comparisons first and compares stable sides", async () => {
- const observedExecutables: string[] = [];
- const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
- const capturePort: ProcessCapturePort = {
- capture: (scenario) => {
- observedExecutables.push(scenario.executable);
- return Promise.resolve(ok(cloneCapture(capture)));
- },
- };
- const result = await runPairedProcessExperiment(experiment(), policy, {
- capturePort,
- now: () => Date.parse("2026-01-01T00:00:01.000Z"),
- });
- expect(result.ok).toBe(true);
- if (!result.ok) return;
- expect(observedExecutables).toEqual([
- "/bin/authority",
- "/bin/authority",
- "/bin/authority",
- "/bin/candidate",
- "/bin/candidate",
- "/bin/candidate",
- ]);
- expect(result.value.authority_repeatability.stable).toBe(true);
- expect(result.value.candidate_repeatability.stable).toBe(true);
- expect(result.value.cross_side?.status).toBe("unchanged");
- expect(result.value.cross_side_blocked_reason).toBeNull();
- });
-
- it("blocks cross-side claims when a candidate has a flaky exit", async () => {
- const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
- let candidateRun = 0;
- const capturePort: ProcessCapturePort = {
- capture: (scenario) => {
- let capture = cloneCapture(base);
- if (scenario.executable === "/bin/candidate") {
- candidateRun += 1;
- if (candidateRun === 2)
- capture = parseProcessCapture({
- ...capture,
- exit: { code: 1, signal: null, reason: "exited" },
- });
- }
- return Promise.resolve(ok(capture));
- },
- };
- const result = await runPairedProcessExperiment(experiment(), policy, {
- capturePort,
- now: () => Date.parse("2026-01-01T00:00:01.000Z"),
- });
- expect(result.ok).toBe(true);
- if (!result.ok) return;
- expect(result.value.candidate_repeatability).toMatchObject({
- stable: false,
- classification: "flaky_exit",
- unstable_dimensions: ["exit"],
- });
- expect(result.value.cross_side).toBeNull();
- expect(result.value.cross_side_blocked_reason).toMatch(/unstable/u);
- });
-});
-
-describe("paired process experiment comparisons", () => {
- it("distinguishes normalized timing variance from semantic output drift", () => {
- const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
- const early = parseProcessCapture({
- ...base,
- frames: [{ sequence: 0, at_ms: 10, data: "ready" }],
- });
- const late = parseProcessCapture({
- ...base,
- frames: [{ sequence: 0, at_ms: 20, data: "ready" }],
- });
- expect(
- analyzeProcessRepeatability([early, late], ["terminal"]),
- ).toMatchObject({
- stable: false,
- classification: "timing_variance",
- unstable_dimensions: ["terminal"],
- });
- const changed = parseProcessCapture({
- ...base,
- frames: [{ sequence: 0, at_ms: 20, data: "failed" }],
- });
- expect(
- analyzeProcessRepeatability([early, changed], ["terminal"]),
- ).toMatchObject({ classification: "nondeterministic_output" });
- });
-
- it("stops before capture when the experiment is cancelled", async () => {
- const controller = new AbortController();
- controller.abort();
- const capturePort: ProcessCapturePort = {
- capture: () => {
- throw new Error("capture must not start after cancellation");
- },
- };
-
- const result = await runPairedProcessExperiment(experiment(), policy, {
- capturePort,
- signal: controller.signal,
- });
-
- expect(result).toMatchObject({
- ok: false,
- error: { _tag: "ProcessCaptureError", reason: "cancelled" },
- });
- });
-
- it("blocks full cross-side output when only a subset was repeatable", async () => {
- const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
- const capturePort: ProcessCapturePort = {
- capture: () => Promise.resolve(ok(cloneCapture(capture))),
- };
- const subset = processPairedExperimentSchema.parse({
- ...experiment(),
- required_dimensions: ["exit"],
- });
-
- const result = await runPairedProcessExperiment(subset, policy, {
- capturePort,
- });
-
- expect(result).toMatchObject({
- ok: true,
- value: {
- cross_side: null,
- cross_side_blocked_reason: expect.stringMatching(/every returned/u),
- },
- });
- });
-});
-
-describe("paired process experiment freshness", () => {
- it("compares the newest stable captures under the freshness policy", async () => {
- const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
- let captureIndex = 0;
- const capturePort: ProcessCapturePort = {
- capture: () => {
- const completedAt =
- captureIndex === 0
- ? "2026-01-01T00:00:00.000Z"
- : "2026-01-01T00:00:09.500Z";
- captureIndex += 1;
- return Promise.resolve(
- ok(
- parseProcessCapture({
- ...structuredClone(base),
- manifest: {
- ...base.manifest,
- started_at: completedAt,
- completed_at: completedAt,
- },
- }),
- ),
- );
- },
- };
- const shortFreshness = processPairedExperimentSchema.parse({
- ...experiment(),
- freshness_policy: { max_capture_age_ms: 1_000 },
- });
-
- const result = await runPairedProcessExperiment(shortFreshness, policy, {
- capturePort,
- now: () => Date.parse("2026-01-01T00:00:10.000Z"),
- });
-
- expect(result).toMatchObject({
- ok: true,
- value: { cross_side: { status: "unchanged" } },
- });
- });
-});
diff --git a/src/application/ProcessPairedExperiment.ts b/src/application/ProcessPairedExperiment.ts
deleted file mode 100644
index 3705cec78..000000000
--- a/src/application/ProcessPairedExperiment.ts
+++ /dev/null
@@ -1,152 +0,0 @@
-import {
- processCaptureCancelled,
- type ProcessCaptureError,
-} from "./ProcessCaptureError.js";
-import { captureProcessScenario } from "./ProcessHarness.js";
-import type {
- ProcessCapture,
- ProcessExecutionPolicy,
- ProcessScenario,
-} from "../domain/processCapture.js";
-import { compareProcessCaptures } from "../domain/processComparison.js";
-import { PROCESS_COMPARISON_DIMENSIONS } from "../domain/processComparison.js";
-import {
- analyzeProcessRepeatability,
- bindProcessScenario,
- type ProcessPairedExperiment,
-} from "../domain/processPairedExperiment.js";
-import { err, ok, type Result } from "../domain/result.js";
-
-/** Capture seam used by paired process experiments. */
-export interface ProcessCapturePort {
- capture(
- scenario: ProcessScenario,
- policy: ProcessExecutionPolicy,
- signal?: AbortSignal,
- ): Promise<Result<ProcessCapture, ProcessCaptureError>>;
-}
-
-interface PairedProcessExperimentEvidence {
- readonly authority_runs: readonly ProcessCapture[];
- readonly candidate_runs: readonly ProcessCapture[];
- readonly authority_repeatability: ReturnType<
- typeof analyzeProcessRepeatability
- >;
- readonly candidate_repeatability: ReturnType<
- typeof analyzeProcessRepeatability
- >;
-}
-
-/** Repeatability evidence and either a stable comparison or its blocking reason. */
-export type PairedProcessExperimentResult = PairedProcessExperimentEvidence &
- (
- | {
- readonly cross_side: ReturnType<typeof compareProcessCaptures>;
- readonly cross_side_blocked_reason: null;
- }
- | {
- readonly cross_side: null;
- readonly cross_side_blocked_reason: string;
- }
- );
-
-const productionCapturePort: ProcessCapturePort = {
- capture: captureProcessScenario,
-};
-
-const captureRepeats = async (input: {
- readonly scenario: ProcessScenario;
- readonly repeatCount: number;
- readonly policy: ProcessExecutionPolicy;
- readonly port: ProcessCapturePort;
- readonly signal?: AbortSignal;
-}): Promise<Result<readonly ProcessCapture[], ProcessCaptureError>> => {
- const captures: ProcessCapture[] = [];
- for (let index = 0; index < input.repeatCount; index += 1) {
- if (input.signal?.aborted === true) return err(processCaptureCancelled());
- const capture = await input.port.capture(
- input.scenario,
- input.policy,
- input.signal,
- );
- if (!capture.ok) return err(capture.error);
- captures.push(capture.value);
- }
- return ok(captures);
-};
-
-/** Execute repeatability-first authority/candidate captures under one contract. */
-export const runPairedProcessExperiment = async (
- experiment: ProcessPairedExperiment,
- policy: ProcessExecutionPolicy,
- options: {
- readonly signal?: AbortSignal;
- readonly capturePort?: ProcessCapturePort;
- readonly now?: () => number;
- } = {},
-): Promise<Result<PairedProcessExperimentResult, ProcessCaptureError>> => {
- const port = options.capturePort ?? productionCapturePort;
- const authorityScenario = bindProcessScenario(
- experiment.shared_scenario,
- experiment.authority,
- );
- const candidateScenario = bindProcessScenario(
- experiment.shared_scenario,
- experiment.candidate,
- );
- const authorityRuns = await captureRepeats({
- scenario: authorityScenario,
- repeatCount: experiment.repeat_count,
- policy,
- port,
- ...(options.signal === undefined ? {} : { signal: options.signal }),
- });
- if (!authorityRuns.ok) return authorityRuns;
- const candidateRuns = await captureRepeats({
- scenario: candidateScenario,
- repeatCount: experiment.repeat_count,
- policy,
- port,
- ...(options.signal === undefined ? {} : { signal: options.signal }),
- });
- if (!candidateRuns.ok) return candidateRuns;
- const authorityRepeatability = analyzeProcessRepeatability(
- authorityRuns.value,
- experiment.required_dimensions,
- );
- const candidateRepeatability = analyzeProcessRepeatability(
- candidateRuns.value,
- experiment.required_dimensions,
- );
- const stable = authorityRepeatability.stable && candidateRepeatability.stable;
- const allDimensionsRequired = PROCESS_COMPARISON_DIMENSIONS.every(
- (dimension) => experiment.required_dimensions.includes(dimension),
- );
- const authorityLatest = authorityRuns.value.at(-1);
- const candidateLatest = candidateRuns.value.at(-1);
- const crossSideOutcome =
- stable &&
- allDimensionsRequired &&
- authorityLatest !== undefined &&
- candidateLatest !== undefined
- ? {
- cross_side: compareProcessCaptures(authorityLatest, candidateLatest, {
- maxCaptureAgeMs: experiment.freshness_policy.max_capture_age_ms,
- ...(options.now === undefined ? {} : { now: options.now }),
- }),
- cross_side_blocked_reason: null,
- }
- : {
- cross_side: null,
- cross_side_blocked_reason: !allDimensionsRequired
- ? "Cross-side comparison is blocked until every returned dimension is required and repeatable."
- : "Cross-side comparison is blocked because a required same-side dimension is unstable.",
- };
- return ok({
- authority_runs: authorityRuns.value,
- candidate_runs: candidateRuns.value,
- authority_repeatability: authorityRepeatability,
- candidate_repeatability: candidateRepeatability,
- ...crossSideOutcome,
- });
-};
diff --git a/src/domain/processPairedExperiment.ts b/src/domain/processPairedExperiment.ts
deleted file mode 100644
index 4d71d1308..000000000
--- a/src/domain/processPairedExperiment.ts
+++ /dev/null
@@ -1,205 +0,0 @@
-import { z } from "zod";
-
-import type { ProcessCapture } from "./processCapture.js";
-import {
- PROCESS_COMPARISON_DIMENSIONS,
- compareProcessCaptures,
- processDimensionObservations,
- type ProcessCaptureComparison,
-} from "./processComparison.js";
-import {
- digestProcessCommitment,
- processScenarioSchema,
- type ProcessScenario,
-} from "./processScenario.js";
-
-const sideIdentifierSchema = z
- .string()
- .regex(/^[A-Za-z][A-Za-z0-9._-]{0,127}$/u);
-const environmentNameSchema = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/u);
-const processDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS);
-
-const sideBindingSchema = z
- .object({
- id: sideIdentifierSchema,
- executable: z.string().startsWith("/"),
- arguments: z.array(z.string()).max(256).optional(),
- environment: z.record(environmentNameSchema, z.string()).default({}),
- })
- .strict();
-
-/** Boundary contract for one bounded authority/candidate process experiment. */
-export const processPairedExperimentSchema = z
- .object({
- shared_scenario: processScenarioSchema,
- authority: sideBindingSchema,
- candidate: sideBindingSchema,
- repeat_count: z.number().int().min(2).max(10).default(3),
- freshness_policy: z
- .object({
- max_capture_age_ms: z.number().int().positive().max(86_400_000),
- })
- .strict(),
- required_dimensions: z
- .array(processDimensionSchema)
- .min(1)
- .max(PROCESS_COMPARISON_DIMENSIONS.length)
- .default([...PROCESS_COMPARISON_DIMENSIONS]),
- })
- .strict()
- .superRefine((experiment, context) => {
- if (experiment.authority.id === experiment.candidate.id)
- context.addIssue({
- code: "custom",
- message: "authority and candidate identities must differ",
- path: ["candidate", "id"],
- });
- if (
- new Set(experiment.required_dimensions).size !==
- experiment.required_dimensions.length
- )
- context.addIssue({
- code: "custom",
- message: "required process dimensions must be unique",
- path: ["required_dimensions"],
- });
- });
-
-export type ProcessPairedExperiment = z.infer<
- typeof processPairedExperimentSchema
->;
-/** Executable and environment values applied to one experiment side. */
-export type ProcessSideBinding = ProcessPairedExperiment[
- | "authority"
- | "candidate"];
-
-type ProcessRepeatabilityClassification =
- | "stable"
- | "timing_variance"
- | "nondeterministic_output"
- | "flaky_exit"
- | "environmental_drift"
- | "insufficient_samples";
-
-/** Stability classification for repeated captures of one experiment side. */
-export interface ProcessSideRepeatability {
- readonly classification: ProcessRepeatabilityClassification;
- readonly stable: boolean;
- readonly digest: string;
- readonly comparisons: readonly ProcessCaptureComparison[];
- readonly unstable_dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][];
-}
-
-/** Apply a side binding without changing the shared observation contract. */
-export const bindProcessScenario = (
- shared: ProcessScenario,
- side: ProcessSideBinding,
-): ProcessScenario =>
- processScenarioSchema.parse({
- ...shared,
- executable: side.executable,
- arguments: side.arguments ?? shared.arguments,
- environment: { ...shared.environment, ...side.environment },
- });
-
-const classifyInstability = (
- dimensions: ReadonlySet<(typeof PROCESS_COMPARISON_DIMENSIONS)[number]>,
-): ProcessRepeatabilityClassification => {
- if (dimensions.has("exit")) return "flaky_exit";
- if (dimensions.has("filesystem") || dimensions.has("process"))
- return "environmental_drift";
- return "nondeterministic_output";
-};
-
-const timingKeys = new Set([
- "at_ms",
- "scheduled_at_ms",
- "dispatched_at_ms",
- "elapsed_ms",
-]);
-
-const withoutTiming = (value: unknown): unknown =>
- typeof value !== "object" || value === null || Array.isArray(value)
- ? value
- : Object.fromEntries(
- Object.entries(value).filter(([key]) => !timingKeys.has(key)),
- );
-
-const observationsWithoutTiming = (
- capture: ProcessCapture,
- dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number],
-): readonly unknown[] =>
- dimension === "exit"
- ? [
- {
- ...capture.exit,
- settlement: {
- state: capture.settlement.state,
- cleanup_outcome: capture.settlement.cleanup_outcome,
- },
- },
- ]
- : processDimensionObservations(capture, dimension).map(withoutTiming);
-
-const hasOnlyTimingVariance = (
- captures: readonly ProcessCapture[],
- dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][],
-): boolean =>
- dimensions.every((dimension) => {
- const first = captures[0];
- if (first === undefined) return false;
- const baseline = JSON.stringify(
- observationsWithoutTiming(first, dimension),
- );
- return captures
- .slice(1)
- .every(
- (capture) =>
- JSON.stringify(observationsWithoutTiming(capture, dimension)) ===
- baseline,
- );
- });
-
-/** Compare same-side repeats before permitting a cross-side claim. */
-export const analyzeProcessRepeatability = (
- captures: readonly ProcessCapture[],
- requiredDimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][],
-): ProcessSideRepeatability => {
- if (captures.length < 2)
- return {
- classification: "insufficient_samples",
- stable: false,
- digest: digestProcessCommitment({ captures: [], requiredDimensions }),
- comparisons: [],
- unstable_dimensions: [],
- };
- const baseline = captures[0];
- if (baseline === undefined)
- throw new TypeError("Repeatability analysis lost its baseline capture");
- const comparisons = captures
- .slice(1)
- .map((capture) => compareProcessCaptures(baseline, capture));
- const unstable = new Set(
- requiredDimensions.filter((dimension) =>
- comparisons.some((comparison) => comparison[dimension] !== "unchanged"),
- ),
- );
- const unstableDimensions = PROCESS_COMPARISON_DIMENSIONS.filter((dimension) =>
- unstable.has(dimension),
- );
- return {
- classification:
- unstable.size === 0
- ? "stable"
- : hasOnlyTimingVariance(captures, unstableDimensions)
- ? "timing_variance"
- : classifyInstability(unstable),
- stable: unstable.size === 0,
- digest: digestProcessCommitment({
- requiredDimensions,
- comparisons,
- }),
- comparisons,
- unstable_dimensions: unstableDimensions,
- };
-};