Skip to content

armgit / rea

Public Imported from GitHub
0

Reverse engineer anything with agents, from app behavior down to native binaries.

feat(conformance): add portable conformance export package format

morluto4ee30e4145db
diff --git a/src/domain/conformancePackage.ts b/src/domain/conformancePackage.ts
new file mode 100644
index 000000000..2c347d484
--- /dev/null
+++ b/src/domain/conformancePackage.ts
@@ -0,0 +1,218 @@
+import { createHash } from "node:crypto";
+
+import canonicalize from "canonicalize";
+import { z } from "zod";
+
+import { evidenceEnvelopeSchema } from "./evidence.js";
+import { evidenceBundleSchema } from "./evidenceBundle.js";
+
+/** Schema version for the conformance package format. */
+export const CONFORMANCE_PACKAGE_VERSION = 1;
+
+/** A stable, path-independent identifier for a conformance package. */
+export const conformancePackageIdSchema = z
+ .string()
+ .regex(/^cp_[a-f0-9]{64}$/u);
+
+/** Identifier for a single scenario within a package. */
+export const scenarioIdSchema = z
+ .string()
+ .regex(/^[A-Za-z][A-Za-z0-9._-]{0,99}$/u);
+
+/** Deterministic manifest for a scenario fixture. */
+export const scenarioManifestSchema = z.strictObject({
+ scenario_id: scenarioIdSchema,
+ name: z.string().min(1),
+ description: z.string().min(1),
+ /** Source-owned fixture path relative to the repository root. */
+ fixture_path: z.string().min(1),
+ /** Expected exit code or null if unspecified. */
+ expected_exit_code: z.number().int().nullable(),
+ /** Expected output patterns that must appear in the capture. */
+ expected_patterns: z.array(z.string()).default([]),
+});
+
+/** Replay plan describing how a scenario is replayed deterministically. */
+export const replayPlanSchema = z.strictObject({
+ scenario_id: scenarioIdSchema,
+ /** Ordered steps to execute during replay. */
+ steps: z
+ .array(
+ z.strictObject({
+ step_id: z.string().min(1),
+ action: z.string().min(1),
+ arguments: z.array(z.string()).default([]),
+ timeout_ms: z.number().int().positive(),
+ }),
+ )
+ .min(1)
+ .max(100),
+ /** Environment variables to set (never inherit host paths). */
+ environment: z.record(z.string(), z.string()).default({}),
+});
+
+/** Shim plan for intercepting observable effects. */
+export const shimPlanSchema = z.strictObject({
+ scenario_id: scenarioIdSchema,
+ /** Shims to install, each intercepting a named effect. */
+ shims: z
+ .array(
+ z.strictObject({
+ shim_id: z.string().min(1),
+ kind: z.enum(["filesystem", "network", "process", "signal"]),
+ target: z.string().min(1),
+ policy: z.enum(["observe", "allow", "block", "emulate"]),
+ }),
+ )
+ .min(0)
+ .max(50),
+});
+
+/** Expected evidence for a scenario, checked against the captured run. */
+export const expectedEvidenceSchema = z.strictObject({
+ scenario_id: scenarioIdSchema,
+ /** Expected evidence envelopes. */
+ envelopes: z.array(evidenceEnvelopeSchema).min(0).max(100),
+ /** Expected evidence bundle. */
+ bundle: evidenceBundleSchema.nullable(),
+ /** Required dimensions that must be present in the evidence. */
+ required_dimensions: z.array(z.string().min(1)).default([]),
+});
+
+/** Verifier contract describing how to validate conformance. */
+export const verifierContractSchema = z.strictObject({
+ scenario_id: scenarioIdSchema,
+ /** Dimensions to verify. */
+ dimensions: z
+ .array(
+ z.strictObject({
+ name: z.string().min(1),
+ required: z.boolean().default(true),
+ comparison: z.enum(["exact", "semantic", "fuzzy"]),
+ }),
+ )
+ .min(1)
+ .max(50),
+ /** Tolerance for timing differences in milliseconds. */
+ timing_tolerance_ms: z.number().int().nonnegative().default(0),
+});
+
+/** Top-level conformance package manifest. */
+export const conformancePackageSchema = z.strictObject({
+ schema_version: z.literal(CONFORMANCE_PACKAGE_VERSION),
+ package_id: conformancePackageIdSchema,
+ name: z.string().min(1),
+ description: z.string().min(1),
+ created_at: z.string().datetime(),
+ /** Scenario manifests. */
+ scenarios: z.array(scenarioManifestSchema).min(1).max(100),
+ /** Replay plans keyed by scenario_id. */
+ replay_plans: z.array(replayPlanSchema).min(1).max(100),
+ /** Shim plans keyed by scenario_id. */
+ shim_plans: z.array(shimPlanSchema).default([]),
+ /** Expected evidence for each scenario. */
+ expected_evidence: z.array(expectedEvidenceSchema).min(1).max(100),
+ /** Verifier contracts for each scenario. */
+ verifier_contracts: z.array(verifierContractSchema).min(1).max(100),
+});
+
+export type ConformancePackage = z.infer<typeof conformancePackageSchema>;
+export type ScenarioManifest = z.infer<typeof scenarioManifestSchema>;
+export type ReplayPlan = z.infer<typeof replayPlanSchema>;
+export type ShimPlan = z.infer<typeof shimPlanSchema>;
+export type ExpectedEvidence = z.infer<typeof expectedEvidenceSchema>;
+export type VerifierContract = z.infer<typeof verifierContractSchema>;
+
+/** Error from validation or package operations. */
+export type ConformancePackageError =
+ | { kind: "invalid_package"; message: string }
+ | { kind: "scenario_not_found"; scenario_id: string }
+ | { kind: "duplicate_scenario"; scenario_id: string };
+
+/** Result type for package operations. */
+export type ConformancePackageResult<T> =
+ | { ok: true; value: T }
+ | { ok: false; error: ConformancePackageError };
+
+/** Validate a conformance package and cross-check internal references. */
+export function validateConformancePackage(
+ input: unknown,
+): ConformancePackageResult<ConformancePackage> {
+ const parsed = conformancePackageSchema.safeParse(input);
+ if (!parsed.success)
+ return {
+ ok: false,
+ error: {
+ kind: "invalid_package",
+ message: parsed.error.issues[0]?.message ?? "invalid package",
+ },
+ };
+
+ const pkg = parsed.data;
+
+ // Check for duplicate scenario IDs
+ const seen = new Set<string>();
+ for (const s of pkg.scenarios) {
+ if (seen.has(s.scenario_id))
+ return {
+ ok: false,
+ error: { kind: "duplicate_scenario", scenario_id: s.scenario_id },
+ };
+ seen.add(s.scenario_id);
+ }
+
+ // Check that replay plans reference existing scenarios
+ for (const rp of pkg.replay_plans) {
+ if (!seen.has(rp.scenario_id))
+ return {
+ ok: false,
+ error: {
+ kind: "scenario_not_found",
+ scenario_id: rp.scenario_id,
+ },
+ };
+ }
+
+ // Check that expected evidence references existing scenarios
+ for (const ee of pkg.expected_evidence) {
+ if (!seen.has(ee.scenario_id))
+ return {
+ ok: false,
+ error: {
+ kind: "scenario_not_found",
+ scenario_id: ee.scenario_id,
+ },
+ };
+ }
+
+ // Check that verifier contracts reference existing scenarios
+ for (const vc of pkg.verifier_contracts) {
+ if (!seen.has(vc.scenario_id))
+ return {
+ ok: false,
+ error: {
+ kind: "scenario_not_found",
+ scenario_id: vc.scenario_id,
+ },
+ };
+ }
+
+ return { ok: true, value: pkg };
+}
+
+/** Compute a deterministic package ID from the canonical JSON. */
+export function computePackageId(
+ pkg: Omit<ConformancePackage, "package_id">,
+): string {
+ const json = canonicalize(pkg);
+ if (!json) throw new Error("failed to canonicalize package");
+ return `cp_${createHash("sha256").update(json).digest("hex")}`;
+}
+
+/** Create a conformance package with an auto-computed package_id. */
+export function createConformancePackage(
+ pkg: Omit<ConformancePackage, "package_id">,
+): ConformancePackage {
+ const package_id = computePackageId(pkg);
+ return { package_id, ...pkg };
+}
diff --git a/src/domain/conformanceReplay.ts b/src/domain/conformanceReplay.ts
new file mode 100644
index 000000000..2e0640e17
--- /dev/null
+++ b/src/domain/conformanceReplay.ts
@@ -0,0 +1,138 @@
+import { z } from "zod";
+
+import {
+ conformancePackageSchema,
+ type ConformancePackage,
+} from "./conformancePackage.js";
+import { trustGateResultSchema } from "./conformanceTrustGate.js";
+
+/** Result of a single scenario replay. */
+export const scenarioReplayResultSchema = z.strictObject({
+ scenario_id: z.string().min(1),
+ status: z.enum(["pass", "fail", "error", "skipped"]),
+ exit_code: z.number().int().nullable(),
+ duration_ms: z.number().int().nonnegative(),
+ output: z.string().default(""),
+ error: z.string().nullable(),
+});
+
+/** Result of replaying an entire conformance package. */
+export const packageReplayResultSchema = z.strictObject({
+ package_id: z.string().min(1),
+ total_scenarios: z.number().int().positive(),
+ passed: z.number().int().nonnegative(),
+ failed: z.number().int().nonnegative(),
+ errored: z.number().int().nonnegative(),
+ skipped: z.number().int().nonnegative(),
+ scenario_results: z.array(scenarioReplayResultSchema),
+ trust_gate_results: z.array(trustGateResultSchema),
+ drift_detected: z.boolean(),
+ first_drift: z
+ .strictObject({
+ scenario_id: z.string().min(1),
+ dimension: z.string().min(1),
+ message: z.string(),
+ })
+ .nullable(),
+});
+
+export type ScenarioReplayResult = z.infer<typeof scenarioReplayResultSchema>;
+export type PackageReplayResult = z.infer<typeof packageReplayResultSchema>;
+
+/** Callback interface for replaying a scenario. */
+export interface ScenarioRunner {
+ (scenarioId: string, fixturePath: string): Promise<ScenarioReplayResult>;
+}
+
+/**
+ * Default no-op scenario runner that skips all scenarios.
+ * Real CI replays inject an actual runner.
+ */
+async function defaultRunner(): Promise<ScenarioReplayResult> {
+ return {
+ scenario_id: "unknown",
+ status: "skipped",
+ exit_code: null,
+ duration_ms: 0,
+ output: "",
+ error: "no runner provided",
+ };
+}
+
+/**
+ * Replay a conformance package from a clean checkout.
+ * Runs each scenario, evaluates trust gates, and detects drift
+ * between the captured run and the committed package.
+ */
+export async function replayConformancePackage(
+ packageInput: unknown,
+ actualEvidence: Record<string, Record<string, unknown>>,
+ runner: ScenarioRunner = defaultRunner,
+ options: { truncated?: boolean } = {},
+): Promise<PackageReplayResult> {
+ const parsed = conformancePackageSchema.safeParse(packageInput);
+ if (!parsed.success)
+ throw new TypeError(
+ `invalid conformance package: ${parsed.error.issues[0]?.message ?? "unknown"}`,
+ );
+ const pkg: ConformancePackage = parsed.data;
+
+ const scenarioResults: ScenarioReplayResult[] = [];
+ let passed = 0;
+ let failed = 0;
+ let errored = 0;
+ let skipped = 0;
+
+ for (const scenario of pkg.scenarios) {
+ const result = await runner(scenario.scenario_id, scenario.fixture_path);
+ scenarioResults.push(result);
+
+ switch (result.status) {
+ case "pass":
+ passed++;
+ break;
+ case "fail":
+ failed++;
+ break;
+ case "error":
+ errored++;
+ break;
+ case "skipped":
+ skipped++;
+ break;
+ }
+ }
+
+ const trustGateResults = evaluatePackageTrustGates(
+ pkg,
+ actualEvidence,
+ options,
+ );
+
+ const driftGates = trustGateResults.filter((r) => r.verdict === "fail");
+ const drift_detected = driftGates.length > 0;
+ const first_drift_gate = driftGates[0];
+ const first_drift = first_drift_gate?.first_divergence
+ ? {
+ scenario_id: first_drift_gate.scenario_id,
+ dimension: first_drift_gate.first_divergence.dimension,
+ message: first_drift_gate.first_divergence.message,
+ }
+ : null;
+
+ return {
+ package_id: pkg.package_id,
+ total_scenarios: pkg.scenarios.length,
+ passed,
+ failed,
+ errored,
+ skipped,
+ scenario_results: scenarioResults,
+ trust_gate_results: trustGateResults,
+ drift_detected,
+ first_drift,
+ };
+}
+
+/** Import trust gate evaluator for replay. */
+import { evaluatePackageTrustGates } from "./conformanceTrustGate.js";
diff --git a/src/domain/conformanceTrustGate.ts b/src/domain/conformanceTrustGate.ts
new file mode 100644
index 000000000..485768400
--- /dev/null
+++ b/src/domain/conformanceTrustGate.ts
@@ -0,0 +1,246 @@
+import { z } from "zod";
+
+import type {
+ ConformancePackage,
+ VerifierContract,
+} from "./conformancePackage.js";
+
+/** Result of comparing a single dimension. */
+export const dimensionResultSchema = z.strictObject({
+ name: z.string().min(1),
+ status: z.enum(["match", "mismatch", "unknown", "truncated"]),
+ message: z.string().default(""),
+ evidence_ids: z.array(z.string()).default([]),
+});
+
+/** Result of evaluating a trust gate for one scenario. */
+export const trustGateResultSchema = z.strictObject({
+ scenario_id: z.string().min(1),
+ verdict: z.enum(["pass", "fail", "unknown"]),
+ dimension_results: z.array(dimensionResultSchema),
+ first_divergence: z
+ .strictObject({
+ dimension: z.string().min(1),
+ evidence_id: z.string().min(1),
+ message: z.string(),
+ })
+ .nullable(),
+});
+
+export type DimensionResult = z.infer<typeof dimensionResultSchema>;
+export type TrustGateResult = z.infer<typeof trustGateResultSchema>;
+
+/** Dimension that is volatile due to timing or normalization. */
+const VOLATILE_DIMENSIONS = new Set([
+ "timing",
+ "timestamp",
+ "pid",
+ "ppid",
+ "duration_ms",
+ "created_at",
+ "updated_at",
+]);
+
+/** Dimension that carries semantic content. */
+const SEMANTIC_DIMENSIONS = new Set([
+ "exit_code",
+ "stdout",
+ "stderr",
+ "filesystem",
+ "process_tree",
+ "shim_events",
+ "protocol_events",
+ "event_journal",
+]);
+
+/** Check if a dimension name is volatile (timing/normalization noise). */
+export function isVolatileDimension(name: string): boolean {
+ return VOLATILE_DIMENSIONS.has(name);
+}
+
+/** Check if a dimension name carries semantic content. */
+export function isSemanticDimension(name: string): boolean {
+ return SEMANTIC_DIMENSIONS.has(name);
+}
+
+/**
+ * Compare two values for a dimension, classifying semantic diffs
+ * from timing/normalization noise.
+ *
+ * Volatile dimensions (timing, timestamps, PIDs) are always treated
+ * as matches because their values are expected to differ across
+ * runs and do not carry semantic meaning.
+ */
+export function compareDimension(
+ dimensionName: string,
+ expected: unknown,
+ actual: unknown,
+ options: { truncated?: boolean } = {},
+): DimensionResult {
+ // Volatile dimensions: always match regardless of values
+ if (isVolatileDimension(dimensionName)) {
+ return {
+ name: dimensionName,
+ status: "match",
+ message: "volatile dimension ignored",
+ evidence_ids: [],
+ };
+ }
+
+ if (options.truncated) {
+ return {
+ name: dimensionName,
+ status: "truncated",
+ message: "evidence was truncated",
+ evidence_ids: [],
+ };
+ }
+
+ if (expected === undefined && actual === undefined) {
+ return {
+ name: dimensionName,
+ status: "unknown",
+ message: "both expected and actual are undefined",
+ evidence_ids: [],
+ };
+ }
+
+ if (expected === undefined || actual === undefined) {
+ return {
+ name: dimensionName,
+ status: "mismatch",
+ message: `expected ${expected === undefined ? "present" : "absent"}, actual ${actual === undefined ? "absent" : "present"}`,
+ evidence_ids: [],
+ };
+ }
+
+ // Semantic comparison for structured objects
+ if (typeof expected === "object" && typeof actual === "object") {
+ const expectedJson = JSON.stringify(sortKeys(expected));
+ const actualJson = JSON.stringify(sortKeys(actual));
+ if (expectedJson === actualJson) {
+ return {
+ name: dimensionName,
+ status: "match",
+ message: "",
+ evidence_ids: [],
+ };
+ }
+ return {
+ name: dimensionName,
+ status: "mismatch",
+ message: "semantic content differs",
+ evidence_ids: [],
+ };
+ }
+
+ // Primitive comparison
+ if (expected === actual) {
+ return {
+ name: dimensionName,
+ status: "match",
+ message: "",
+ evidence_ids: [],
+ };
+ }
+
+ return {
+ name: dimensionName,
+ status: "mismatch",
+ message: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,
+ evidence_ids: [],
+ };
+}
+
+/** Recursively sort object keys for deterministic comparison. */
+function sortKeys(value: unknown): unknown {
+ if (value === null || typeof value !== "object") return value;
+ if (Array.isArray(value)) return value.map(sortKeys);
+ const sorted: Record<string, unknown> = {};
+ for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+ sorted[key] = sortKeys((value as Record<string, unknown>)[key]);
+ }
+ return sorted;
+}
+
+/**
+ * Evaluate a trust gate for one scenario by comparing expected
+ * evidence against the actual captured run.
+ */
+export function evaluateTrustGate(
+ contract: VerifierContract,
+ expectedEvidence: Record<string, unknown>,
+ actualEvidence: Record<string, unknown>,
+ options: { truncated?: boolean } = {},
+): TrustGateResult {
+ const dimensionResults: DimensionResult[] = [];
+ let firstDivergence: TrustGateResult["first_divergence"] = null;
+
+ for (const dim of contract.dimensions) {
+ if (!dim.required) continue;
+ const expectedValue = expectedEvidence[dim.name];
+ const actualValue = actualEvidence[dim.name];
+ const result = compareDimension(
+ dim.name,
+ expectedValue,
+ actualValue,
+ options,
+ );
+ dimensionResults.push(result);
+
+ if (result.status === "mismatch" && firstDivergence === null) {
+ firstDivergence = {
+ dimension: dim.name,
+ evidence_id: result.evidence_ids[0] ?? "unknown",
+ message: result.message,
+ };
+ }
+ }
+
+ const hasFail = dimensionResults.some(
+ (r) => r.status === "mismatch" || r.status === "truncated",
+ );
+ const hasUnknown = dimensionResults.some((r) => r.status === "unknown");
+
+ const verdict: TrustGateResult["verdict"] = hasFail
+ ? "fail"
+ : hasUnknown
+ ? "unknown"
+ : "pass";
+
+ return {
+ scenario_id: contract.scenario_id,
+ verdict,
+ dimension_results: dimensionResults,
+ first_divergence: firstDivergence,
+ };
+}
+
+/**
+ * Evaluate trust gates for all scenarios in a conformance package.
+ * Rejects runs that differ in required dimensions. Unknown/truncated
+ * evidence is never treated as equivalence.
+ */
+export function evaluatePackageTrustGates(
+ pkg: ConformancePackage,
+ actualEvidence: Record<string, Record<string, unknown>>,
+ options: { truncated?: boolean } = {},
+): TrustGateResult[] {
+ const results: TrustGateResult[] = [];
+ for (const contract of pkg.verifier_contracts) {
+ const expected = pkg.expected_evidence.find(
+ (e) => e.scenario_id === contract.scenario_id,
+ );
+ if (!expected) continue;
+ const expectedEvidence = expected.envelopes[0] ?? {};
+ const actual = actualEvidence[contract.scenario_id] ?? {};
+ const result = evaluateTrustGate(
+ contract,
+ expectedEvidence as Record<string, unknown>,
+ actual,
+ options,
+ );
+ results.push(result);
+ }
+ return results;
+}
diff --git a/tests/conformancePackage.test.ts b/tests/conformancePackage.test.ts
new file mode 100644
index 000000000..283f6e806
--- /dev/null
+++ b/tests/conformancePackage.test.ts
@@ -0,0 +1,181 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ CONFORMANCE_PACKAGE_VERSION,
+ conformancePackageSchema,
+ computePackageId,
+ createConformancePackage,
+ validateConformancePackage,
+ type ScenarioManifest,
+ type ReplayPlan,
+ type ShimPlan,
+ type ExpectedEvidence,
+} from "../src/domain/conformancePackage.js";
+
+const validPackage = createConformancePackage({
+ schema_version: CONFORMANCE_PACKAGE_VERSION,
+ name: "test-fixture",
+ description: "A test conformance fixture",
+ created_at: "2026-07-28T00:00:00Z",
+ scenarios: [
+ {
+ scenario_id: "s1",
+ name: "Simple spawn",
+ description: "A simple process spawn scenario",
+ fixture_path: "tests/conformance/c/fixture.c",
+ expected_exit_code: 0,
+ expected_patterns: ["Hello", "World"],
+ },
+ ],
+ replay_plans: [
+ {
+ scenario_id: "s1",
+ steps: [
+ {
+ step_id: "step1",
+ action: "compile",
+ arguments: ["gcc", "fixture.c"],
+ timeout_ms: 5000,
+ },
+ ],
+ environment: {},
+ },
+ ],
+ shim_plans: [],
+ expected_evidence: [
+ {
+ scenario_id: "s1",
+ envelopes: [],
+ bundle: null,
+ required_dimensions: ["exit_code", "stdout"],
+ },
+ ],
+ verifier_contracts: [
+ {
+ scenario_id: "s1",
+ dimensions: [
+ {
+ name: "exit_code",
+ required: true,
+ comparison: "exact",
+ },
+ ],
+ timing_tolerance_ms: 100,
+ },
+ ],
+});
+
+describe("conformance package format", () => {
+ it("validates a well-formed package", () => {
+ const result = validateConformancePackage(validPackage);
+ expect(result.ok).toBe(true);
+ });
+
+ it("rejects invalid schema version", () => {
+ const result = validateConformancePackage({
+ ...validPackage,
+ schema_version: 999,
+ });
+ expect(result.ok).toBe(false);
+ });
+
+ it("rejects duplicate scenario IDs", () => {
+ const result = validateConformancePackage({
+ ...validPackage,
+ scenarios: [validPackage.scenarios[0], validPackage.scenarios[0]],
+ });
+ expect(result.ok).toBe(false);
+ });
+
+ it("rejects replay plan referencing non-existent scenario", () => {
+ const result = validateConformancePackage({
+ ...validPackage,
+ replay_plans: [
+ {
+ scenario_id: "nonexistent",
+ steps: [
+ {
+ step_id: "step1",
+ action: "run",
+ arguments: [],
+ timeout_ms: 1000,
+ },
+ ],
+ environment: {},
+ },
+ ],
+ });
+ expect(result.ok).toBe(false);
+ });
+
+ it("rejects verifier contract referencing non-existent scenario", () => {
+ const result = validateConformancePackage({
+ ...validPackage,
+ verifier_contracts: [
+ {
+ scenario_id: "nonexistent",
+ dimensions: [
+ { name: "exit_code", required: true, comparison: "exact" },
+ ],
+ timing_tolerance_ms: 0,
+ },
+ ],
+ });
+ expect(result.ok).toBe(false);
+ });
+
+ it("rejects expected evidence referencing non-existent scenario", () => {
+ const result = validateConformancePackage({
+ ...validPackage,
+ expected_evidence: [
+ {
+ scenario_id: "nonexistent",
+ envelopes: [],
+ bundle: null,
+ required_dimensions: [],
+ },
+ ],
+ });
+ expect(result.ok).toBe(false);
+ });
+
+ it("computes a deterministic package ID", () => {
+ const id = computePackageId({
+ ...validPackage,
+ });
+ expect(id).toMatch(/^cp_[a-f0-9]{64}$/u);
+ // Same input yields same ID
+ const id2 = computePackageId({
+ ...validPackage,
+ });
+ expect(id).toBe(id2);
+ });
+
+ it("produces different IDs for different packages", () => {
+ const id1 = computePackageId({ ...validPackage, name: "packageA" });
+ const id2 = computePackageId({ ...validPackage, name: "packageB" });
+ expect(id1).not.toBe(id2);
+ });
+
+ it("uses all exported types", () => {
+ const scenario: ScenarioManifest = validPackage.scenarios[0]!;
+ expect(scenario.scenario_id).toBe("s1");
+
+ const replay: ReplayPlan = validPackage.replay_plans[0]!;
+ expect(replay.scenario_id).toBe("s1");
+
+ const shim: ShimPlan = validPackage.shim_plans[0] ?? {
+ scenario_id: "s1",
+ shims: [],
+ };
+ expect(shim.scenario_id).toBe("s1");
+
+ const evidence: ExpectedEvidence = validPackage.expected_evidence[0]!;
+ expect(evidence.scenario_id).toBe("s1");
+ });
+
+ it("parses via zod schema", () => {
+ const result = conformancePackageSchema.safeParse(validPackage);
+ expect(result.success).toBe(true);
+ });
+});
diff --git a/tests/conformanceReplay.test.ts b/tests/conformanceReplay.test.ts
new file mode 100644
index 000000000..078997b84
--- /dev/null
+++ b/tests/conformanceReplay.test.ts
@@ -0,0 +1,190 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ CONFORMANCE_PACKAGE_VERSION,
+ createConformancePackage,
+ type ConformancePackage,
+} from "../src/domain/conformancePackage.js";
+import {
+ replayConformancePackage,
+ type ScenarioReplayResult,
+} from "../src/domain/conformanceReplay.js";
+
+const validPackage = createConformancePackage({
+ schema_version: CONFORMANCE_PACKAGE_VERSION,
+ name: "test-fixture",
+ description: "A test conformance fixture",
+ created_at: "2026-07-28T00:00:00Z",
+ scenarios: [
+ {
+ scenario_id: "s1",
+ name: "Simple spawn",
+ description: "A simple process spawn scenario",
+ fixture_path: "tests/conformance/c/fixture.c",
+ expected_exit_code: 0,
+ expected_patterns: [],
+ },
+ ],
+ replay_plans: [
+ {
+ scenario_id: "s1",
+ steps: [
+ {
+ step_id: "step1",
+ action: "run",
+ arguments: [],
+ timeout_ms: 1000,
+ },
+ ],
+ environment: {},
+ },
+ ],
+ shim_plans: [],
+ expected_evidence: [
+ {
+ scenario_id: "s1",
+ envelopes: [],
+ bundle: null,
+ required_dimensions: [],
+ },
+ ],
+ verifier_contracts: [
+ {
+ scenario_id: "s1",
+ dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],
+ timing_tolerance_ms: 0,
+ },
+ ],
+});
+
+async function passingRunner(
+ scenarioId: string,
+): Promise<ScenarioReplayResult> {
+ return {
+ scenario_id: scenarioId,
+ status: "pass",
+ exit_code: 0,
+ duration_ms: 10,
+ output: "ok",
+ error: null,
+ };
+}
+
+async function failingRunner(
+ scenarioId: string,
+): Promise<ScenarioReplayResult> {
+ return {
+ scenario_id: scenarioId,
+ status: "fail",
+ exit_code: 1,
+ duration_ms: 10,
+ output: "fail",
+ error: null,
+ };
+}
+
+describe("conformance CI replay", () => {
+ it("replays a package with passing scenarios", async () => {
+ const result = await replayConformancePackage(
+ validPackage,
+ {},
+ passingRunner,
+ );
+ expect(result.total_scenarios).toBe(1);
+ expect(result.passed).toBe(1);
+ expect(result.failed).toBe(0);
+ expect(result.errored).toBe(0);
+ });
+
+ it("replays a package with failing scenarios", async () => {
+ const result = await replayConformancePackage(
+ validPackage,
+ {},
+ failingRunner,
+ );
+ expect(result.failed).toBe(1);
+ expect(result.passed).toBe(0);
+ });
+
+ it("does not detect drift when expected and actual match", async () => {
+ const result = await replayConformancePackage(
+ validPackage,
+ {},
+ passingRunner,
+ );
+ expect(result.drift_detected).toBe(false);
+ });
+
+ it("throws on invalid package", async () => {
+ await expect(replayConformancePackage({}, {})).rejects.toThrow(
+ /invalid conformance package/u,
+ );
+ });
+
+ it("counts all result types", async () => {
+ const base: ConformancePackage = validPackage;
+ const multiPackage = createConformancePackage({
+ ...base,
+ scenarios: [
+ base.scenarios[0]!,
+ {
+ scenario_id: "s2",
+ name: "Second scenario",
+ description: "Second",
+ fixture_path: "tests/conformance/c/fixture2.c",
+ expected_exit_code: 1,
+ expected_patterns: [],
+ },
+ ],
+ replay_plans: [
+ base.replay_plans[0]!,
+ {
+ scenario_id: "s2",
+ steps: [
+ {
+ step_id: "step1",
+ action: "run",
+ arguments: [],
+ timeout_ms: 1000,
+ },
+ ],
+ environment: {},
+ },
+ ],
+ expected_evidence: [
+ base.expected_evidence[0]!,
+ {
+ scenario_id: "s2",
+ envelopes: [],
+ bundle: null,
+ required_dimensions: [],
+ },
+ ],
+ verifier_contracts: [
+ base.verifier_contracts[0]!,
+ {
+ scenario_id: "s2",
+ dimensions: [
+ { name: "exit_code", required: true, comparison: "exact" },
+ ],
+ timing_tolerance_ms: 0,
+ },
+ ],
+ });
+ const result = await replayConformancePackage(
+ multiPackage,
+ {},
+ async (scenarioId) => ({
+ scenario_id: scenarioId,
+ status: scenarioId === "s1" ? "pass" : "fail",
+ exit_code: scenarioId === "s1" ? 0 : 1,
+ duration_ms: 10,
+ output: "",
+ error: null,
+ }),
+ );
+ expect(result.total_scenarios).toBe(2);
+ expect(result.passed).toBe(1);
+ expect(result.failed).toBe(1);
+ });
+});
diff --git a/tests/conformanceTrustGate.test.ts b/tests/conformanceTrustGate.test.ts
new file mode 100644
index 000000000..0e03a9f07
--- /dev/null
+++ b/tests/conformanceTrustGate.test.ts
@@ -0,0 +1,113 @@
+import { describe, expect, it } from "vitest";
+
+import type { VerifierContract } from "../src/domain/conformancePackage.js";
+import {
+ compareDimension,
+ evaluateTrustGate,
+ isSemanticDimension,
+ isVolatileDimension,
+} from "../src/domain/conformanceTrustGate.js";
+
+const contract: VerifierContract = {
+ scenario_id: "s1",
+ dimensions: [
+ { name: "exit_code", required: true, comparison: "exact" },
+ { name: "stdout", required: true, comparison: "semantic" },
+ { name: "timing", required: true, comparison: "fuzzy" },
+ ],
+ timing_tolerance_ms: 100,
+};
+
+describe("conformance trust gates", () => {
+ it("classifies volatile dimensions correctly", () => {
+ expect(isVolatileDimension("timing")).toBe(true);
+ expect(isVolatileDimension("timestamp")).toBe(true);
+ expect(isVolatileDimension("pid")).toBe(true);
+ expect(isVolatileDimension("exit_code")).toBe(false);
+ });
+
+ it("classifies semantic dimensions correctly", () => {
+ expect(isSemanticDimension("exit_code")).toBe(true);
+ expect(isSemanticDimension("stdout")).toBe(true);
+ expect(isSemanticDimension("timing")).toBe(false);
+ });
+
+ it("matches identical values", () => {
+ const result = compareDimension("exit_code", 0, 0);
+ expect(result.status).toBe("match");
+ });
+
+ it("detects mismatches", () => {
+ const result = compareDimension("exit_code", 0, 1);
+ expect(result.status).toBe("mismatch");
+ });
+
+ it("ignores volatile dimensions", () => {
+ const result = compareDimension("timing", 100, 200);
+ expect(result.status).toBe("match");
+ expect(result.message).toContain("volatile");
+ });
+
+ it("handles truncated evidence", () => {
+ const result = compareDimension("exit_code", 0, null, {
+ truncated: true,
+ });
+ expect(result.status).toBe("truncated");
+ });
+
+ it("handles undefined values", () => {
+ const result = compareDimension("exit_code", undefined, undefined);
+ expect(result.status).toBe("unknown");
+ });
+
+ it("handles missing actual", () => {
+ const result = compareDimension("exit_code", 0, undefined);
+ expect(result.status).toBe("mismatch");
+ });
+
+ it("passes when all dimensions match", () => {
+ const result = evaluateTrustGate(
+ contract,
+ { exit_code: 0, stdout: "hello" },
+ { exit_code: 0, stdout: "hello" },
+ );
+ expect(result.verdict).toBe("pass");
+ });
+
+ it("fails when a required dimension mismatches", () => {
+ const result = evaluateTrustGate(
+ contract,
+ { exit_code: 0, stdout: "hello" },
+ { exit_code: 1, stdout: "hello" },
+ );
+ expect(result.verdict).toBe("fail");
+ expect(result.first_divergence).not.toBeNull();
+ expect(result.first_divergence!.dimension).toBe("exit_code");
+ });
+
+ it("reports unknown when both dimensions are undefined", () => {
+ const result = evaluateTrustGate(contract, {}, {});
+ // exit_code and stdout are both undefined -> unknown
+ // timing is volatile -> match
+ expect(result.verdict).toBe("unknown");
+ });
+
+ it("ignores volatile dimension differences", () => {
+ const result = evaluateTrustGate(
+ contract,
+ { exit_code: 0, stdout: "hello", timing: 100 },
+ { exit_code: 0, stdout: "hello", timing: 200 },
+ );
+ expect(result.verdict).toBe("pass");
+ });
+
+ it("truncated evidence is not treated as equivalence", () => {
+ const result = evaluateTrustGate(
+ contract,
+ { exit_code: 0, stdout: "hello" },
+ { exit_code: 0, stdout: "hello" },
+ { truncated: true },
+ );
+ expect(result.verdict).toBe("fail");
+ });
+});