Reverse engineer anything with agents, from app behavior down to native binaries.
TypeScript agent-skillsai-agentsbinary-analysisclaude-codeclicodexcordisdecompilerdeveloper-toolsdshdsh-pluginghidrahopperllmmcpmodel-context-protocolreareverse-engineeringstatic-analysistouhou
feat(conformance): add portable conformance export package format
morluto4ee30e4145db
diff --git a/src/domain/conformancePackage.ts b/src/domain/conformancePackage.tsnew file mode 100644index 000000000..2c347d484--- /dev/null+++ b/src/domain/conformancePackage.ts@@ -0,0 +1,218 @@+import { createHash } from "node:crypto";++import canonicalize from "canonicalize";+import { z } from "zod";++import { evidenceEnvelopeSchema } from "./evidence.js";+import { evidenceBundleSchema } from "./evidenceBundle.js";++/** Schema version for the conformance package format. */+export const CONFORMANCE_PACKAGE_VERSION = 1;++/** A stable, path-independent identifier for a conformance package. */+export const conformancePackageIdSchema = z+ .string()+ .regex(/^cp_[a-f0-9]{64}$/u);++/** Identifier for a single scenario within a package. */+export const scenarioIdSchema = z+ .string()+ .regex(/^[A-Za-z][A-Za-z0-9._-]{0,99}$/u);++/** Deterministic manifest for a scenario fixture. */+export const scenarioManifestSchema = z.strictObject({+ scenario_id: scenarioIdSchema,+ name: z.string().min(1),+ description: z.string().min(1),+ /** Source-owned fixture path relative to the repository root. */+ fixture_path: z.string().min(1),+ /** Expected exit code or null if unspecified. */+ expected_exit_code: z.number().int().nullable(),+ /** Expected output patterns that must appear in the capture. */+ expected_patterns: z.array(z.string()).default([]),+});++/** Replay plan describing how a scenario is replayed deterministically. */+export const replayPlanSchema = z.strictObject({+ scenario_id: scenarioIdSchema,+ /** Ordered steps to execute during replay. */+ steps: z+ .array(+ z.strictObject({+ step_id: z.string().min(1),+ action: z.string().min(1),+ arguments: z.array(z.string()).default([]),+ timeout_ms: z.number().int().positive(),+ }),+ )+ .min(1)+ .max(100),+ /** Environment variables to set (never inherit host paths). */+ environment: z.record(z.string(), z.string()).default({}),+});++/** Shim plan for intercepting observable effects. */+export const shimPlanSchema = z.strictObject({+ scenario_id: scenarioIdSchema,+ /** Shims to install, each intercepting a named effect. */+ shims: z+ .array(+ z.strictObject({+ shim_id: z.string().min(1),+ kind: z.enum(["filesystem", "network", "process", "signal"]),+ target: z.string().min(1),+ policy: z.enum(["observe", "allow", "block", "emulate"]),+ }),+ )+ .min(0)+ .max(50),+});++/** Expected evidence for a scenario, checked against the captured run. */+export const expectedEvidenceSchema = z.strictObject({+ scenario_id: scenarioIdSchema,+ /** Expected evidence envelopes. */+ envelopes: z.array(evidenceEnvelopeSchema).min(0).max(100),+ /** Expected evidence bundle. */+ bundle: evidenceBundleSchema.nullable(),+ /** Required dimensions that must be present in the evidence. */+ required_dimensions: z.array(z.string().min(1)).default([]),+});++/** Verifier contract describing how to validate conformance. */+export const verifierContractSchema = z.strictObject({+ scenario_id: scenarioIdSchema,+ /** Dimensions to verify. */+ dimensions: z+ .array(+ z.strictObject({+ name: z.string().min(1),+ required: z.boolean().default(true),+ comparison: z.enum(["exact", "semantic", "fuzzy"]),+ }),+ )+ .min(1)+ .max(50),+ /** Tolerance for timing differences in milliseconds. */+ timing_tolerance_ms: z.number().int().nonnegative().default(0),+});++/** Top-level conformance package manifest. */+export const conformancePackageSchema = z.strictObject({+ schema_version: z.literal(CONFORMANCE_PACKAGE_VERSION),+ package_id: conformancePackageIdSchema,+ name: z.string().min(1),+ description: z.string().min(1),+ created_at: z.string().datetime(),+ /** Scenario manifests. */+ scenarios: z.array(scenarioManifestSchema).min(1).max(100),+ /** Replay plans keyed by scenario_id. */+ replay_plans: z.array(replayPlanSchema).min(1).max(100),+ /** Shim plans keyed by scenario_id. */+ shim_plans: z.array(shimPlanSchema).default([]),+ /** Expected evidence for each scenario. */+ expected_evidence: z.array(expectedEvidenceSchema).min(1).max(100),+ /** Verifier contracts for each scenario. */+ verifier_contracts: z.array(verifierContractSchema).min(1).max(100),+});++export type ConformancePackage = z.infer<typeof conformancePackageSchema>;+export type ScenarioManifest = z.infer<typeof scenarioManifestSchema>;+export type ReplayPlan = z.infer<typeof replayPlanSchema>;+export type ShimPlan = z.infer<typeof shimPlanSchema>;+export type ExpectedEvidence = z.infer<typeof expectedEvidenceSchema>;+export type VerifierContract = z.infer<typeof verifierContractSchema>;++/** Error from validation or package operations. */+export type ConformancePackageError =+ | { kind: "invalid_package"; message: string }+ | { kind: "scenario_not_found"; scenario_id: string }+ | { kind: "duplicate_scenario"; scenario_id: string };++/** Result type for package operations. */+export type ConformancePackageResult<T> =+ | { ok: true; value: T }+ | { ok: false; error: ConformancePackageError };++/** Validate a conformance package and cross-check internal references. */+export function validateConformancePackage(+ input: unknown,+): ConformancePackageResult<ConformancePackage> {+ const parsed = conformancePackageSchema.safeParse(input);+ if (!parsed.success)+ return {+ ok: false,+ error: {+ kind: "invalid_package",+ message: parsed.error.issues[0]?.message ?? "invalid package",+ },+ };++ const pkg = parsed.data;++ // Check for duplicate scenario IDs+ const seen = new Set<string>();+ for (const s of pkg.scenarios) {+ if (seen.has(s.scenario_id))+ return {+ ok: false,+ error: { kind: "duplicate_scenario", scenario_id: s.scenario_id },+ };+ seen.add(s.scenario_id);+ }++ // Check that replay plans reference existing scenarios+ for (const rp of pkg.replay_plans) {+ if (!seen.has(rp.scenario_id))+ return {+ ok: false,+ error: {+ kind: "scenario_not_found",+ scenario_id: rp.scenario_id,+ },+ };+ }++ // Check that expected evidence references existing scenarios+ for (const ee of pkg.expected_evidence) {+ if (!seen.has(ee.scenario_id))+ return {+ ok: false,+ error: {+ kind: "scenario_not_found",+ scenario_id: ee.scenario_id,+ },+ };+ }++ // Check that verifier contracts reference existing scenarios+ for (const vc of pkg.verifier_contracts) {+ if (!seen.has(vc.scenario_id))+ return {+ ok: false,+ error: {+ kind: "scenario_not_found",+ scenario_id: vc.scenario_id,+ },+ };+ }++ return { ok: true, value: pkg };+}++/** Compute a deterministic package ID from the canonical JSON. */+export function computePackageId(+ pkg: Omit<ConformancePackage, "package_id">,+): string {+ const json = canonicalize(pkg);+ if (!json) throw new Error("failed to canonicalize package");+ return `cp_${createHash("sha256").update(json).digest("hex")}`;+}++/** Create a conformance package with an auto-computed package_id. */+export function createConformancePackage(+ pkg: Omit<ConformancePackage, "package_id">,+): ConformancePackage {+ const package_id = computePackageId(pkg);+ return { package_id, ...pkg };+}diff --git a/src/domain/conformanceReplay.ts b/src/domain/conformanceReplay.tsnew file mode 100644index 000000000..2e0640e17--- /dev/null+++ b/src/domain/conformanceReplay.ts@@ -0,0 +1,138 @@+import { z } from "zod";++import {+ conformancePackageSchema,+ type ConformancePackage,+} from "./conformancePackage.js";+import { trustGateResultSchema } from "./conformanceTrustGate.js";++/** Result of a single scenario replay. */+export const scenarioReplayResultSchema = z.strictObject({+ scenario_id: z.string().min(1),+ status: z.enum(["pass", "fail", "error", "skipped"]),+ exit_code: z.number().int().nullable(),+ duration_ms: z.number().int().nonnegative(),+ output: z.string().default(""),+ error: z.string().nullable(),+});++/** Result of replaying an entire conformance package. */+export const packageReplayResultSchema = z.strictObject({+ package_id: z.string().min(1),+ total_scenarios: z.number().int().positive(),+ passed: z.number().int().nonnegative(),+ failed: z.number().int().nonnegative(),+ errored: z.number().int().nonnegative(),+ skipped: z.number().int().nonnegative(),+ scenario_results: z.array(scenarioReplayResultSchema),+ trust_gate_results: z.array(trustGateResultSchema),+ drift_detected: z.boolean(),+ first_drift: z+ .strictObject({+ scenario_id: z.string().min(1),+ dimension: z.string().min(1),+ message: z.string(),+ })+ .nullable(),+});++export type ScenarioReplayResult = z.infer<typeof scenarioReplayResultSchema>;+export type PackageReplayResult = z.infer<typeof packageReplayResultSchema>;++/** Callback interface for replaying a scenario. */+export interface ScenarioRunner {+ (scenarioId: string, fixturePath: string): Promise<ScenarioReplayResult>;+}++/**+ * Default no-op scenario runner that skips all scenarios.+ * Real CI replays inject an actual runner.+ */+async function defaultRunner(): Promise<ScenarioReplayResult> {+ return {+ scenario_id: "unknown",+ status: "skipped",+ exit_code: null,+ duration_ms: 0,+ output: "",+ error: "no runner provided",+ };+}++/**+ * Replay a conformance package from a clean checkout.+ * Runs each scenario, evaluates trust gates, and detects drift+ * between the captured run and the committed package.+ */+export async function replayConformancePackage(+ packageInput: unknown,+ actualEvidence: Record<string, Record<string, unknown>>,+ runner: ScenarioRunner = defaultRunner,+ options: { truncated?: boolean } = {},+): Promise<PackageReplayResult> {+ const parsed = conformancePackageSchema.safeParse(packageInput);+ if (!parsed.success)+ throw new TypeError(+ `invalid conformance package: ${parsed.error.issues[0]?.message ?? "unknown"}`,+ );+ const pkg: ConformancePackage = parsed.data;++ const scenarioResults: ScenarioReplayResult[] = [];+ let passed = 0;+ let failed = 0;+ let errored = 0;+ let skipped = 0;++ for (const scenario of pkg.scenarios) {+ const result = await runner(scenario.scenario_id, scenario.fixture_path);+ scenarioResults.push(result);++ switch (result.status) {+ case "pass":+ passed++;+ break;+ case "fail":+ failed++;+ break;+ case "error":+ errored++;+ break;+ case "skipped":+ skipped++;+ break;+ }+ }++ const trustGateResults = evaluatePackageTrustGates(+ pkg,+ actualEvidence,+ options,+ );++ const driftGates = trustGateResults.filter((r) => r.verdict === "fail");+ const drift_detected = driftGates.length > 0;+ const first_drift_gate = driftGates[0];+ const first_drift = first_drift_gate?.first_divergence+ ? {+ scenario_id: first_drift_gate.scenario_id,+ dimension: first_drift_gate.first_divergence.dimension,+ message: first_drift_gate.first_divergence.message,+ }+ : null;++ return {+ package_id: pkg.package_id,+ total_scenarios: pkg.scenarios.length,+ passed,+ failed,+ errored,+ skipped,+ scenario_results: scenarioResults,+ trust_gate_results: trustGateResults,+ drift_detected,+ first_drift,+ };+}++/** Import trust gate evaluator for replay. */+import { evaluatePackageTrustGates } from "./conformanceTrustGate.js";diff --git a/src/domain/conformanceTrustGate.ts b/src/domain/conformanceTrustGate.tsnew file mode 100644index 000000000..485768400--- /dev/null+++ b/src/domain/conformanceTrustGate.ts@@ -0,0 +1,246 @@+import { z } from "zod";++import type {+ ConformancePackage,+ VerifierContract,+} from "./conformancePackage.js";++/** Result of comparing a single dimension. */+export const dimensionResultSchema = z.strictObject({+ name: z.string().min(1),+ status: z.enum(["match", "mismatch", "unknown", "truncated"]),+ message: z.string().default(""),+ evidence_ids: z.array(z.string()).default([]),+});++/** Result of evaluating a trust gate for one scenario. */+export const trustGateResultSchema = z.strictObject({+ scenario_id: z.string().min(1),+ verdict: z.enum(["pass", "fail", "unknown"]),+ dimension_results: z.array(dimensionResultSchema),+ first_divergence: z+ .strictObject({+ dimension: z.string().min(1),+ evidence_id: z.string().min(1),+ message: z.string(),+ })+ .nullable(),+});++export type DimensionResult = z.infer<typeof dimensionResultSchema>;+export type TrustGateResult = z.infer<typeof trustGateResultSchema>;++/** Dimension that is volatile due to timing or normalization. */+const VOLATILE_DIMENSIONS = new Set([+ "timing",+ "timestamp",+ "pid",+ "ppid",+ "duration_ms",+ "created_at",+ "updated_at",+]);++/** Dimension that carries semantic content. */+const SEMANTIC_DIMENSIONS = new Set([+ "exit_code",+ "stdout",+ "stderr",+ "filesystem",+ "process_tree",+ "shim_events",+ "protocol_events",+ "event_journal",+]);++/** Check if a dimension name is volatile (timing/normalization noise). */+export function isVolatileDimension(name: string): boolean {+ return VOLATILE_DIMENSIONS.has(name);+}++/** Check if a dimension name carries semantic content. */+export function isSemanticDimension(name: string): boolean {+ return SEMANTIC_DIMENSIONS.has(name);+}++/**+ * Compare two values for a dimension, classifying semantic diffs+ * from timing/normalization noise.+ *+ * Volatile dimensions (timing, timestamps, PIDs) are always treated+ * as matches because their values are expected to differ across+ * runs and do not carry semantic meaning.+ */+export function compareDimension(+ dimensionName: string,+ expected: unknown,+ actual: unknown,+ options: { truncated?: boolean } = {},+): DimensionResult {+ // Volatile dimensions: always match regardless of values+ if (isVolatileDimension(dimensionName)) {+ return {+ name: dimensionName,+ status: "match",+ message: "volatile dimension ignored",+ evidence_ids: [],+ };+ }++ if (options.truncated) {+ return {+ name: dimensionName,+ status: "truncated",+ message: "evidence was truncated",+ evidence_ids: [],+ };+ }++ if (expected === undefined && actual === undefined) {+ return {+ name: dimensionName,+ status: "unknown",+ message: "both expected and actual are undefined",+ evidence_ids: [],+ };+ }++ if (expected === undefined || actual === undefined) {+ return {+ name: dimensionName,+ status: "mismatch",+ message: `expected ${expected === undefined ? "present" : "absent"}, actual ${actual === undefined ? "absent" : "present"}`,+ evidence_ids: [],+ };+ }++ // Semantic comparison for structured objects+ if (typeof expected === "object" && typeof actual === "object") {+ const expectedJson = JSON.stringify(sortKeys(expected));+ const actualJson = JSON.stringify(sortKeys(actual));+ if (expectedJson === actualJson) {+ return {+ name: dimensionName,+ status: "match",+ message: "",+ evidence_ids: [],+ };+ }+ return {+ name: dimensionName,+ status: "mismatch",+ message: "semantic content differs",+ evidence_ids: [],+ };+ }++ // Primitive comparison+ if (expected === actual) {+ return {+ name: dimensionName,+ status: "match",+ message: "",+ evidence_ids: [],+ };+ }++ return {+ name: dimensionName,+ status: "mismatch",+ message: `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`,+ evidence_ids: [],+ };+}++/** Recursively sort object keys for deterministic comparison. */+function sortKeys(value: unknown): unknown {+ if (value === null || typeof value !== "object") return value;+ if (Array.isArray(value)) return value.map(sortKeys);+ const sorted: Record<string, unknown> = {};+ for (const key of Object.keys(value as Record<string, unknown>).sort()) {+ sorted[key] = sortKeys((value as Record<string, unknown>)[key]);+ }+ return sorted;+}++/**+ * Evaluate a trust gate for one scenario by comparing expected+ * evidence against the actual captured run.+ */+export function evaluateTrustGate(+ contract: VerifierContract,+ expectedEvidence: Record<string, unknown>,+ actualEvidence: Record<string, unknown>,+ options: { truncated?: boolean } = {},+): TrustGateResult {+ const dimensionResults: DimensionResult[] = [];+ let firstDivergence: TrustGateResult["first_divergence"] = null;++ for (const dim of contract.dimensions) {+ if (!dim.required) continue;+ const expectedValue = expectedEvidence[dim.name];+ const actualValue = actualEvidence[dim.name];+ const result = compareDimension(+ dim.name,+ expectedValue,+ actualValue,+ options,+ );+ dimensionResults.push(result);++ if (result.status === "mismatch" && firstDivergence === null) {+ firstDivergence = {+ dimension: dim.name,+ evidence_id: result.evidence_ids[0] ?? "unknown",+ message: result.message,+ };+ }+ }++ const hasFail = dimensionResults.some(+ (r) => r.status === "mismatch" || r.status === "truncated",+ );+ const hasUnknown = dimensionResults.some((r) => r.status === "unknown");++ const verdict: TrustGateResult["verdict"] = hasFail+ ? "fail"+ : hasUnknown+ ? "unknown"+ : "pass";++ return {+ scenario_id: contract.scenario_id,+ verdict,+ dimension_results: dimensionResults,+ first_divergence: firstDivergence,+ };+}++/**+ * Evaluate trust gates for all scenarios in a conformance package.+ * Rejects runs that differ in required dimensions. Unknown/truncated+ * evidence is never treated as equivalence.+ */+export function evaluatePackageTrustGates(+ pkg: ConformancePackage,+ actualEvidence: Record<string, Record<string, unknown>>,+ options: { truncated?: boolean } = {},+): TrustGateResult[] {+ const results: TrustGateResult[] = [];+ for (const contract of pkg.verifier_contracts) {+ const expected = pkg.expected_evidence.find(+ (e) => e.scenario_id === contract.scenario_id,+ );+ if (!expected) continue;+ const expectedEvidence = expected.envelopes[0] ?? {};+ const actual = actualEvidence[contract.scenario_id] ?? {};+ const result = evaluateTrustGate(+ contract,+ expectedEvidence as Record<string, unknown>,+ actual,+ options,+ );+ results.push(result);+ }+ return results;+}diff --git a/tests/conformancePackage.test.ts b/tests/conformancePackage.test.tsnew file mode 100644index 000000000..283f6e806--- /dev/null+++ b/tests/conformancePackage.test.ts@@ -0,0 +1,181 @@+import { describe, expect, it } from "vitest";++import {+ CONFORMANCE_PACKAGE_VERSION,+ conformancePackageSchema,+ computePackageId,+ createConformancePackage,+ validateConformancePackage,+ type ScenarioManifest,+ type ReplayPlan,+ type ShimPlan,+ type ExpectedEvidence,+} from "../src/domain/conformancePackage.js";++const validPackage = createConformancePackage({+ schema_version: CONFORMANCE_PACKAGE_VERSION,+ name: "test-fixture",+ description: "A test conformance fixture",+ created_at: "2026-07-28T00:00:00Z",+ scenarios: [+ {+ scenario_id: "s1",+ name: "Simple spawn",+ description: "A simple process spawn scenario",+ fixture_path: "tests/conformance/c/fixture.c",+ expected_exit_code: 0,+ expected_patterns: ["Hello", "World"],+ },+ ],+ replay_plans: [+ {+ scenario_id: "s1",+ steps: [+ {+ step_id: "step1",+ action: "compile",+ arguments: ["gcc", "fixture.c"],+ timeout_ms: 5000,+ },+ ],+ environment: {},+ },+ ],+ shim_plans: [],+ expected_evidence: [+ {+ scenario_id: "s1",+ envelopes: [],+ bundle: null,+ required_dimensions: ["exit_code", "stdout"],+ },+ ],+ verifier_contracts: [+ {+ scenario_id: "s1",+ dimensions: [+ {+ name: "exit_code",+ required: true,+ comparison: "exact",+ },+ ],+ timing_tolerance_ms: 100,+ },+ ],+});++describe("conformance package format", () => {+ it("validates a well-formed package", () => {+ const result = validateConformancePackage(validPackage);+ expect(result.ok).toBe(true);+ });++ it("rejects invalid schema version", () => {+ const result = validateConformancePackage({+ ...validPackage,+ schema_version: 999,+ });+ expect(result.ok).toBe(false);+ });++ it("rejects duplicate scenario IDs", () => {+ const result = validateConformancePackage({+ ...validPackage,+ scenarios: [validPackage.scenarios[0], validPackage.scenarios[0]],+ });+ expect(result.ok).toBe(false);+ });++ it("rejects replay plan referencing non-existent scenario", () => {+ const result = validateConformancePackage({+ ...validPackage,+ replay_plans: [+ {+ scenario_id: "nonexistent",+ steps: [+ {+ step_id: "step1",+ action: "run",+ arguments: [],+ timeout_ms: 1000,+ },+ ],+ environment: {},+ },+ ],+ });+ expect(result.ok).toBe(false);+ });++ it("rejects verifier contract referencing non-existent scenario", () => {+ const result = validateConformancePackage({+ ...validPackage,+ verifier_contracts: [+ {+ scenario_id: "nonexistent",+ dimensions: [+ { name: "exit_code", required: true, comparison: "exact" },+ ],+ timing_tolerance_ms: 0,+ },+ ],+ });+ expect(result.ok).toBe(false);+ });++ it("rejects expected evidence referencing non-existent scenario", () => {+ const result = validateConformancePackage({+ ...validPackage,+ expected_evidence: [+ {+ scenario_id: "nonexistent",+ envelopes: [],+ bundle: null,+ required_dimensions: [],+ },+ ],+ });+ expect(result.ok).toBe(false);+ });++ it("computes a deterministic package ID", () => {+ const id = computePackageId({+ ...validPackage,+ });+ expect(id).toMatch(/^cp_[a-f0-9]{64}$/u);+ // Same input yields same ID+ const id2 = computePackageId({+ ...validPackage,+ });+ expect(id).toBe(id2);+ });++ it("produces different IDs for different packages", () => {+ const id1 = computePackageId({ ...validPackage, name: "packageA" });+ const id2 = computePackageId({ ...validPackage, name: "packageB" });+ expect(id1).not.toBe(id2);+ });++ it("uses all exported types", () => {+ const scenario: ScenarioManifest = validPackage.scenarios[0]!;+ expect(scenario.scenario_id).toBe("s1");++ const replay: ReplayPlan = validPackage.replay_plans[0]!;+ expect(replay.scenario_id).toBe("s1");++ const shim: ShimPlan = validPackage.shim_plans[0] ?? {+ scenario_id: "s1",+ shims: [],+ };+ expect(shim.scenario_id).toBe("s1");++ const evidence: ExpectedEvidence = validPackage.expected_evidence[0]!;+ expect(evidence.scenario_id).toBe("s1");+ });++ it("parses via zod schema", () => {+ const result = conformancePackageSchema.safeParse(validPackage);+ expect(result.success).toBe(true);+ });+});diff --git a/tests/conformanceReplay.test.ts b/tests/conformanceReplay.test.tsnew file mode 100644index 000000000..078997b84--- /dev/null+++ b/tests/conformanceReplay.test.ts@@ -0,0 +1,190 @@+import { describe, expect, it } from "vitest";++import {+ CONFORMANCE_PACKAGE_VERSION,+ createConformancePackage,+ type ConformancePackage,+} from "../src/domain/conformancePackage.js";+import {+ replayConformancePackage,+ type ScenarioReplayResult,+} from "../src/domain/conformanceReplay.js";++const validPackage = createConformancePackage({+ schema_version: CONFORMANCE_PACKAGE_VERSION,+ name: "test-fixture",+ description: "A test conformance fixture",+ created_at: "2026-07-28T00:00:00Z",+ scenarios: [+ {+ scenario_id: "s1",+ name: "Simple spawn",+ description: "A simple process spawn scenario",+ fixture_path: "tests/conformance/c/fixture.c",+ expected_exit_code: 0,+ expected_patterns: [],+ },+ ],+ replay_plans: [+ {+ scenario_id: "s1",+ steps: [+ {+ step_id: "step1",+ action: "run",+ arguments: [],+ timeout_ms: 1000,+ },+ ],+ environment: {},+ },+ ],+ shim_plans: [],+ expected_evidence: [+ {+ scenario_id: "s1",+ envelopes: [],+ bundle: null,+ required_dimensions: [],+ },+ ],+ verifier_contracts: [+ {+ scenario_id: "s1",+ dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],+ timing_tolerance_ms: 0,+ },+ ],+});++async function passingRunner(+ scenarioId: string,+): Promise<ScenarioReplayResult> {+ return {+ scenario_id: scenarioId,+ status: "pass",+ exit_code: 0,+ duration_ms: 10,+ output: "ok",+ error: null,+ };+}++async function failingRunner(+ scenarioId: string,+): Promise<ScenarioReplayResult> {+ return {+ scenario_id: scenarioId,+ status: "fail",+ exit_code: 1,+ duration_ms: 10,+ output: "fail",+ error: null,+ };+}++describe("conformance CI replay", () => {+ it("replays a package with passing scenarios", async () => {+ const result = await replayConformancePackage(+ validPackage,+ {},+ passingRunner,+ );+ expect(result.total_scenarios).toBe(1);+ expect(result.passed).toBe(1);+ expect(result.failed).toBe(0);+ expect(result.errored).toBe(0);+ });++ it("replays a package with failing scenarios", async () => {+ const result = await replayConformancePackage(+ validPackage,+ {},+ failingRunner,+ );+ expect(result.failed).toBe(1);+ expect(result.passed).toBe(0);+ });++ it("does not detect drift when expected and actual match", async () => {+ const result = await replayConformancePackage(+ validPackage,+ {},+ passingRunner,+ );+ expect(result.drift_detected).toBe(false);+ });++ it("throws on invalid package", async () => {+ await expect(replayConformancePackage({}, {})).rejects.toThrow(+ /invalid conformance package/u,+ );+ });++ it("counts all result types", async () => {+ const base: ConformancePackage = validPackage;+ const multiPackage = createConformancePackage({+ ...base,+ scenarios: [+ base.scenarios[0]!,+ {+ scenario_id: "s2",+ name: "Second scenario",+ description: "Second",+ fixture_path: "tests/conformance/c/fixture2.c",+ expected_exit_code: 1,+ expected_patterns: [],+ },+ ],+ replay_plans: [+ base.replay_plans[0]!,+ {+ scenario_id: "s2",+ steps: [+ {+ step_id: "step1",+ action: "run",+ arguments: [],+ timeout_ms: 1000,+ },+ ],+ environment: {},+ },+ ],+ expected_evidence: [+ base.expected_evidence[0]!,+ {+ scenario_id: "s2",+ envelopes: [],+ bundle: null,+ required_dimensions: [],+ },+ ],+ verifier_contracts: [+ base.verifier_contracts[0]!,+ {+ scenario_id: "s2",+ dimensions: [+ { name: "exit_code", required: true, comparison: "exact" },+ ],+ timing_tolerance_ms: 0,+ },+ ],+ });+ const result = await replayConformancePackage(+ multiPackage,+ {},+ async (scenarioId) => ({+ scenario_id: scenarioId,+ status: scenarioId === "s1" ? "pass" : "fail",+ exit_code: scenarioId === "s1" ? 0 : 1,+ duration_ms: 10,+ output: "",+ error: null,+ }),+ );+ expect(result.total_scenarios).toBe(2);+ expect(result.passed).toBe(1);+ expect(result.failed).toBe(1);+ });+});diff --git a/tests/conformanceTrustGate.test.ts b/tests/conformanceTrustGate.test.tsnew file mode 100644index 000000000..0e03a9f07--- /dev/null+++ b/tests/conformanceTrustGate.test.ts@@ -0,0 +1,113 @@+import { describe, expect, it } from "vitest";++import type { VerifierContract } from "../src/domain/conformancePackage.js";+import {+ compareDimension,+ evaluateTrustGate,+ isSemanticDimension,+ isVolatileDimension,+} from "../src/domain/conformanceTrustGate.js";++const contract: VerifierContract = {+ scenario_id: "s1",+ dimensions: [+ { name: "exit_code", required: true, comparison: "exact" },+ { name: "stdout", required: true, comparison: "semantic" },+ { name: "timing", required: true, comparison: "fuzzy" },+ ],+ timing_tolerance_ms: 100,+};++describe("conformance trust gates", () => {+ it("classifies volatile dimensions correctly", () => {+ expect(isVolatileDimension("timing")).toBe(true);+ expect(isVolatileDimension("timestamp")).toBe(true);+ expect(isVolatileDimension("pid")).toBe(true);+ expect(isVolatileDimension("exit_code")).toBe(false);+ });++ it("classifies semantic dimensions correctly", () => {+ expect(isSemanticDimension("exit_code")).toBe(true);+ expect(isSemanticDimension("stdout")).toBe(true);+ expect(isSemanticDimension("timing")).toBe(false);+ });++ it("matches identical values", () => {+ const result = compareDimension("exit_code", 0, 0);+ expect(result.status).toBe("match");+ });++ it("detects mismatches", () => {+ const result = compareDimension("exit_code", 0, 1);+ expect(result.status).toBe("mismatch");+ });++ it("ignores volatile dimensions", () => {+ const result = compareDimension("timing", 100, 200);+ expect(result.status).toBe("match");+ expect(result.message).toContain("volatile");+ });++ it("handles truncated evidence", () => {+ const result = compareDimension("exit_code", 0, null, {+ truncated: true,+ });+ expect(result.status).toBe("truncated");+ });++ it("handles undefined values", () => {+ const result = compareDimension("exit_code", undefined, undefined);+ expect(result.status).toBe("unknown");+ });++ it("handles missing actual", () => {+ const result = compareDimension("exit_code", 0, undefined);+ expect(result.status).toBe("mismatch");+ });++ it("passes when all dimensions match", () => {+ const result = evaluateTrustGate(+ contract,+ { exit_code: 0, stdout: "hello" },+ { exit_code: 0, stdout: "hello" },+ );+ expect(result.verdict).toBe("pass");+ });++ it("fails when a required dimension mismatches", () => {+ const result = evaluateTrustGate(+ contract,+ { exit_code: 0, stdout: "hello" },+ { exit_code: 1, stdout: "hello" },+ );+ expect(result.verdict).toBe("fail");+ expect(result.first_divergence).not.toBeNull();+ expect(result.first_divergence!.dimension).toBe("exit_code");+ });++ it("reports unknown when both dimensions are undefined", () => {+ const result = evaluateTrustGate(contract, {}, {});+ // exit_code and stdout are both undefined -> unknown+ // timing is volatile -> match+ expect(result.verdict).toBe("unknown");+ });++ it("ignores volatile dimension differences", () => {+ const result = evaluateTrustGate(+ contract,+ { exit_code: 0, stdout: "hello", timing: 100 },+ { exit_code: 0, stdout: "hello", timing: 200 },+ );+ expect(result.verdict).toBe("pass");+ });++ it("truncated evidence is not treated as equivalence", () => {+ const result = evaluateTrustGate(+ contract,+ { exit_code: 0, stdout: "hello" },+ { exit_code: 0, stdout: "hello" },+ { truncated: true },+ );+ expect(result.verdict).toBe("fail");+ });+});