Skip to content

The open-source app everyone uses to manage agents at work

test(runner): add credential-free acceptance foundation (#12652)

Dottabfb98aff5dc1
diff --git a/package.json b/package.json
index 67c699d3b2..1e73bb19d3 100644
--- a/package.json
+++ b/package.json
@@ -23,6 +23,8 @@
"test": "pnpm run test:run",
"test:watch": "pnpm run preflight:workspace-links && vitest",
"test:run": "pnpm run preflight:workspace-links && node scripts/run-vitest-stable.mjs",
+ "test:runner-acceptance": "vitest run --config tests/runner-acceptance/vitest.config.ts",
+ "test:runner-acceptance:typecheck": "tsc -p tests/runner-acceptance/tsconfig.json",
"test:run:general": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && node scripts/run-vitest-stable.mjs --mode general",
"test:run:serialized": "pnpm run preflight:workspace-links && pnpm --filter @paperclipai/plugin-sdk ensure-build-deps && node scripts/run-vitest-stable.mjs --mode serialized",
"db:generate": "pnpm --filter @paperclipai/db generate",
diff --git a/packages/paperclip-runner/src/live/live-session.test.ts b/packages/paperclip-runner/src/live/live-session.test.ts
index 4881dc4851..12ec300ebf 100644
--- a/packages/paperclip-runner/src/live/live-session.test.ts
+++ b/packages/paperclip-runner/src/live/live-session.test.ts
@@ -1118,6 +1118,7 @@ describe("Capability live runnerd and Codex session", () => {
]));
await resumedService.shutdown(resumed.id, "test complete");
},
- 15_000,
+ // CI exercises two real process generations here and can exceed the unit default under load.
+ 30_000,
);
});
diff --git a/tests/runner-acceptance/README.md b/tests/runner-acceptance/README.md
new file mode 100644
index 0000000000..04339d3176
--- /dev/null
+++ b/tests/runner-acceptance/README.md
@@ -0,0 +1,70 @@
+# Credential-free Runner acceptance foundation
+
+This directory defines the reviewable, deterministic foundation for Runner
+acceptance checks. It contains no launcher and grants no authority to start
+Paperclip, a provider process, a browser, a remote environment, or a billable
+model request.
+
+The catalog covers:
+
+- every current built-in direct adapter except the explicitly deferred Pi
+ adapter;
+- the external-plugin direct-adapter compatibility contract;
+- Paperclip Runner with Codex;
+- Paperclip Runner with the qualified OpenCode model; and
+- Paperclip Runner with the qualified ACPX Claude and Codex profiles.
+
+The direct-adapter cells assert the legacy boundary: runnerd does not start,
+native records are not created, direct finalization remains authoritative, and
+classic task controls remain available. Native cells assert persisted provider
+identity, runtime authority, structured-question behavior, and recovery of an
+already-recorded run after the rollout flag changes.
+
+Registered compatibility-only adapters, including the retired `acpx_local`
+entry, remain in the direct catalog so a future selection check can prove they
+never fall into native execution. Their presence is not a claim that the
+adapter can start a provider session.
+
+## Commands
+
+Run the isolated unit suite:
+
+```sh
+pnpm test:runner-acceptance
+```
+
+Check the standalone TypeScript boundary:
+
+```sh
+pnpm test:runner-acceptance:typecheck
+```
+
+These commands are credential-free. They validate the catalog and pure support
+utilities; they do not claim that a provider was contacted.
+
+## Result boundary
+
+Future fixture executors may emit `paperclip.runner-acceptance.result/v1`
+objects and pass them to `buildRunnerAcceptanceReport`. Results must name a
+catalog cell, report every expected assertion, carry successful redaction, and
+contain no sensitive-looking structured values. The aggregator produces only
+in-memory normalized data plus optional Markdown or JUnit strings. It does not
+write evidence, screenshots, history, or public reports.
+
+## Deliberate exclusions
+
+This foundation does not include:
+
+- Pi or any Pi transitive package;
+- Claude Managed or AWS AgentCore;
+- Daytona, remote images, live provider execution, or cost accounting;
+- provider-key loading, secret references, auth-file discovery, or tracked env
+ templates;
+- screenshots, traces, raw logs, evidence archives, dashboards, history, or
+ publication workflows; or
+- the eval kernel, scenario explorer, browser SDK, or package/runtime
+ replacement work.
+
+Add an executor only in a later, explicitly authorized change. Keep its launch
+authority separate from this catalog, and make any billable or secret-bearing
+mode opt-in and independently reviewed.
diff --git a/tests/runner-acceptance/catalog.test.ts b/tests/runner-acceptance/catalog.test.ts
new file mode 100644
index 0000000000..ca9850c678
--- /dev/null
+++ b/tests/runner-acceptance/catalog.test.ts
@@ -0,0 +1,64 @@
+import { describe, expect, it } from "vitest";
+
+import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
+import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js";
+import {
+ directAcceptanceProfiles,
+ nativeAcceptanceProfiles,
+ runnerAcceptanceCases,
+ runnerAcceptanceMatrix,
+ validateRunnerAcceptanceCatalog,
+} from "./catalog.js";
+
+describe("credential-free Runner acceptance catalog", () => {
+ it("covers direct adapters without activating the native runtime", () => {
+ expect(directAcceptanceProfiles.length).toBeGreaterThan(10);
+ expect(directAcceptanceProfiles.every((profile) =>
+ profile.generation === "direct"
+ && profile.expectedRuntimeMode === "legacy"
+ && profile.invariants.includes("runnerd.start_count=0")
+ && profile.invariants.includes("native_record.count=0"))).toBe(true);
+ expect(directAcceptanceProfiles.some(({ adapterType }) => adapterType === "process")).toBe(true);
+ expect(directAcceptanceProfiles.some(({ adapterType }) => adapterType === "http")).toBe(true);
+ expect(directAcceptanceProfiles.some(({ adapterScope }) =>
+ adapterScope === "external_plugin_contract")).toBe(true);
+ });
+
+ it("contains only the approved native providers and ACPX profiles", () => {
+ expect(nativeAcceptanceProfiles.map(({ id }) => id)).toEqual([
+ "runner-codex",
+ "runner-opencode",
+ "runner-acpx-claude",
+ "runner-acpx-codex",
+ ]);
+ expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-opencode")?.model)
+ .toBe(QUALIFIED_OPENCODE_MODEL);
+ expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-acpx-claude")?.model)
+ .toBe(QUALIFIED_ACPX_PROFILES.claude.qualificationModel);
+ expect(nativeAcceptanceProfiles.find(({ id }) => id === "runner-acpx-codex")?.model)
+ .toBe(QUALIFIED_ACPX_PROFILES.codex.qualificationModel);
+ expect(nativeAcceptanceProfiles.some(({ adapterConfig }) =>
+ adapterConfig.acpxAgent === "pi")).toBe(false);
+ expect(directAcceptanceProfiles.some(({ adapterType }) =>
+ adapterType === "pi_local")).toBe(false);
+ });
+
+ it("contains no launch authority, provider credentials, or managed providers", () => {
+ const serialized = JSON.stringify({
+ profiles: [...directAcceptanceProfiles, ...nativeAcceptanceProfiles],
+ cases: runnerAcceptanceCases,
+ });
+ expect(serialized).not.toMatch(/claude_managed|aws_agentcore|daytona/i);
+ expect(serialized).not.toMatch(/api.?key|secret_ref|authorization/i);
+ });
+
+ it("builds a stable, unique matrix and applies recovery only to native profiles", () => {
+ expect(validateRunnerAcceptanceCatalog()).toEqual(runnerAcceptanceMatrix);
+ expect(new Set(runnerAcceptanceMatrix.map(({ id }) => id)).size)
+ .toBe(runnerAcceptanceMatrix.length);
+ expect(runnerAcceptanceMatrix
+ .filter(({ acceptanceCase }) => acceptanceCase.id === "flagged-native-recovery")
+ .every(({ profile }) => profile.generation === "native"))
+ .toBe(true);
+ });
+});
diff --git a/tests/runner-acceptance/catalog.ts b/tests/runner-acceptance/catalog.ts
new file mode 100644
index 0000000000..57978474dd
--- /dev/null
+++ b/tests/runner-acceptance/catalog.ts
@@ -0,0 +1,328 @@
+import { createHash } from "node:crypto";
+
+import { DEFAULT_CODEX_LOCAL_MODEL } from "../../packages/adapters/codex-local/src/index.js";
+import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js";
+import { QUALIFIED_OPENCODE_MODEL } from "../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.js";
+import { BUILTIN_ADAPTER_TYPES } from "../../server/src/adapters/builtin-adapter-types.js";
+
+import { findSensitiveValue } from "./redaction.js";
+import type {
+ JsonValue,
+ RunnerAcceptanceCase,
+ RunnerAcceptanceCell,
+ RunnerAcceptanceProfile,
+} from "./types.js";
+
+const excludedBuiltInAdapterTypes = new Set(["paperclip_runner", "pi_local"]);
+
+const directBuiltInAdapterTypes = [
+ "acpx_local",
+ "claude_local",
+ "codex_local",
+ "cursor_cloud",
+ "cursor",
+ "gemini_local",
+ "grok_local",
+ "hermes_gateway",
+ "hermes_local",
+ "kimi_local",
+ "openclaw_gateway",
+ "opencode_local",
+ "process",
+ "http",
+] as const;
+
+function directProfile(adapterType: string): RunnerAcceptanceProfile {
+ return {
+ id: `direct-${adapterType.replaceAll("_", "-")}`,
+ label: `Direct ${adapterType}`,
+ generation: "direct",
+ adapterScope: "built_in",
+ adapterType,
+ expectedRuntimeMode: "legacy",
+ provider: null,
+ model: null,
+ adapterConfig: {},
+ invariants: [
+ "runtime.mode=legacy",
+ "runnerd.start_count=0",
+ "native_record.count=0",
+ "finalization.path=direct_adapter",
+ "task_controls.mode=classic",
+ ],
+ };
+}
+
+function nativeProfile(input: {
+ id: string;
+ label: string;
+ provider: "codex" | "opencode" | "acpx";
+ model: string;
+ adapterConfig: Record<string, JsonValue>;
+}): RunnerAcceptanceProfile {
+ return {
+ ...input,
+ generation: "native",
+ adapterScope: "built_in",
+ adapterType: "paperclip_runner",
+ expectedRuntimeMode: "native",
+ invariants: [
+ "runtime.mode=native",
+ "runnerd.start_count=1",
+ "native_record.count>=1",
+ "finalization.path=native",
+ "provider.persisted=true",
+ ],
+ };
+}
+
+export const directAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
+ ...directBuiltInAdapterTypes.map(directProfile),
+ {
+ id: "direct-external-plugin-contract",
+ label: "Direct external plugin contract",
+ generation: "direct",
+ adapterScope: "external_plugin_contract",
+ adapterType: "external_plugin:*",
+ expectedRuntimeMode: "legacy",
+ provider: null,
+ model: null,
+ adapterConfig: {},
+ invariants: [
+ "runtime.mode=legacy",
+ "runnerd.start_count=0",
+ "native_record.count=0",
+ "finalization.path=plugin_adapter",
+ "task_controls.mode=classic",
+ ],
+ },
+] as const;
+
+export const nativeAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
+ nativeProfile({
+ id: "runner-codex",
+ label: "Paperclip Runner Codex",
+ provider: "codex",
+ model: DEFAULT_CODEX_LOCAL_MODEL,
+ adapterConfig: {
+ provider: "codex",
+ codexPermissionMode: "untrusted",
+ lifecycleMode: "per_turn",
+ },
+ }),
+ nativeProfile({
+ id: "runner-opencode",
+ label: "Paperclip Runner OpenCode",
+ provider: "opencode",
+ model: QUALIFIED_OPENCODE_MODEL,
+ adapterConfig: {
+ provider: "opencode",
+ model: QUALIFIED_OPENCODE_MODEL,
+ opencodePermissionMode: "ask",
+ lifecycleMode: "per_turn",
+ },
+ }),
+ nativeProfile({
+ id: "runner-acpx-claude",
+ label: "Paperclip Runner ACPX Claude",
+ provider: "acpx",
+ model: QUALIFIED_ACPX_PROFILES.claude.qualificationModel,
+ adapterConfig: {
+ provider: "acpx",
+ acpxAgent: "claude",
+ model: QUALIFIED_ACPX_PROFILES.claude.qualificationModel,
+ acpxPermissionMode: "approve-reads",
+ lifecycleMode: "per_turn",
+ },
+ }),
+ nativeProfile({
+ id: "runner-acpx-codex",
+ label: "Paperclip Runner ACPX Codex",
+ provider: "acpx",
+ model: QUALIFIED_ACPX_PROFILES.codex.qualificationModel,
+ adapterConfig: {
+ provider: "acpx",
+ acpxAgent: "codex",
+ model: QUALIFIED_ACPX_PROFILES.codex.qualificationModel,
+ acpxPermissionMode: "approve-reads",
+ lifecycleMode: "per_turn",
+ },
+ }),
+] as const;
+
+export const runnerAcceptanceProfiles: readonly RunnerAcceptanceProfile[] = [
+ ...directAcceptanceProfiles,
+ ...nativeAcceptanceProfiles,
+];
+
+export const runnerAcceptanceCases: readonly RunnerAcceptanceCase[] = [
+ {
+ id: "runtime-selection",
+ label: "Runtime selection remains isolated",
+ revision: 1,
+ appliesTo: "all",
+ assertions: [
+ "runtime_selection.is_stable",
+ "status_authority.matches_runtime",
+ "execution_occurs_once",
+ ],
+ },
+ {
+ id: "task-thread-baseline",
+ label: "Task thread preserves baseline behavior",
+ revision: 1,
+ appliesTo: "all",
+ assertions: [
+ "task_thread.active_state",
+ "task_thread.settled_state",
+ "task_thread.empty_transcript_state",
+ "task_thread.classic_interface_state",
+ ],
+ },
+ {
+ id: "question-round-trip",
+ label: "Structured question round trip",
+ revision: 1,
+ appliesTo: "all",
+ assertions: [
+ "question.rendered_once",
+ "question.response_validated",
+ "question.resume_occurs_once",
+ "finalization.byte_stable",
+ ],
+ },
+ {
+ id: "flagged-native-recovery",
+ label: "Persisted native recovery after flag change",
+ revision: 1,
+ appliesTo: "native",
+ assertions: [
+ "fresh_start.flag_required",
+ "persisted_run.readable_when_disabled",
+ "persisted_run.recoverable_when_disabled",
+ "provider.identity_does_not_drift",
+ ],
+ },
+] as const;
+
+function suiteDefinitionHash() {
+ return createHash("sha256")
+ .update(JSON.stringify({
+ profiles: runnerAcceptanceProfiles.map((profile) => ({
+ id: profile.id,
+ adapterType: profile.adapterType,
+ generation: profile.generation,
+ provider: profile.provider,
+ model: profile.model,
+ adapterConfig: profile.adapterConfig,
+ invariants: profile.invariants,
+ })),
+ cases: runnerAcceptanceCases.map((acceptanceCase) => ({
+ id: acceptanceCase.id,
+ revision: acceptanceCase.revision,
+ appliesTo: acceptanceCase.appliesTo,
+ assertions: acceptanceCase.assertions,
+ })),
+ }))
+ .digest("hex");
+}
+
+export const runnerAcceptanceSuiteDefinitionHash = suiteDefinitionHash();
+
+export function buildRunnerAcceptanceMatrix(): RunnerAcceptanceCell[] {
+ return runnerAcceptanceProfiles.flatMap((profile) =>
+ runnerAcceptanceCases
+ .filter((acceptanceCase) =>
+ acceptanceCase.appliesTo === "all"
+ || acceptanceCase.appliesTo === profile.generation)
+ .map((acceptanceCase) => ({
+ id: `${profile.id}.${acceptanceCase.id}`,
+ suiteId: "runner-compatibility" as const,
+ suiteDefinitionHash: runnerAcceptanceSuiteDefinitionHash,
+ profile,
+ acceptanceCase,
+ assertions: [...new Set([
+ ...profile.invariants,
+ ...acceptanceCase.assertions,
+ ])],
+ })),
+ );
+}
+
+function duplicates(values: readonly string[]) {
+ const seen = new Set<string>();
+ return values.filter((value) => {
+ if (seen.has(value)) return true;
+ seen.add(value);
+ return false;
+ });
+}
+
+function assertSafeConfig(value: JsonValue, path = "adapterConfig") {
+ if (typeof value === "string") {
+ const leak = findSensitiveValue(value);
+ if (leak) throw new Error(`Acceptance catalog contains ${leak} at ${path}`);
+ return;
+ }
+ if (Array.isArray(value)) {
+ value.forEach((entry, index) => assertSafeConfig(entry, `${path}[${index}]`));
+ return;
+ }
+ if (!value || typeof value !== "object") return;
+ for (const [key, entry] of Object.entries(value)) {
+ if (/(?:authorization|api.?key|token|password|secret|credential|env)/i.test(key)) {
+ throw new Error(`Acceptance catalog contains prohibited field ${path}.${key}`);
+ }
+ assertSafeConfig(entry, `${path}.${key}`);
+ }
+}
+
+export function validateRunnerAcceptanceCatalog() {
+ const duplicateProfileIds = duplicates(runnerAcceptanceProfiles.map(({ id }) => id));
+ if (duplicateProfileIds.length > 0) {
+ throw new Error(`Duplicate runner acceptance profile ids: ${duplicateProfileIds.join(", ")}`);
+ }
+ const duplicateCaseIds = duplicates(runnerAcceptanceCases.map(({ id }) => id));
+ if (duplicateCaseIds.length > 0) {
+ throw new Error(`Duplicate runner acceptance case ids: ${duplicateCaseIds.join(", ")}`);
+ }
+
+ const expectedDirectBuiltIns = new Set(
+ [...BUILTIN_ADAPTER_TYPES].filter((type) => !excludedBuiltInAdapterTypes.has(type)),
+ );
+ const catalogedDirectBuiltIns = new Set(
+ directAcceptanceProfiles
+ .filter(({ adapterScope }) => adapterScope === "built_in")
+ .map(({ adapterType }) => adapterType),
+ );
+ const missing = [...expectedDirectBuiltIns].filter((type) => !catalogedDirectBuiltIns.has(type));
+ const unexpected = [...catalogedDirectBuiltIns].filter((type) => !expectedDirectBuiltIns.has(type));
+ if (missing.length > 0 || unexpected.length > 0) {
+ throw new Error(
+ `Direct adapter catalog drift; missing=${missing.join(",") || "none"}; unexpected=${unexpected.join(",") || "none"}`,
+ );
+ }
+
+ for (const profile of runnerAcceptanceProfiles) {
+ assertSafeConfig(profile.adapterConfig);
+ if (profile.adapterType === "pi_local") {
+ throw new Error("Pi is outside the acceptance foundation boundary");
+ }
+ if (
+ profile.provider === "acpx"
+ && profile.adapterConfig.acpxAgent !== "claude"
+ && profile.adapterConfig.acpxAgent !== "codex"
+ ) {
+ throw new Error(`Unqualified ACPX profile ${profile.id}`);
+ }
+ }
+
+ const matrix = buildRunnerAcceptanceMatrix();
+ const duplicateCellIds = duplicates(matrix.map(({ id }) => id));
+ if (duplicateCellIds.length > 0) {
+ throw new Error(`Duplicate runner acceptance cell ids: ${duplicateCellIds.join(", ")}`);
+ }
+ return matrix;
+}
+
+export const runnerAcceptanceMatrix = validateRunnerAcceptanceCatalog();
diff --git a/tests/runner-acceptance/failure-classifier.ts b/tests/runner-acceptance/failure-classifier.ts
new file mode 100644
index 0000000000..b76b0741e6
--- /dev/null
+++ b/tests/runner-acceptance/failure-classifier.ts
@@ -0,0 +1,28 @@
+import type { FailureClass } from "./types.js";
+
+const TRANSIENT =
+ /(?:\b429\b|\b5\d\d\b|rate.?limit|ECONN(?:RESET|REFUSED)|socket hang up|network (?:error|interruption|timeout)|service unavailable|(?:provider|server|bootstrap|browser|webserver|health|connection|harness).*(?:temporar|timed? out|timeout|closed|failed|unavailable|interrupt|reset|refused|start|connect)|(?:timed? out|timeout).*(?:provider|server|bootstrap|browser|webserver|health|connection|harness))/i;
+const PERMANENT =
+ /(?:unauthorized|forbidden|qualification|model.*(?:unsupported|incompatible)|artifact.*incompatible|runner_remote_.*(?:incompatible|unavailable)|provider.*unsupported)/i;
+const CANDIDATE =
+ /(?:assertion|matcher|expected.*observed|marker|issue status|run status|runtime mode|wrong output|missing output)/i;
+
+export function classifyFailure(error: unknown): FailureClass {
+ const message = error instanceof Error
+ ? `${error.name}: ${error.message}`
+ : String(error);
+ if (/secret.*(?:leak|plaintext|redaction)/i.test(message)) return "secret_leak";
+ if (/cleanup|teardown|lease.*release/i.test(message)) {
+ return TRANSIENT.test(message)
+ ? "transient_infrastructure"
+ : "cleanup_failure";
+ }
+ if (PERMANENT.test(message)) return "permanent_infrastructure";
+ if (TRANSIENT.test(message)) return "transient_infrastructure";
+ if (CANDIDATE.test(message)) return "candidate_failure";
+ return "candidate_failure";
+}
+
+export function shouldRetryFailure(failureClass: FailureClass) {
+ return failureClass === "transient_infrastructure";
+}
diff --git a/tests/runner-acceptance/redaction.ts b/tests/runner-acceptance/redaction.ts
new file mode 100644
index 0000000000..becf0ca9aa
--- /dev/null
+++ b/tests/runner-acceptance/redaction.ts
@@ -0,0 +1,118 @@
+import { redactDiagnosticText } from "../../packages/adapter-utils/src/command-redaction.js";
+
+const SECRET_SHAPES = [
+ /\bsk-ant-[A-Za-z0-9_-]{16,}\b/g,
+ /\bsk-(?:proj-)?[A-Za-z0-9_-]{16,}\b/g,
+ /\bopenrouter[-_]?(?:api)?[-_]?key["'=:\s]+[A-Za-z0-9._-]{12,}\b/gi,
+] as const;
+const SENSITIVE_KEY = /(?:authorization|api.?key|token|password|secret|credential|private.?key)/i;
+
+export function normalizedSensitiveValues(
+ values: readonly (string | undefined)[],
+) {
+ return [...new Set(
+ values
+ .map((value) => value?.trim())
+ .filter((value): value is string => Boolean(value)),
+ )].sort((left, right) => right.length - left.length);
+}
+
+function redactKnownValuesAndShapes(
+ value: string,
+ sensitiveValues: readonly string[],
+) {
+ let redacted = value;
+ for (const sensitiveValue of normalizedSensitiveValues(sensitiveValues)) {
+ redacted = redacted.split(sensitiveValue).join("[REDACTED]");
+ }
+ for (const pattern of SECRET_SHAPES) {
+ pattern.lastIndex = 0;
+ redacted = redacted.replace(pattern, "[REDACTED_SECRET_SHAPE]");
+ }
+ return redacted;
+}
+
+export function redactText(
+ value: string,
+ sensitiveValues: readonly string[] = [],
+) {
+ return redactKnownValuesAndShapes(
+ redactDiagnosticText(value, "[REDACTED]"),
+ sensitiveValues,
+ );
+}
+
+export function findSensitiveValue(
+ value: string | Buffer,
+ sensitiveValues: readonly string[] = [],
+): string | null {
+ const text = Buffer.isBuffer(value) ? value.toString("utf8") : value;
+ for (const sensitiveValue of normalizedSensitiveValues(sensitiveValues)) {
+ if (text.includes(sensitiveValue)) return "exact sensitive value";
+ }
+ for (const pattern of SECRET_SHAPES) {
+ pattern.lastIndex = 0;
+ if (pattern.test(text)) return "secret-shaped value";
+ }
+ return null;
+}
+
+export function findSensitiveJsonValue(
+ value: unknown,
+ sensitiveValues: readonly string[] = [],
+): string | null {
+ if (typeof value === "string") return findSensitiveValue(value, sensitiveValues);
+ if (Array.isArray(value)) {
+ for (const entry of value) {
+ const leak = findSensitiveJsonValue(entry, sensitiveValues);
+ if (leak) return leak;
+ }
+ return null;
+ }
+ if (value && typeof value === "object") {
+ for (const [key, entry] of Object.entries(value)) {
+ if (
+ SENSITIVE_KEY.test(key)
+ && entry !== null
+ && entry !== undefined
+ && entry !== "[REDACTED]"
+ && entry !== "[REDACTED_SECRET_SHAPE]"
+ ) {
+ return `sensitive field ${key}`;
+ }
+ const leak = findSensitiveJsonValue(entry, sensitiveValues);
+ if (leak) return leak;
+ }
+ }
+ return null;
+}
+
+export function sanitizeJson(
+ value: unknown,
+ sensitiveValues: readonly string[] = [],
+): unknown {
+ if (typeof value === "string") return redactText(value, sensitiveValues);
+ if (Array.isArray(value)) {
+ return value.map((entry) => sanitizeJson(entry, sensitiveValues));
+ }
+ if (value && typeof value === "object") {
+ return Object.fromEntries(
+ Object.entries(value).map(([key, entry]) => [
+ key,
+ SENSITIVE_KEY.test(key) && entry !== null && entry !== undefined
+ ? "[REDACTED]"
+ : sanitizeJson(entry, sensitiveValues),
+ ]),
+ );
+ }
+ return value;
+}
+
+export function assertSensitiveValueFree(
+ value: string | Buffer,
+ sensitiveValues: readonly string[],
+ label: string,
+) {
+ const leak = findSensitiveValue(value, sensitiveValues);
+ if (leak) throw new Error(`Secret leak in ${label}: ${leak}`);
+}
diff --git a/tests/runner-acceptance/report.test.ts b/tests/runner-acceptance/report.test.ts
new file mode 100644
index 0000000000..e8c5bf7071
--- /dev/null
+++ b/tests/runner-acceptance/report.test.ts
@@ -0,0 +1,189 @@
+import { describe, expect, it } from "vitest";
+
+import { runnerAcceptanceMatrix } from "./catalog.js";
+import {
+ buildRunnerAcceptanceReport,
+ renderRunnerAcceptanceJUnit,
+ renderRunnerAcceptanceMarkdown,
+} from "./report.js";
+import type { RunnerAcceptanceCell, RunnerAcceptanceResult } from "./types.js";
+
+function passingResult(
+ cell: RunnerAcceptanceCell,
+ attempt = 1,
+): RunnerAcceptanceResult {
+ return {
+ schema: "paperclip.runner-acceptance.result/v1",
+ cellId: cell.id,
+ attempt,
+ status: "passed",
+ startedAt: "2026-09-01T00:00:00.000Z",
+ finishedAt: `2026-09-01T00:00:0${attempt}.000Z`,
+ durationMs: attempt * 1_000,
+ redaction: "passed",
+ assertions: cell.assertions.map((id) => ({ id, passed: true })),
+ };
+}
+
+describe("Runner acceptance report", () => {
+ const cells = runnerAcceptanceMatrix.slice(0, 2);
+
+ it("selects the newest attempt and reports missing cells without publishing evidence", () => {
+ const retry = passingResult(cells[0]!, 2);
+ const report = buildRunnerAcceptanceReport({
+ cells,
+ generatedAt: "2026-09-01T00:01:00.000Z",
+ results: [
+ {
+ ...passingResult(cells[0]!),
+ status: "failed",
+ failureClass: "transient_infrastructure",
+ error: "connection timed out",
+ },
+ retry,
+ ],
+ });
+
+ expect(report).toMatchObject({
+ selected: 2,
+ passed: 1,
+ failed: 1,
+ retries: 1,
+ });
+ expect(report.results[0]).toMatchObject({ attempt: 2, valid: true });
+ expect(report.results[1]).toMatchObject({
+ attempt: 0,
+ valid: false,
+ error: "No result was supplied",
+ });
+ });
+
+ it("fails closed on incomplete assertions or unsafe structured values", () => {
+ const cell = cells[0]!;
+ const incomplete = passingResult(cell);
+ const report = buildRunnerAcceptanceReport({
+ cells: [cell],
+ results: [{
+ ...incomplete,
+ error: "redacted diagnostic",
+ assertions: incomplete.assertions.slice(1),
+ }],
+ });
+ expect(report.failed).toBe(1);
+ expect(report.results[0]?.validationErrors).toContain(
+ `missing assertion ${cell.assertions[0]}`,
+ );
+
+ const unsafeResult = {
+ ...passingResult(cell),
+ diagnostic: { accessToken: "not-a-real-sensitive-value" },
+ } as unknown as RunnerAcceptanceResult;
+ const unsafe = buildRunnerAcceptanceReport({
+ cells: [cell],
+ results: [unsafeResult],
+ });
+ expect(unsafe.failed).toBe(1);
+ expect(unsafe.results[0]?.validationErrors).toContain(
+ "unsafe result payload: sensitive field accessToken",
+ );
+ expect(JSON.stringify(unsafe.results[0])).not.toContain(
+ "not-a-real-sensitive-value",
+ );
+ expect(unsafe.results[0]).not.toHaveProperty("diagnostic");
+ });
+
+ it("fails closed when assertions are missing or are not an array", () => {
+ const cell = cells[0]!;
+ const withoutAssertions = { ...passingResult(cell) } as Record<string, unknown>;
+ delete withoutAssertions.assertions;
+ const missing = buildRunnerAcceptanceReport({
+ cells: [cell],
+ results: [withoutAssertions],
+ });
+ const nonArray = buildRunnerAcceptanceReport({
+ cells: [cell],
+ results: [{ ...passingResult(cell), assertions: { passed: true } }],
+ });
+
+ for (const report of [missing, nonArray]) {
+ expect(report).toMatchObject({ selected: 1, passed: 0, failed: 1 });
+ expect(report.results[0]).toMatchObject({
+ valid: false,
+ assertions: [],
+ });
+ expect(report.results[0]?.validationErrors).toEqual(expect.arrayContaining([
+ "assertions must be an array",
+ `missing assertion ${cell.assertions[0]}`,
+ ]));
+ }
+ });
+
+ it("normalizes invalid scalar and assertion types without throwing", () => {
+ const cell = cells[0]!;
+ const report = buildRunnerAcceptanceReport({
+ cells: [cell],
+ results: [{
+ schema: 1,
+ cellId: cell.id,
+ attempt: "1",
+ status: true,
+ failureClass: 42,
+ error: { message: "failed" },
+ startedAt: 1,
+ finishedAt: null,
+ durationMs: "1000",
+ redaction: false,
+ assertions: [
+ null,
+ { id: cell.assertions[0], passed: "yes", detail: 42 },
+ ],
+ }],
+ });
+
+ expect(report).toMatchObject({ selected: 1, passed: 0, failed: 1 });
+ expect(report.results[0]).toMatchObject({
+ attempt: 0,
+ status: "failed",
+ startedAt: "",
+ finishedAt: "",
+ durationMs: 0,
+ redaction: "failed",
+ valid: false,
+ });
+ expect(report.results[0]?.validationErrors).toEqual(expect.arrayContaining([
+ "unsupported result schema",
+ "attempt must be a positive safe integer",
+ "duration must be a non-negative finite number",
+ "timestamps must be valid ISO-compatible values",
+ "status must be passed or failed",
+ "failureClass is unsupported",
+ "error must be a string when present",
+ "redaction did not pass",
+ "assertion 0 must be an object",
+ `assertion ${cell.assertions[0]} passed must be a boolean`,
+ `assertion ${cell.assertions[0]} detail must be a string when present`,
+ ]));
+ });
+
+ it("renders deterministic Markdown and JUnit summaries", () => {
+ const report = buildRunnerAcceptanceReport({
+ cells,
+ results: cells.map((cell) => passingResult(cell)),
+ generatedAt: "2026-09-01T00:01:00.000Z",
+ });
+ const markdown = renderRunnerAcceptanceMarkdown(report);
+ const junit = renderRunnerAcceptanceJUnit(report);
+
+ expect(markdown).toContain("Passed: 2/2");
+ expect(markdown).toContain(cells[0]!.id);
+ expect(junit).toContain('tests="2" failures="0"');
+ expect(junit).toContain('classname="runner-acceptance"');
+ });
+
+ it("rejects results for cells outside the declared catalog", () => {
+ expect(() => buildRunnerAcceptanceReport({
+ cells,
+ results: [{ ...passingResult(cells[0]!), cellId: "unknown.cell" }],
+ })).toThrow("unknown acceptance cells");
+ });
+});
diff --git a/tests/runner-acceptance/report.ts b/tests/runner-acceptance/report.ts
new file mode 100644
index 0000000000..8a99adc2a9
--- /dev/null
+++ b/tests/runner-acceptance/report.ts
@@ -0,0 +1,316 @@
+import { runnerAcceptanceMatrix } from "./catalog.js";
+import { findSensitiveJsonValue, redactText } from "./redaction.js";
+import type {
+ AggregatedAcceptanceResult,
+ FailureClass,
+ RunnerAcceptanceCell,
+ RunnerAcceptanceReport,
+ RunnerAcceptanceResult,
+} from "./types.js";
+
+const RESULT_SCHEMA = "paperclip.runner-acceptance.result/v1" as const;
+const ALLOWED_FAILURE_CLASSES = new Set<FailureClass>([
+ "candidate_failure",
+ "transient_infrastructure",
+ "permanent_infrastructure",
+ "secret_leak",
+ "cleanup_failure",
+]);
+
+function record(value: unknown): Record<string, unknown> | null {
+ return value !== null && typeof value === "object" && !Array.isArray(value)
+ ? value as Record<string, unknown>
+ : null;
+}
+
+function field(value: Record<string, unknown>, key: string): unknown {
+ try {
+ return value[key];
+ } catch {
+ return undefined;
+ }
+}
+
+function textField(value: Record<string, unknown>, key: string): string | null {
+ const candidate = field(value, key);
+ return typeof candidate === "string" ? candidate : null;
+}
+
+function numericField(value: Record<string, unknown>, key: string): number | null {
+ const candidate = field(value, key);
+ return typeof candidate === "number" && Number.isFinite(candidate)
+ ? candidate
+ : null;
+}
+
+function safeSensitivePayloadError(value: unknown): string | null {
+ try {
+ const leak = findSensitiveJsonValue(value);
+ return leak ? `unsafe result payload: ${leak}` : null;
+ } catch {
+ return "result payload could not be inspected safely";
+ }
+}
+
+function validationErrors(
+ value: unknown,
+ cell: RunnerAcceptanceCell,
+) {
+ const errors: string[] = [];
+ const result = record(value);
+ if (!result) return ["result must be an object"];
+
+ if (field(result, "schema") !== RESULT_SCHEMA) {
+ errors.push("unsupported result schema");
+ }
+ if (textField(result, "cellId") !== cell.id) {
+ errors.push("result cellId does not match the acceptance cell");
+ }
+ const attempt = numericField(result, "attempt");
+ if (attempt === null || !Number.isSafeInteger(attempt) || attempt < 1) {
+ errors.push("attempt must be a positive safe integer");
+ }
+ const durationMs = numericField(result, "durationMs");
+ if (durationMs === null || durationMs < 0) {
+ errors.push("duration must be a non-negative finite number");
+ }
+ const startedAt = textField(result, "startedAt");
+ const finishedAt = textField(result, "finishedAt");
+ if (
+ startedAt === null
+ || finishedAt === null
+ || !Number.isFinite(Date.parse(startedAt))
+ || !Number.isFinite(Date.parse(finishedAt))
+ ) {
+ errors.push("timestamps must be valid ISO-compatible values");
+ }
+ const status = field(result, "status");
+ if (status !== "passed" && status !== "failed") {
+ errors.push("status must be passed or failed");
+ }
+ const failureClass = field(result, "failureClass");
+ if (failureClass !== undefined && !ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass)) {
+ errors.push("failureClass is unsupported");
+ }
+ const error = field(result, "error");
+ if (error !== undefined && typeof error !== "string") {
+ errors.push("error must be a string when present");
+ }
+ if (field(result, "redaction") !== "passed") errors.push("redaction did not pass");
+ const sensitivePayloadError = safeSensitivePayloadError(value);
+ if (sensitivePayloadError) errors.push(sensitivePayloadError);
+
+ const rawAssertions = field(result, "assertions");
+ const assertions = Array.isArray(rawAssertions) ? rawAssertions : [];
+ if (!Array.isArray(rawAssertions)) errors.push("assertions must be an array");
+
+ const outcomes = new Map<string, { passed: boolean }>();
+ let duplicateAssertionId = false;
+ for (const [index, rawAssertion] of assertions.entries()) {
+ const assertion = record(rawAssertion);
+ if (!assertion) {
+ errors.push(`assertion ${index} must be an object`);
+ continue;
+ }
+ const id = textField(assertion, "id");
+ if (!id || id.trim().length === 0) {
+ errors.push(`assertion ${index} id must be a non-empty string`);
+ continue;
+ }
+ const passed = field(assertion, "passed");
+ if (typeof passed !== "boolean") {
+ errors.push(`assertion ${id} passed must be a boolean`);
+ }
+ const detail = field(assertion, "detail");
+ if (detail !== undefined && typeof detail !== "string") {
+ errors.push(`assertion ${id} detail must be a string when present`);
+ }
+ if (outcomes.has(id)) duplicateAssertionId = true;
+ else outcomes.set(id, { passed: passed === true });
+ }
+ if (duplicateAssertionId) errors.push("duplicate assertion ids");
+
+ for (const assertionId of cell.assertions) {
+ const outcome = outcomes.get(assertionId);
+ if (!outcome) errors.push(`missing assertion ${assertionId}`);
+ else if (!outcome.passed) errors.push(`failed assertion ${assertionId}`);
+ }
+ const unknownAssertions = [...outcomes.keys()]
+ .filter((id) => !cell.assertions.includes(id));
+ if (unknownAssertions.length > 0) {
+ errors.push(`unknown assertions: ${redactText(unknownAssertions.join(", "))}`);
+ }
+ if (status !== "passed") {
+ errors.push(
+ typeof error === "string" && error.length > 0
+ ? redactText(error)
+ : typeof failureClass === "string" && ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass)
+ ? failureClass
+ : "acceptance cell failed",
+ );
+ }
+ return errors;
+}
+
+function normalizedResult(
+ value: unknown,
+ cell: RunnerAcceptanceCell,
+): RunnerAcceptanceResult {
+ const result = record(value) ?? {};
+ const attempt = numericField(result, "attempt");
+ const durationMs = numericField(result, "durationMs");
+ const status = field(result, "status");
+ const failureClass = field(result, "failureClass");
+ const error = textField(result, "error");
+ const startedAt = textField(result, "startedAt");
+ const finishedAt = textField(result, "finishedAt");
+ const rawAssertions = field(result, "assertions");
+ const assertions = Array.isArray(rawAssertions) ? rawAssertions : [];
+ return {
+ schema: RESULT_SCHEMA,
+ cellId: cell.id,
+ attempt: attempt !== null && Number.isSafeInteger(attempt) && attempt >= 1
+ ? attempt
+ : 0,
+ status: status === "passed" ? "passed" : "failed",
+ ...(typeof failureClass === "string" && ALLOWED_FAILURE_CLASSES.has(failureClass as FailureClass)
+ ? { failureClass: failureClass as FailureClass }
+ : {}),
+ ...(error ? { error: redactText(error) } : {}),
+ startedAt: startedAt === null ? "" : redactText(startedAt),
+ finishedAt: finishedAt === null ? "" : redactText(finishedAt),
+ durationMs: durationMs !== null && durationMs >= 0
+ ? durationMs
+ : 0,
+ redaction: field(result, "redaction") === "passed" ? "passed" : "failed",
+ assertions: cell.assertions.flatMap((id) => {
+ const assertion = assertions
+ .map(record)
+ .find((candidate) => candidate && textField(candidate, "id") === id);
+ return assertion
+ ? [{
+ id,
+ passed: field(assertion, "passed") === true,
+ ...(typeof field(assertion, "detail") === "string"
+ ? { detail: redactText(field(assertion, "detail") as string) }
+ : {}),
+ }]
+ : [];
+ }),
+ };
+}
+
+function missingResult(cell: RunnerAcceptanceCell, generatedAt: string): AggregatedAcceptanceResult {
+ return {
+ schema: "paperclip.runner-acceptance.result/v1",
+ cellId: cell.id,
+ attempt: 0,
+ status: "failed",
+ failureClass: "permanent_infrastructure",
+ error: "No result was supplied",
+ startedAt: generatedAt,
+ finishedAt: generatedAt,
+ durationMs: 0,
+ redaction: "passed",
+ assertions: [],
+ valid: false,
+ validationErrors: ["No result was supplied"],
+ };
+}
+
+export function buildRunnerAcceptanceReport(input: {
+ results: readonly unknown[];
+ cells?: readonly RunnerAcceptanceCell[];
+ generatedAt?: string;
+}): RunnerAcceptanceReport {
+ const cells = input.cells ?? runnerAcceptanceMatrix;
+ const generatedAt = input.generatedAt ?? new Date().toISOString();
+ const candidates = new Map<string, Array<{ index: number; value: unknown }>>();
+ for (const [index, value] of input.results.entries()) {
+ const result = record(value);
+ const cellId = result ? textField(result, "cellId") : null;
+ const candidateCellId = cellId ?? `malformed-result-${index + 1}`;
+ candidates.set(candidateCellId, [
+ ...(candidates.get(candidateCellId) ?? []),
+ { index, value },
+ ]);
+ }
+
+ const selected = cells.map((cell) => {
+ const attempts = (candidates.get(cell.id) ?? []).sort((left, right) => {
+ const leftRecord = record(left.value) ?? {};
+ const rightRecord = record(right.value) ?? {};
+ const leftAttempt = numericField(leftRecord, "attempt") ?? 0;
+ const rightAttempt = numericField(rightRecord, "attempt") ?? 0;
+ const leftFinishedAt = textField(leftRecord, "finishedAt");
+ const rightFinishedAt = textField(rightRecord, "finishedAt");
+ const leftFinishedTime = leftFinishedAt === null ? 0 : Date.parse(leftFinishedAt) || 0;
+ const rightFinishedTime = rightFinishedAt === null ? 0 : Date.parse(rightFinishedAt) || 0;
+ return rightAttempt - leftAttempt
+ || rightFinishedTime - leftFinishedTime
+ || left.index - right.index;
+ });
+ const candidate = attempts[0];
+ if (!candidate) return missingResult(cell, generatedAt);
+ const errors = validationErrors(candidate.value, cell);
+ return {
+ ...normalizedResult(candidate.value, cell),
+ valid: errors.length === 0,
+ validationErrors: errors,
+ } satisfies AggregatedAcceptanceResult;
+ });
+
+ const knownCellIds = new Set(cells.map(({ id }) => id));
+ const unknownCellIds = [...candidates.keys()].filter((id) => !knownCellIds.has(id));
+ if (unknownCellIds.length > 0) {
+ throw new Error(`Results reference unknown acceptance cells: ${unknownCellIds.join(", ")}`);
+ }
+
+ return {
+ schema: "paperclip.runner-acceptance.report/v1",
+ generatedAt,
+ suiteDefinitionHash: cells[0]?.suiteDefinitionHash ?? "empty",
+ selected: selected.length,
+ passed: selected.filter(({ valid }) => valid).length,
+ failed: selected.filter(({ valid }) => !valid).length,
+ retries: selected.reduce((sum, result) => sum + Math.max(0, result.attempt - 1), 0),
+ results: selected,
+ };
+}
+
+function escapeMarkdown(value: string) {
+ return value.replaceAll("|", "\\|").replaceAll("\n", " ");
+}
+
+export function renderRunnerAcceptanceMarkdown(report: RunnerAcceptanceReport) {
+ return [
+ "# Runner acceptance",
+ "",
+ `Passed: ${report.passed}/${report.selected}`,
+ "",
+ "| Cell | Attempt | Result | Duration | Detail |",
+ "|---|---:|---|---:|---|",
+ ...report.results.map((result) =>
+ `| ${escapeMarkdown(result.cellId)} | ${result.attempt} | ${result.valid ? "pass" : "fail"} | ${Math.round(result.durationMs / 1000)}s | ${escapeMarkdown(result.validationErrors.join("; ") || "ok")} |`),
+ "",
+ ].join("\n");
+}
+
+function xml(value: unknown) {
+ return String(value ?? "")
+ .replaceAll("&", "&amp;")
+ .replaceAll("<", "&lt;")
+ .replaceAll(">", "&gt;")
+ .replaceAll('"', "&quot;")
+ .replaceAll("'", "&apos;");
+}
+
+export function renderRunnerAcceptanceJUnit(report: RunnerAcceptanceReport) {
+ const cases = report.results.map((result) => {
+ const failure = result.valid
+ ? ""
+ : `<failure message="${xml(result.validationErrors.join("; "))}"/>`;
+ return `<testcase classname="runner-acceptance" name="${xml(result.cellId)}" time="${result.durationMs / 1000}">${failure}</testcase>`;
+ }).join("");
+ return `<?xml version="1.0" encoding="UTF-8"?><testsuite name="Runner acceptance" tests="${report.selected}" failures="${report.failed}">${cases}</testsuite>\n`;
+}
diff --git a/tests/runner-acceptance/support.test.ts b/tests/runner-acceptance/support.test.ts
new file mode 100644
index 0000000000..78e793cbbc
--- /dev/null
+++ b/tests/runner-acceptance/support.test.ts
@@ -0,0 +1,62 @@
+import { describe, expect, it } from "vitest";
+
+import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js";
+import {
+ findSensitiveJsonValue,
+ findSensitiveValue,
+ normalizedSensitiveValues,
+ redactText,
+ sanitizeJson,
+} from "./redaction.js";
+
+describe("Runner acceptance failure classification", () => {
+ it.each([
+ ["assertion runtime mode expected legacy observed native", "candidate_failure"],
+ ["provider connection timed out", "transient_infrastructure"],
+ ["provider unsupported by this runner", "permanent_infrastructure"],
+ ["secret redaction leak in result", "secret_leak"],
+ ["teardown left a process running", "cleanup_failure"],
+ ] as const)("classifies %s", (message, expected) => {
+ expect(classifyFailure(new Error(message))).toBe(expected);
+ });
+
+ it("retries only transient infrastructure failures", () => {
+ expect(shouldRetryFailure("transient_infrastructure")).toBe(true);
+ expect(shouldRetryFailure("candidate_failure")).toBe(false);
+ expect(shouldRetryFailure("secret_leak")).toBe(false);
+ });
+});
+
+describe("Runner acceptance redaction", () => {
+ const fakeSensitiveValue = ["fixture", "sensitive", "value"].join("-");
+ const secretShapedValue = ["sk", "proj", "fixturevalue1234567890"].join("-");
+
+ it("deduplicates and orders known sensitive values longest first", () => {
+ expect(normalizedSensitiveValues([" short ", undefined, "long-value", "short"]))
+ .toEqual(["long-value", "short"]);
+ });
+
+ it("redacts exact values, secret shapes, and structured sensitive fields", () => {
+ expect(redactText(
+ `value=${fakeSensitiveValue} shaped=${secretShapedValue}`,
+ [fakeSensitiveValue],
+ )).toBe("value=[REDACTED] shaped=[REDACTED]");
+ expect(sanitizeJson({
+ nested: { accessToken: fakeSensitiveValue },
+ detail: `received ${secretShapedValue}`,
+ }, [fakeSensitiveValue])).toEqual({
+ nested: { accessToken: "[REDACTED]" },
+ detail: "received [REDACTED]",
+ });
+ });
+
+ it("detects exact, shaped, and sensitive-key leaks without scanning files", () => {
+ expect(findSensitiveValue(fakeSensitiveValue, [fakeSensitiveValue]))
+ .toBe("exact sensitive value");
+ expect(findSensitiveValue(secretShapedValue)).toBe("secret-shaped value");
+ expect(findSensitiveJsonValue({ password: "fixture-password" }))
+ .toBe("sensitive field password");
+ expect(findSensitiveJsonValue({ schema: "paperclip.runner-acceptance.result/v1" }))
+ .toBeNull();
+ });
+});
diff --git a/tests/runner-acceptance/tsconfig.json b/tests/runner-acceptance/tsconfig.json
new file mode 100644
index 0000000000..dceb2c2d5f
--- /dev/null
+++ b/tests/runner-acceptance/tsconfig.json
@@ -0,0 +1,13 @@
+{
+ "extends": "../../tsconfig.base.json",
+ "compilerOptions": {
+ "noEmit": true,
+ "rootDir": "../..",
+ "types": ["node"],
+ "typeRoots": [
+ "../../cli/node_modules/@types",
+ "../../server/node_modules/@types"
+ ]
+ },
+ "include": ["./**/*.ts"]
+}
diff --git a/tests/runner-acceptance/types.ts b/tests/runner-acceptance/types.ts
new file mode 100644
index 0000000000..1a7997d3c7
--- /dev/null
+++ b/tests/runner-acceptance/types.ts
@@ -0,0 +1,85 @@
+export type JsonValue =
+ | null
+ | boolean
+ | number
+ | string
+ | JsonValue[]
+ | { [key: string]: JsonValue };
+
+export type RunnerGeneration = "direct" | "native";
+export type ExpectedRuntimeMode = "legacy" | "native";
+export type NativeProvider = "codex" | "opencode" | "acpx";
+export type QualifiedAcpxAgent = "claude" | "codex";
+
+export interface RunnerAcceptanceProfile {
+ id: string;
+ label: string;
+ generation: RunnerGeneration;
+ adapterScope: "built_in" | "external_plugin_contract";
+ adapterType: string;
+ expectedRuntimeMode: ExpectedRuntimeMode;
+ provider: NativeProvider | null;
+ model: string | null;
+ adapterConfig: Readonly<Record<string, JsonValue>>;
+ invariants: readonly string[];
+}
+
+export interface RunnerAcceptanceCase {
+ id: string;
+ label: string;
+ revision: number;
+ appliesTo: "all" | RunnerGeneration;
+ assertions: readonly string[];
+}
+
+export interface RunnerAcceptanceCell {
+ id: string;
+ suiteId: "runner-compatibility";
+ suiteDefinitionHash: string;
+ profile: RunnerAcceptanceProfile;
+ acceptanceCase: RunnerAcceptanceCase;
+ assertions: readonly string[];
+}
+
+export type FailureClass =
+ | "candidate_failure"
+ | "transient_infrastructure"
+ | "permanent_infrastructure"
+ | "secret_leak"
+ | "cleanup_failure";
+
+export interface AcceptanceAssertionResult {
+ id: string;
+ passed: boolean;
+ detail?: string;
+}
+
+export interface RunnerAcceptanceResult {
+ schema: "paperclip.runner-acceptance.result/v1";
+ cellId: string;
+ attempt: number;
+ status: "passed" | "failed";
+ failureClass?: FailureClass;
+ error?: string;
+ startedAt: string;
+ finishedAt: string;
+ durationMs: number;
+ redaction: "passed" | "failed";
+ assertions: readonly AcceptanceAssertionResult[];
+}
+
+export interface AggregatedAcceptanceResult extends RunnerAcceptanceResult {
+ valid: boolean;
+ validationErrors: readonly string[];
+}
+
+export interface RunnerAcceptanceReport {
+ schema: "paperclip.runner-acceptance.report/v1";
+ generatedAt: string;
+ suiteDefinitionHash: string;
+ selected: number;
+ passed: number;
+ failed: number;
+ retries: number;
+ results: readonly AggregatedAcceptanceResult[];
+}
diff --git a/tests/runner-acceptance/vitest.config.ts b/tests/runner-acceptance/vitest.config.ts
new file mode 100644
index 0000000000..43dd349591
--- /dev/null
+++ b/tests/runner-acceptance/vitest.config.ts
@@ -0,0 +1,10 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+ root: import.meta.dirname,
+ test: {
+ name: "runner-acceptance",
+ include: ["**/*.test.ts"],
+ environment: "node",
+ },
+});