Skip to content

The open-source app everyone uses to manage agents at work

feat(runner): define package API and verification boundary (#12129)

Dottaffff1fe6e38b
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index b1f4138959..e356bb8a09 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -280,6 +280,9 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
+ - name: Verify Paperclip Runner
+ run: pnpm --filter @paperclipai/paperclip-runner check:all
+
- name: Build
run: pnpm build
diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml
index df146e7faa..55d457a24d 100644
--- a/.github/workflows/release-verify.yml
+++ b/.github/workflows/release-verify.yml
@@ -182,5 +182,8 @@ jobs:
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
+ - name: Verify Paperclip Runner
+ run: pnpm --filter @paperclipai/paperclip-runner check:all
+
- name: Build
run: pnpm build
diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md
index b66e45a521..9d3a5d83a5 100644
--- a/packages/paperclip-runner/README.md
+++ b/packages/paperclip-runner/README.md
@@ -22,8 +22,16 @@ undiscoverable because no production application binding or server authority
has landed. Catalog membership alone does not grant authority. See
[`SEMANTIC_ACTIONS.md`](SEMANTIC_ACTIONS.md) for the catalog boundary.
-The root export is intentionally narrow. The `./testing` entry point and package
-release boundary will arrive with the later package-boundary change.
+The package has two initial public surfaces:
+
+- `@paperclipai/paperclip-runner` contains runtime contracts, validation,
+ replay/reducer logic, the semantic catalog, and the authorization dispatcher.
+- `@paperclipai/paperclip-runner/testing` adds Node-only fixture loading and a
+ provider-neutral semantic conformance kit for deterministic test adapters.
+
+No SDK, browser, React, eval, live-console, lab, or provider-experiment entry
+point is exported. The package remains private in this wave, and no production
+adapter starts it yet.
Run the complete contract gate with:
@@ -37,9 +45,11 @@ Run the Rust runner gate with:
pnpm --filter @paperclipai/paperclip-runner check:runner
```
-This command checks Rust formatting, builds and tests the minimal workspace,
-verifies bounded process cleanup, exercises the fake local runner, and compares
-the Rust conformance and replay summaries with the shared fixtures.
+This command checks Rust formatting, builds and tests the minimal workspace in
+release mode, verifies bounded process cleanup, launches the real
+`paperclip-runnerd` binary through the fake harness, and compares the Rust
+conformance and replay summaries with the shared fixtures. The checked-in Cargo
+lock and pinned Rust toolchain keep this verification reproducible.
Durability and failure semantics are documented in
[`runner/DURABLE_TRANSPORT.md`](runner/DURABLE_TRANSPORT.md). The fault suite
diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json
index 0d497231f3..626d410d8e 100644
--- a/packages/paperclip-runner/package.json
+++ b/packages/paperclip-runner/package.json
@@ -11,8 +11,13 @@
".": {
"types": "./dist/index.d.ts",
"import": "./dist/index.js"
+ },
+ "./testing": {
+ "types": "./dist/testing.d.ts",
+ "import": "./dist/testing.js"
}
},
+ "sideEffects": false,
"files": [
"dist",
"protocol",
@@ -27,7 +32,7 @@
"typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace",
"test": "pnpm run test:typescript && pnpm run test:rust",
"test:typescript": "node --test test/protocol-contract.test.mjs && vitest run",
- "test:rust": "cargo test --manifest-path runner/Cargo.toml --locked --workspace",
+ "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace",
"test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider",
"test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::",
"generate:protocol-manifest": "node scripts/generate-protocol-manifest.mjs",
diff --git a/packages/paperclip-runner/rust-toolchain.toml b/packages/paperclip-runner/rust-toolchain.toml
new file mode 100644
index 0000000000..cf01515b3a
--- /dev/null
+++ b/packages/paperclip-runner/rust-toolchain.toml
@@ -0,0 +1,4 @@
+[toolchain]
+channel = "1.97.1"
+profile = "minimal"
+components = ["rustfmt"]
diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts
new file mode 100644
index 0000000000..d44c42184f
--- /dev/null
+++ b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts
@@ -0,0 +1,124 @@
+import { describe, expect, it } from "vitest";
+
+import {
+ SemanticConformanceMismatchError,
+ runSemanticConformanceKit,
+ type SemanticConformanceAdapter,
+ type SemanticConformanceObservation,
+} from "./semantic-conformance.js";
+
+const allowed: SemanticConformanceObservation = {
+ authorization: { outcome: "allowed" },
+ state: { task: { status: "done" } },
+ effects: [{ kind: "issue_status", status: "done" }],
+ audit: [{ action: "finish_task" }],
+};
+
+function adapter(
+ id: string,
+ observation: SemanticConformanceObservation,
+): SemanticConformanceAdapter {
+ return { id, execute: async () => structuredClone(observation) };
+}
+
+describe("semantic conformance kit", () => {
+ it("accepts equivalent observations independent of object key order", async () => {
+ const report = await runSemanticConformanceKit({
+ vectors: [
+ {
+ id: "finish",
+ operationId: "finish_task",
+ input: { summary: "done" },
+ },
+ ],
+ adapters: [
+ adapter("mock", allowed),
+ adapter("real", {
+ audit: [{ action: "finish_task" }],
+ effects: [{ status: "done", kind: "issue_status" }],
+ state: { task: { status: "done" } },
+ authorization: { outcome: "allowed" },
+ }),
+ ],
+ });
+
+ expect(report.schema).toBe("paperclip.semantic-conformance-report.v1");
+ expect(report.rows).toHaveLength(1);
+ expect(report.rows[0]?.adapterIds).toEqual(["mock", "real"]);
+ });
+
+ it("fails explicitly when adapters diverge", async () => {
+ await expect(
+ runSemanticConformanceKit({
+ vectors: [{ id: "finish", operationId: "finish_task", input: {} }],
+ adapters: [
+ adapter("mock", allowed),
+ adapter("real", {
+ ...allowed,
+ authorization: { outcome: "denied", code: "forbidden" },
+ }),
+ ],
+ }),
+ ).rejects.toBeInstanceOf(SemanticConformanceMismatchError);
+ });
+
+ it("requires at least two adapters", async () => {
+ await expect(
+ runSemanticConformanceKit({
+ vectors: [],
+ adapters: [adapter("only", allowed)],
+ }),
+ ).rejects.toThrow("semantic_conformance_requires_two_adapters");
+ });
+
+ it("requires unique adapter identities", async () => {
+ await expect(
+ runSemanticConformanceKit({
+ vectors: [],
+ adapters: [
+ adapter("duplicate", allowed),
+ adapter("duplicate", allowed),
+ ],
+ }),
+ ).rejects.toThrow("semantic_conformance_adapter_ids_must_be_unique");
+ });
+
+ it("fails closed for non-JSON normalized observations", async () => {
+ await expect(
+ runSemanticConformanceKit({
+ vectors: [{ id: "finish", operationId: "finish_task", input: {} }],
+ adapters: [
+ adapter("mock", allowed),
+ {
+ id: "invalid",
+ execute: async () =>
+ ({
+ ...allowed,
+ state: new Date(),
+ }) as unknown as SemanticConformanceObservation,
+ },
+ ],
+ }),
+ ).rejects.toThrow("semantic_conformance_non_json_observation");
+ });
+
+ it("fails closed for sparse normalized arrays", async () => {
+ const sparseEffects = Array(1);
+ await expect(
+ runSemanticConformanceKit({
+ vectors: [{ id: "finish", operationId: "finish_task", input: {} }],
+ adapters: [
+ adapter("mock", allowed),
+ {
+ id: "invalid",
+ execute: async () =>
+ ({
+ ...allowed,
+ effects: sparseEffects,
+ }) as unknown as SemanticConformanceObservation,
+ },
+ ],
+ }),
+ ).rejects.toThrow("semantic_conformance_non_json_observation");
+ });
+});
diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.ts
new file mode 100644
index 0000000000..38408d6d98
--- /dev/null
+++ b/packages/paperclip-runner/src/conformance/semantic-conformance.ts
@@ -0,0 +1,167 @@
+export type SemanticConformanceAuthorization =
+ | { readonly outcome: "allowed" }
+ | { readonly outcome: "denied"; readonly code: string };
+
+export type SemanticConformanceJsonValue =
+ | null
+ | boolean
+ | number
+ | string
+ | readonly SemanticConformanceJsonValue[]
+ | { readonly [key: string]: SemanticConformanceJsonValue };
+
+export interface SemanticConformanceVector {
+ readonly id: string;
+ readonly operationId: string;
+ readonly input: SemanticConformanceJsonValue;
+}
+
+export interface SemanticConformanceObservation {
+ readonly authorization: SemanticConformanceAuthorization;
+ readonly state: SemanticConformanceJsonValue;
+ readonly effects: readonly SemanticConformanceJsonValue[];
+ readonly audit: readonly SemanticConformanceJsonValue[];
+}
+
+export interface SemanticConformanceAdapter {
+ readonly id: string;
+ execute(
+ vector: SemanticConformanceVector,
+ ): Promise<SemanticConformanceObservation>;
+}
+
+export interface SemanticConformanceReportRow {
+ readonly vectorId: string;
+ readonly operationId: string;
+ readonly adapterIds: readonly string[];
+ readonly observation: SemanticConformanceObservation;
+}
+
+export interface SemanticConformanceReport {
+ readonly schema: "paperclip.semantic-conformance-report.v1";
+ readonly rows: readonly SemanticConformanceReportRow[];
+}
+
+export class SemanticConformanceMismatchError extends Error {
+ readonly code = "semantic_conformance_mismatch" as const;
+
+ constructor(
+ readonly vectorId: string,
+ readonly baselineAdapterId: string,
+ readonly mismatchedAdapterId: string,
+ ) {
+ super(
+ `Semantic conformance mismatch for ${vectorId}: ${mismatchedAdapterId} differs from ${baselineAdapterId}`,
+ );
+ this.name = "SemanticConformanceMismatchError";
+ }
+}
+
+/**
+ * Compare normalized authorization, state, effects, and audit output from two
+ * or more adapters. Adapters own setup and normalization; the kit owns a
+ * deterministic, provider-neutral comparison.
+ */
+export async function runSemanticConformanceKit(input: {
+ readonly vectors: readonly SemanticConformanceVector[];
+ readonly adapters: readonly SemanticConformanceAdapter[];
+}): Promise<SemanticConformanceReport> {
+ if (input.adapters.length < 2) {
+ throw new Error("semantic_conformance_requires_two_adapters");
+ }
+ if (
+ new Set(input.adapters.map((adapter) => adapter.id)).size !==
+ input.adapters.length
+ ) {
+ throw new Error("semantic_conformance_adapter_ids_must_be_unique");
+ }
+
+ const rows: SemanticConformanceReportRow[] = [];
+ for (const vector of input.vectors) {
+ const observations = await Promise.all(
+ input.adapters.map(async (adapter) => ({
+ adapter,
+ observation: await adapter.execute(vector),
+ })),
+ );
+ const baseline = observations[0]!;
+ const baselineJson = canonicalJson(baseline.observation);
+ for (const candidate of observations.slice(1)) {
+ if (canonicalJson(candidate.observation) !== baselineJson) {
+ throw new SemanticConformanceMismatchError(
+ vector.id,
+ baseline.adapter.id,
+ candidate.adapter.id,
+ );
+ }
+ }
+ rows.push(
+ Object.freeze({
+ vectorId: vector.id,
+ operationId: vector.operationId,
+ adapterIds: Object.freeze(input.adapters.map((adapter) => adapter.id)),
+ observation: structuredClone(baseline.observation),
+ }),
+ );
+ }
+
+ return Object.freeze({
+ schema: "paperclip.semantic-conformance-report.v1",
+ rows: Object.freeze(rows),
+ });
+}
+
+function canonicalJson(
+ value: unknown,
+ ancestors = new WeakSet<object>(),
+): string {
+ if (
+ value === null ||
+ typeof value === "boolean" ||
+ typeof value === "string"
+ ) {
+ return JSON.stringify(value);
+ }
+ if (typeof value === "number") {
+ if (!Number.isFinite(value)) throw invalidObservation();
+ return JSON.stringify(value);
+ }
+ if (typeof value !== "object") throw invalidObservation();
+ if (ancestors.has(value)) {
+ throw new Error("semantic_conformance_cyclic_observation");
+ }
+
+ const prototype = Object.getPrototypeOf(value);
+ if (
+ !Array.isArray(value) &&
+ prototype !== Object.prototype &&
+ prototype !== null
+ ) {
+ throw invalidObservation();
+ }
+ ancestors.add(value);
+ try {
+ if (Array.isArray(value)) {
+ const entries: string[] = [];
+ for (let index = 0; index < value.length; index += 1) {
+ if (!Object.hasOwn(value, index)) throw invalidObservation();
+ entries.push(canonicalJson(value[index], ancestors));
+ }
+ return `[${entries.join(",")}]`;
+ }
+ const record = value as Record<string, unknown>;
+ return `{${Object.keys(record)
+ .sort()
+ .map(
+ (key) =>
+ `${JSON.stringify(key)}:${canonicalJson(record[key], ancestors)}`,
+ )
+ .join(",")}}`;
+ } finally {
+ ancestors.delete(value);
+ }
+}
+
+function invalidObservation(): Error {
+ return new Error("semantic_conformance_non_json_observation");
+}
diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts
index 27d6eb7b86..c43eec2060 100644
--- a/packages/paperclip-runner/src/index.ts
+++ b/packages/paperclip-runner/src/index.ts
@@ -2,7 +2,6 @@ export * from "./catalog/index.js";
export * from "./contracts/completion-result.js";
export * from "./contracts/question-set.js";
export * from "./protocol/replay-contract.js";
-export * from "./protocol/replay-loader.js";
export * from "./protocol/result-normalization.js";
export * from "./reducer/session-reducer.js";
export * from "./semantic-tools/index.js";
diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts
new file mode 100644
index 0000000000..68ab11bfac
--- /dev/null
+++ b/packages/paperclip-runner/src/testing.ts
@@ -0,0 +1,10 @@
+/**
+ * Public test-only surface for deterministic fixtures and conformance kits.
+ *
+ * Production consumers import the package root. Tests import this explicit
+ * subpath so Node-only fixture loading and comparison helpers cannot become an
+ * accidental production dependency.
+ */
+export * from "./index.js";
+export * from "./conformance/semantic-conformance.js";
+export * from "./protocol/replay-loader.js";
diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs
index d32f08ebcc..7ddc23bf08 100644
--- a/scripts/__tests__/release-verify-workflow.test.mjs
+++ b/scripts/__tests__/release-verify-workflow.test.mjs
@@ -104,6 +104,7 @@ test("release verify workflow covers the same split test surface as stable PR ve
assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/);
assert.match(verifyWorkflow, /pnpm -r typecheck/);
assert.match(verifyWorkflow, /pnpm build/);
+ assert.match(verifyWorkflow, /pnpm --filter @paperclipai\/paperclip-runner check:all/);
for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) {
assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
diff --git a/server/package.json b/server/package.json
index e7d9b3f610..e4cc6bf6d0 100644
--- a/server/package.json
+++ b/server/package.json
@@ -36,7 +36,7 @@
"dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts",
"prepare:ui-dist": "bash ../scripts/prepare-server-ui-dist.sh",
"build": "tsc && mkdir -p dist/onboarding-assets dist/built-ins dist/services/scripts && cp -R src/onboarding-assets/. dist/onboarding-assets/ && cp -R src/built-ins/. dist/built-ins/ && cp -R src/services/scripts/. dist/services/scripts/ && node scripts/write-build-stamp.mjs",
- "prepack": "pnpm run prepare:ui-dist",
+ "prepack": "pnpm run prepare:ui-dist && pnpm run build",
"postpack": "rm -rf ui-dist",
"clean": "rm -rf dist",
"start": "node dist/index.js",
diff --git a/server/src/__tests__/server-package-build-script.test.ts b/server/src/__tests__/server-package-build-script.test.ts
index 9c4d16a916..92b410ce70 100644
--- a/server/src/__tests__/server-package-build-script.test.ts
+++ b/server/src/__tests__/server-package-build-script.test.ts
@@ -5,6 +5,14 @@ import { describe, expect, it } from "vitest";
const packageJsonPath = fileURLToPath(new URL("../../package.json", import.meta.url));
describe("server package build script", () => {
+ it("builds the compiled package entry during prepack", () => {
+ const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as {
+ scripts?: Record<string, string>;
+ };
+
+ expect(packageJson.scripts?.prepack).toBe("pnpm run prepare:ui-dist && pnpm run build");
+ });
+
it("copies static runtime asset directories into dist", () => {
const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as {
scripts?: Record<string, string>;