Skip to content

The open-source app everyone uses to manage agents at work

fix: run the cargo-building native-runner CI suite in the Rust-cached vitest lane (#13586)

Devin Foley352153b5edf0
diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json
index e7fbc955dc..fbf51aa98b 100644
--- a/packages/paperclip-runner/package.json
+++ b/packages/paperclip-runner/package.json
@@ -75,7 +75,7 @@
"test": "pnpm run test:typescript && pnpm run test:rust",
"test:typescript": "pnpm run test:typescript:prep && vitest run",
"test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs",
- "test:typescript:vitest": "pnpm run ensure:eval-build-deps && pnpm run build:rust && vitest run",
+ "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs",
"test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace",
"test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider",
"test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::",
diff --git a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs
new file mode 100644
index 0000000000..e383bd9024
--- /dev/null
+++ b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs
@@ -0,0 +1,69 @@
+// Runs this package's vitest shard for a Verify Paperclip Runner lane, then —
+// on the final shard of the PR workflow only — the server package's
+// general-server-native-runner group.
+//
+// That server suite rebuilds the Runner release binaries with cargo in
+// beforeAll. The PR workflow's plain General tests server shards carry no
+// Rust cache, so hosting it there cold-compiled every third-party crate on
+// each run (277s of a 291s shard vitest step, actions run 35246999382,
+// 2026-09-17) and made that shard the slowest check of the whole run. The
+// Verify Paperclip Runner lanes already restore the shared release-runner-v1
+// Rust cache read-only, which turns that build into an incremental rebuild,
+// and the workflow files themselves list this lane's command as a package
+// script — so the suite moves here without a workflow-file change.
+//
+// Contract, mirrored in scripts/run-vitest-stable.mjs (prWorkflowName) and
+// pinned by scripts/__tests__/run-vitest-stable-shard.test.mjs:
+// - Only the PR workflow (pr.yml, whose GITHUB_WORKFLOW the reusable
+// pr-trusted.yml jobs inherit) excludes the suite from the server shards,
+// and only there does this wrapper run it. Any other caller — local runs,
+// release-verify.yml — keeps the suite in the server group, so a renamed
+// workflow degrades to the slower covered path instead of losing coverage.
+// - The suite runs on the lane whose --shard=N/M has N === M (or an unsharded
+// invocation), so exactly one PR lane carries it.
+import { spawnSync } from "node:child_process";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const workspaceRoot = resolve(packageRoot, "../..");
+const args = process.argv.slice(2).filter((value) => value !== "--dry-run");
+const dryRun = process.argv.includes("--dry-run");
+
+const shardArg = args.find((value) => value.startsWith("--shard="));
+const shardMatch = shardArg ? /^--shard=(\d+)\/(\d+)$/.exec(shardArg) : null;
+if (shardArg && !shardMatch) {
+ console.error(`[pr-vitest-lane] unrecognized shard argument: ${shardArg}`);
+ process.exit(1);
+}
+const isFinalShard = !shardArg || shardMatch[1] === shardMatch[2];
+const isPrWorkflow = process.env.GITHUB_WORKFLOW === "PR";
+const runNativeRunnerGroup = isPrWorkflow && isFinalShard;
+
+const plannedCommands = [
+ { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: packageRoot },
+ { command: "pnpm", args: ["run", "build:rust"], cwd: packageRoot },
+ { command: "pnpm", args: ["exec", "vitest", "run", ...args], cwd: packageRoot },
+ ...(runNativeRunnerGroup
+ ? [{
+ command: "pnpm",
+ args: ["test:run:general", "--", "--group", "general-server-native-runner"],
+ cwd: workspaceRoot,
+ }]
+ : []),
+];
+
+if (dryRun) {
+ console.log(JSON.stringify({ isPrWorkflow, isFinalShard, runNativeRunnerGroup, plannedCommands }, null, 2));
+ process.exit(0);
+}
+
+for (const planned of plannedCommands) {
+ const result = spawnSync(planned.command, planned.args, {
+ cwd: planned.cwd,
+ stdio: "inherit",
+ });
+ if (result.status !== 0) {
+ process.exit(result.status ?? 1);
+ }
+}
diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs
index f320fa4859..ffe7fb2d0b 100644
--- a/scripts/__tests__/run-vitest-stable-shard.test.mjs
+++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs
@@ -1,5 +1,6 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
+import { readFileSync } from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import test from "node:test";
@@ -21,16 +22,29 @@ const serializedDurationsManifest = path.join(
"serialized-shard-durations.json",
);
-function dryRun(args) {
+// Membership of general-server-without-chat depends on GITHUB_WORKFLOW (see
+// prWorkflowName in run-vitest-stable.mjs), so strip the ambient value and
+// make every test pin the caller it mirrors explicitly — these tests
+// themselves run inside a workflow on CI.
+function workflowEnv(envOverrides = {}) {
+ const env = { ...process.env, ...envOverrides };
+ if (!("GITHUB_WORKFLOW" in envOverrides)) {
+ delete env.GITHUB_WORKFLOW;
+ }
+ return env;
+}
+
+function dryRun(args, envOverrides = {}) {
const result = spawnSync(process.execPath, [script, ...args, "--dry-run"], {
cwd: repoRoot,
encoding: "utf8",
+ env: workflowEnv(envOverrides),
});
return result;
}
-function dryRunJson(args) {
- const result = dryRun(args);
+function dryRunJson(args, envOverrides = {}) {
+ const result = dryRun(args, envOverrides);
assert.equal(result.status, 0, `expected success for ${args.join(" ")}: ${result.stderr}`);
return JSON.parse(result.stdout);
}
@@ -222,11 +236,15 @@ test("the real serialized shard partition is duration-balanced", () => {
test("the real shard partition is duration-balanced", () => {
// Mirrors the PR matrix: general-server-without-chat across SHARD_COUNT
- // runners, with the chat suite carried by the dedicated general-chat lanes.
+ // runners, with the chat suite carried by the dedicated general-chat lanes
+ // and the native-runner suite by the Rust-cached PR vitest lane.
const durations = loadShardDurations(durationsManifest);
const fallback = defaultSuiteWeight(durations);
const shards = Array.from({ length: SHARD_COUNT }, (_, index) =>
- dryRunJson(["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)]),
+ dryRunJson(
+ ["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)],
+ { GITHUB_WORKFLOW: "PR" },
+ ),
);
const totals = shards.map((shard) =>
@@ -235,10 +253,15 @@ test("the real shard partition is duration-balanced", () => {
const maxTotal = Math.max(...totals);
const minTotal = Math.min(...totals);
// LPT keeps the spread within the heaviest single suite; use that as the
- // bound. The chat suite runs in its own lanes, so exclude it here.
+ // bound. The chat and native-runner suites run in their own lanes, so
+ // exclude them here.
const chat = "server/src/__tests__/chat-channels.integration.test.ts";
+ const nativeRunner =
+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";
const heaviest = Math.max(
- ...Object.entries(durations).filter(([file]) => file !== chat).map(([, ms]) => ms),
+ ...Object.entries(durations)
+ .filter(([file]) => file !== chat && file !== nativeRunner)
+ .map(([, ms]) => ms),
);
assert.ok(
maxTotal - minTotal <= heaviest,
@@ -247,24 +270,131 @@ test("the real shard partition is duration-balanced", () => {
});
-// 12 mirrors pr-trusted.yml, 10 mirrors release-verify.yml.
-for (const withoutChatShardCount of [10, 12]) {
- test(`${withoutChatShardCount} without-chat shards plus the dedicated chat file cover the original server group exactly`, () => {
- const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]);
- const shards = Array.from({ length: withoutChatShardCount }, (_, index) => dryRunJson([
+const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts";
+const nativeRunnerSuitePath =
+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";
+
+// Mirrors pr-trusted.yml (12 shards, called by pr.yml so GITHUB_WORKFLOW is
+// "PR"): the chat suite runs in its dedicated lanes and the cargo-dependent
+// native-runner suite in the Rust-cached final Verify Paperclip Runner vitest
+// shard, so together the three cover the full server group exactly.
+test("12 PR without-chat shards plus the dedicated chat and native-runner lanes cover the original server group exactly", () => {
+ const prEnv = { GITHUB_WORKFLOW: "PR" };
+ const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], prEnv);
+ const shards = Array.from({ length: 12 }, (_, index) => dryRunJson([
+ "--mode", "general", "--group", "general-server-without-chat",
+ "--shard-index", String(index), "--shard-count", "12",
+ ], prEnv));
+ const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);
+ assert.ok(!files.includes(chatSuitePath));
+ assert.ok(!files.includes(nativeRunnerSuitePath));
+ assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath].sort(), full.selectedGeneralServerSuites.sort());
+ assert.equal(new Set(files).size, files.length);
+ const defaultRun = dryRunJson([], prEnv);
+ assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
+});
+
+// Mirrors release-verify.yml (10 shards, called by the Release and Cloud
+// readiness workflows) and local runs: no Rust-cached vitest lane exists
+// there, so the native-runner suite must stay in the server shards.
+for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" }], ["no ambient workflow", {}]]) {
+ test(`10 without-chat shards under ${caller} keep the native-runner suite and cover the server group with chat alone`, () => {
+ const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], envOverrides);
+ const shards = Array.from({ length: 10 }, (_, index) => dryRunJson([
"--mode", "general", "--group", "general-server-without-chat",
- "--shard-index", String(index), "--shard-count", String(withoutChatShardCount),
- ]));
+ "--shard-index", String(index), "--shard-count", "10",
+ ], envOverrides));
const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);
- const chat = "server/src/__tests__/chat-channels.integration.test.ts";
- assert.ok(!files.includes(chat));
- assert.deepEqual([...files, chat].sort(), full.selectedGeneralServerSuites.sort());
+ assert.ok(!files.includes(chatSuitePath));
+ assert.ok(files.includes(nativeRunnerSuitePath));
+ assert.deepEqual([...files, chatSuitePath].sort(), full.selectedGeneralServerSuites.sort());
assert.equal(new Set(files).size, files.length);
- const defaultRun = dryRunJson([]);
- assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);
});
}
+test("the native-runner lane runs exactly the cargo-dependent vertical-slice suite", () => {
+ const lane = dryRunJson(["--mode", "general", "--group", "general-server-native-runner"]);
+ assert.deepEqual(lane.selectedGeneralServerSuites, [
+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts",
+ ]);
+});
+
+test("shard flags are rejected for the native-runner group", () => {
+ const result = dryRun(["--mode", "general", "--group", "general-server-native-runner", "--shard-index", "0", "--shard-count", "2"]);
+ assert.notEqual(result.status, 0, "the native-runner lane is a single suite and must not accept shard flags");
+});
+
+// The PR-side exclusion above is safe only while the wiring it assumes holds:
+// pr.yml (the caller whose name reusable pr-trusted.yml jobs see as
+// GITHUB_WORKFLOW) is named PR, the sharded vitest lanes partition cleanly
+// with exactly one final shard, and that lane's package script routes through
+// the wrapper that runs the native-runner group.
+test("the PR workflow wiring for the native-runner lane holds", () => {
+ const prWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr.yml"), "utf8");
+ assert.match(prWorkflow, /^name: PR$/m,
+ "renaming pr.yml silently moves the native-runner suite back into the uncached server shards");
+
+ const trustedWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr-trusted.yml"), "utf8");
+ const lanes = [...trustedWorkflow.matchAll(/command: test:typescript:vitest --shard=(\d+)\/(\d+)/g)]
+ .map((match) => [Number(match[1]), Number(match[2])]);
+ assert.ok(lanes.length > 0, "expected sharded test:typescript:vitest lanes in pr-trusted.yml");
+ assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "vitest lanes must agree on the shard count");
+ const shardCount = lanes[0][1];
+ assert.deepEqual(
+ lanes.map(([index]) => index).sort((left, right) => left - right),
+ Array.from({ length: shardCount }, (_, index) => index + 1),
+ "vitest lanes must cover every shard exactly once",
+ );
+ assert.equal(lanes.filter(([index, count]) => index === count).length, 1,
+ "exactly one final vitest shard carries the native-runner group");
+
+ const runnerPackage = JSON.parse(
+ readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8"),
+ );
+ assert.equal(runnerPackage.scripts["test:typescript:vitest"], "node ./scripts/run-pr-vitest-lane.mjs");
+});
+
+const laneWrapper = path.join(repoRoot, "packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs");
+
+function wrapperPlan(args, envOverrides = {}) {
+ const result = spawnSync(process.execPath, [laneWrapper, ...args, "--dry-run"], {
+ cwd: repoRoot,
+ encoding: "utf8",
+ env: workflowEnv(envOverrides),
+ });
+ assert.equal(result.status, 0, `expected wrapper dry run to succeed for ${args.join(" ")}: ${result.stderr}`);
+ return JSON.parse(result.stdout);
+}
+
+test("the PR vitest lane wrapper runs the native-runner group exactly on the final PR shard", () => {
+ for (const [args, envOverrides, expected] of [
+ [["--shard=1/2"], { GITHUB_WORKFLOW: "PR" }, false],
+ [["--shard=2/2"], { GITHUB_WORKFLOW: "PR" }, true],
+ [[], { GITHUB_WORKFLOW: "PR" }, true],
+ [["--shard=2/2"], { GITHUB_WORKFLOW: "Release" }, false],
+ [["--shard=2/2"], {}, false],
+ [[], {}, false],
+ ]) {
+ const plan = wrapperPlan(args, envOverrides);
+ assert.equal(plan.runNativeRunnerGroup, expected,
+ `args ${JSON.stringify(args)} env ${JSON.stringify(envOverrides)}`);
+ const commands = plan.plannedCommands.map((planned) => planned.args.join(" "));
+ assert.ok(commands.some((command) => command.startsWith(`exec vitest run${args.length ? ` ${args.join(" ")}` : ""}`)),
+ "the wrapper must pass shard flags through to vitest");
+ assert.equal(
+ commands.some((command) => command.includes("--group general-server-native-runner")),
+ expected,
+ );
+ }
+
+ const malformed = spawnSync(process.execPath, [laneWrapper, "--shard=nonsense", "--dry-run"], {
+ cwd: repoRoot,
+ encoding: "utf8",
+ env: workflowEnv({ GITHUB_WORKFLOW: "PR" }),
+ });
+ assert.notEqual(malformed.status, 0, "a malformed shard flag must fail rather than guess a lane");
+});
+
const lineShardFile = path.join(repoRoot, "server/src/__tests__/chat-channels.integration.test.ts");
const caseAt = (line, name) => ({ name, file: lineShardFile, projectName: "@paperclipai/server", location: { line, column: 3 } });
diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs
index 1e7a91df35..d98a814a12 100644
--- a/scripts/run-vitest-stable.mjs
+++ b/scripts/run-vitest-stable.mjs
@@ -70,13 +70,39 @@ const allModeName = "all";
const generalServerGroupName = "general-server";
const generalServerWithoutChatGroupName = "general-server-without-chat";
const generalChatGroupName = "general-chat";
+const generalServerNativeRunnerGroupName = "general-server-native-runner";
const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts";
+// This suite rebuilds the Runner release binaries with cargo in beforeAll.
+// Inside the PR workflow's plain server shards, which carry no Rust cache,
+// that build was a ~4m30s cold compile of every third-party crate on each run
+// (277s of a 291s shard vitest step, actions run 35246999382, 2026-09-17).
+const nativeRunnerSuite =
+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";
+// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable
+// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip
+// Runner vitest shard runs the native-runner group instead, because those
+// lanes restore the shared release-runner-v1 Rust cache (see
+// packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs). Every other
+// caller — local runs, release-verify.yml under the Release and Cloud
+// readiness workflows — keeps the suite in the server shards, so a renamed or
+// unknown workflow degrades to today's slower-but-covered behavior rather
+// than dropping the suite.
+const prWorkflowName = "PR";
+const nativeRunnerSuiteRunsInRustCachedLane = process.env.GITHUB_WORKFLOW === prWorkflowName;
+const withoutChatExcludedSuites = nativeRunnerSuiteRunsInRustCachedLane
+ ? [chatSuite, nativeRunnerSuite]
+ : [chatSuite];
const generalWorkspacesAGroupName = "general-workspaces-a";
const generalWorkspacesBGroupName = "general-workspaces-b";
const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"];
const generalWorkspacesBProjects = nonServerProjects.filter((project) => !generalWorkspacesAProjects.includes(project));
const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, generalWorkspacesBGroupName];
-const allowedGeneralGroupNames = [...generalGroupNames, generalServerWithoutChatGroupName, generalChatGroupName];
+const allowedGeneralGroupNames = [
+ ...generalGroupNames,
+ generalServerWithoutChatGroupName,
+ generalChatGroupName,
+ generalServerNativeRunnerGroupName,
+];
const serializedServerVitestArgs = [
"--no-file-parallelism",
"--maxWorkers=1",
@@ -355,9 +381,21 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount =
"chat integration test shard", { index: shardIndex ?? 0, count: shardCount ?? 1 });
return;
}
+ if (groupName === generalServerNativeRunnerGroupName) {
+ runVitest(
+ ["--project", "@paperclipai/server", ...serializedServerVitestArgs, nativeRunnerSuite],
+ "native runner vertical-slice suite",
+ );
+ return;
+ }
if (groupName === generalServerGroupName || groupName === generalServerWithoutChatGroupName) {
+ // In the PR workflow the without-chat group also leaves the native-runner
+ // suite to the Rust-cached vitest lane; the full general-server group
+ // (local runs) keeps both.
const withoutChat = groupName === generalServerWithoutChatGroupName;
- const files = withoutChat ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles;
+ const files = withoutChat
+ ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file))
+ : generalServerTestFiles;
if (shardCount !== null && shardCount > 1) {
const shardFiles = selectGeneralServerShard(
files,
@@ -385,7 +423,11 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount =
}
const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]);
- if (withoutChat) excludeRouteArgs.push("--exclude", "src/__tests__/chat-channels.integration.test.ts");
+ if (withoutChat) {
+ for (const suite of withoutChatExcludedSuites) {
+ excludeRouteArgs.push("--exclude", suite.replace(/^server\//, ""));
+ }
+ }
runVitest(
[
"--project",
@@ -474,16 +516,20 @@ if (options.dryRun) {
selectedSerializedSuites: serializedSuites.map((routeTest) => routeTest.repoPath),
generalServerSuiteCount: generalServerTestFiles.length,
selectedGeneralServerSuites:
- options.mode === generalModeName &&
- [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) &&
- options.shardCount !== null
- ? selectGeneralServerShard(
- options.group === generalServerWithoutChatGroupName ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles,
- options.shardIndex,
- options.shardCount,
- generalServerShardDurations,
- )
- : null,
+ options.mode === generalModeName && options.group === generalServerNativeRunnerGroupName
+ ? [nativeRunnerSuite]
+ : options.mode === generalModeName &&
+ [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) &&
+ options.shardCount !== null
+ ? selectGeneralServerShard(
+ options.group === generalServerWithoutChatGroupName
+ ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file))
+ : generalServerTestFiles,
+ options.shardIndex,
+ options.shardCount,
+ generalServerShardDurations,
+ )
+ : null,
workspaceProjects:
options.group === generalWorkspacesAGroupName
? generalWorkspacesAProjects