The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix: run the cargo-building native-runner CI suite in the Rust-cached vitest lane (#13586)
Devin Foley352153b5edf0
diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.jsonindex e7fbc955dc..fbf51aa98b 100644--- a/packages/paperclip-runner/package.json+++ b/packages/paperclip-runner/package.json@@ -75,7 +75,7 @@"test": "pnpm run test:typescript && pnpm run test:rust","test:typescript": "pnpm run test:typescript:prep && vitest run","test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs",- "test:typescript:vitest": "pnpm run ensure:eval-build-deps && pnpm run build:rust && vitest run",+ "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs","test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace","test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider","test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::",diff --git a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjsnew file mode 100644index 0000000000..e383bd9024--- /dev/null+++ b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs@@ -0,0 +1,69 @@+// Runs this package's vitest shard for a Verify Paperclip Runner lane, then —+// on the final shard of the PR workflow only — the server package's+// general-server-native-runner group.+//+// That server suite rebuilds the Runner release binaries with cargo in+// beforeAll. The PR workflow's plain General tests server shards carry no+// Rust cache, so hosting it there cold-compiled every third-party crate on+// each run (277s of a 291s shard vitest step, actions run 35246999382,+// 2026-09-17) and made that shard the slowest check of the whole run. The+// Verify Paperclip Runner lanes already restore the shared release-runner-v1+// Rust cache read-only, which turns that build into an incremental rebuild,+// and the workflow files themselves list this lane's command as a package+// script — so the suite moves here without a workflow-file change.+//+// Contract, mirrored in scripts/run-vitest-stable.mjs (prWorkflowName) and+// pinned by scripts/__tests__/run-vitest-stable-shard.test.mjs:+// - Only the PR workflow (pr.yml, whose GITHUB_WORKFLOW the reusable+// pr-trusted.yml jobs inherit) excludes the suite from the server shards,+// and only there does this wrapper run it. Any other caller — local runs,+// release-verify.yml — keeps the suite in the server group, so a renamed+// workflow degrades to the slower covered path instead of losing coverage.+// - The suite runs on the lane whose --shard=N/M has N === M (or an unsharded+// invocation), so exactly one PR lane carries it.+import { spawnSync } from "node:child_process";+import { dirname, resolve } from "node:path";+import { fileURLToPath } from "node:url";++const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");+const workspaceRoot = resolve(packageRoot, "../..");+const args = process.argv.slice(2).filter((value) => value !== "--dry-run");+const dryRun = process.argv.includes("--dry-run");++const shardArg = args.find((value) => value.startsWith("--shard="));+const shardMatch = shardArg ? /^--shard=(\d+)\/(\d+)$/.exec(shardArg) : null;+if (shardArg && !shardMatch) {+ console.error(`[pr-vitest-lane] unrecognized shard argument: ${shardArg}`);+ process.exit(1);+}+const isFinalShard = !shardArg || shardMatch[1] === shardMatch[2];+const isPrWorkflow = process.env.GITHUB_WORKFLOW === "PR";+const runNativeRunnerGroup = isPrWorkflow && isFinalShard;++const plannedCommands = [+ { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: packageRoot },+ { command: "pnpm", args: ["run", "build:rust"], cwd: packageRoot },+ { command: "pnpm", args: ["exec", "vitest", "run", ...args], cwd: packageRoot },+ ...(runNativeRunnerGroup+ ? [{+ command: "pnpm",+ args: ["test:run:general", "--", "--group", "general-server-native-runner"],+ cwd: workspaceRoot,+ }]+ : []),+];++if (dryRun) {+ console.log(JSON.stringify({ isPrWorkflow, isFinalShard, runNativeRunnerGroup, plannedCommands }, null, 2));+ process.exit(0);+}++for (const planned of plannedCommands) {+ const result = spawnSync(planned.command, planned.args, {+ cwd: planned.cwd,+ stdio: "inherit",+ });+ if (result.status !== 0) {+ process.exit(result.status ?? 1);+ }+}diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjsindex f320fa4859..ffe7fb2d0b 100644--- a/scripts/__tests__/run-vitest-stable-shard.test.mjs+++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs@@ -1,5 +1,6 @@import assert from "node:assert/strict";import { spawnSync } from "node:child_process";+import { readFileSync } from "node:fs";import path from "node:path";import { fileURLToPath } from "node:url";import test from "node:test";@@ -21,16 +22,29 @@ const serializedDurationsManifest = path.join("serialized-shard-durations.json",);-function dryRun(args) {+// Membership of general-server-without-chat depends on GITHUB_WORKFLOW (see+// prWorkflowName in run-vitest-stable.mjs), so strip the ambient value and+// make every test pin the caller it mirrors explicitly — these tests+// themselves run inside a workflow on CI.+function workflowEnv(envOverrides = {}) {+ const env = { ...process.env, ...envOverrides };+ if (!("GITHUB_WORKFLOW" in envOverrides)) {+ delete env.GITHUB_WORKFLOW;+ }+ return env;+}++function dryRun(args, envOverrides = {}) {const result = spawnSync(process.execPath, [script, ...args, "--dry-run"], {cwd: repoRoot,encoding: "utf8",+ env: workflowEnv(envOverrides),});return result;}-function dryRunJson(args) {- const result = dryRun(args);+function dryRunJson(args, envOverrides = {}) {+ const result = dryRun(args, envOverrides);assert.equal(result.status, 0, `expected success for ${args.join(" ")}: ${result.stderr}`);return JSON.parse(result.stdout);}@@ -222,11 +236,15 @@ test("the real serialized shard partition is duration-balanced", () => {test("the real shard partition is duration-balanced", () => {// Mirrors the PR matrix: general-server-without-chat across SHARD_COUNT- // runners, with the chat suite carried by the dedicated general-chat lanes.+ // runners, with the chat suite carried by the dedicated general-chat lanes+ // and the native-runner suite by the Rust-cached PR vitest lane.const durations = loadShardDurations(durationsManifest);const fallback = defaultSuiteWeight(durations);const shards = Array.from({ length: SHARD_COUNT }, (_, index) =>- dryRunJson(["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)]),+ dryRunJson(+ ["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)],+ { GITHUB_WORKFLOW: "PR" },+ ),);const totals = shards.map((shard) =>@@ -235,10 +253,15 @@ test("the real shard partition is duration-balanced", () => {const maxTotal = Math.max(...totals);const minTotal = Math.min(...totals);// LPT keeps the spread within the heaviest single suite; use that as the- // bound. The chat suite runs in its own lanes, so exclude it here.+ // bound. The chat and native-runner suites run in their own lanes, so+ // exclude them here.const chat = "server/src/__tests__/chat-channels.integration.test.ts";+ const nativeRunner =+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";const heaviest = Math.max(- ...Object.entries(durations).filter(([file]) => file !== chat).map(([, ms]) => ms),+ ...Object.entries(durations)+ .filter(([file]) => file !== chat && file !== nativeRunner)+ .map(([, ms]) => ms),);assert.ok(maxTotal - minTotal <= heaviest,@@ -247,24 +270,131 @@ test("the real shard partition is duration-balanced", () => {});-// 12 mirrors pr-trusted.yml, 10 mirrors release-verify.yml.-for (const withoutChatShardCount of [10, 12]) {- test(`${withoutChatShardCount} without-chat shards plus the dedicated chat file cover the original server group exactly`, () => {- const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]);- const shards = Array.from({ length: withoutChatShardCount }, (_, index) => dryRunJson([+const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts";+const nativeRunnerSuitePath =+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";++// Mirrors pr-trusted.yml (12 shards, called by pr.yml so GITHUB_WORKFLOW is+// "PR"): the chat suite runs in its dedicated lanes and the cargo-dependent+// native-runner suite in the Rust-cached final Verify Paperclip Runner vitest+// shard, so together the three cover the full server group exactly.+test("12 PR without-chat shards plus the dedicated chat and native-runner lanes cover the original server group exactly", () => {+ const prEnv = { GITHUB_WORKFLOW: "PR" };+ const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], prEnv);+ const shards = Array.from({ length: 12 }, (_, index) => dryRunJson([+ "--mode", "general", "--group", "general-server-without-chat",+ "--shard-index", String(index), "--shard-count", "12",+ ], prEnv));+ const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);+ assert.ok(!files.includes(chatSuitePath));+ assert.ok(!files.includes(nativeRunnerSuitePath));+ assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath].sort(), full.selectedGeneralServerSuites.sort());+ assert.equal(new Set(files).size, files.length);+ const defaultRun = dryRunJson([], prEnv);+ assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);+});++// Mirrors release-verify.yml (10 shards, called by the Release and Cloud+// readiness workflows) and local runs: no Rust-cached vitest lane exists+// there, so the native-runner suite must stay in the server shards.+for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" }], ["no ambient workflow", {}]]) {+ test(`10 without-chat shards under ${caller} keep the native-runner suite and cover the server group with chat alone`, () => {+ const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], envOverrides);+ const shards = Array.from({ length: 10 }, (_, index) => dryRunJson(["--mode", "general", "--group", "general-server-without-chat",- "--shard-index", String(index), "--shard-count", String(withoutChatShardCount),- ]));+ "--shard-index", String(index), "--shard-count", "10",+ ], envOverrides));const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites);- const chat = "server/src/__tests__/chat-channels.integration.test.ts";- assert.ok(!files.includes(chat));- assert.deepEqual([...files, chat].sort(), full.selectedGeneralServerSuites.sort());+ assert.ok(!files.includes(chatSuitePath));+ assert.ok(files.includes(nativeRunnerSuitePath));+ assert.deepEqual([...files, chatSuitePath].sort(), full.selectedGeneralServerSuites.sort());assert.equal(new Set(files).size, files.length);- const defaultRun = dryRunJson([]);- assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount);});}+test("the native-runner lane runs exactly the cargo-dependent vertical-slice suite", () => {+ const lane = dryRunJson(["--mode", "general", "--group", "general-server-native-runner"]);+ assert.deepEqual(lane.selectedGeneralServerSuites, [+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts",+ ]);+});++test("shard flags are rejected for the native-runner group", () => {+ const result = dryRun(["--mode", "general", "--group", "general-server-native-runner", "--shard-index", "0", "--shard-count", "2"]);+ assert.notEqual(result.status, 0, "the native-runner lane is a single suite and must not accept shard flags");+});++// The PR-side exclusion above is safe only while the wiring it assumes holds:+// pr.yml (the caller whose name reusable pr-trusted.yml jobs see as+// GITHUB_WORKFLOW) is named PR, the sharded vitest lanes partition cleanly+// with exactly one final shard, and that lane's package script routes through+// the wrapper that runs the native-runner group.+test("the PR workflow wiring for the native-runner lane holds", () => {+ const prWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr.yml"), "utf8");+ assert.match(prWorkflow, /^name: PR$/m,+ "renaming pr.yml silently moves the native-runner suite back into the uncached server shards");++ const trustedWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr-trusted.yml"), "utf8");+ const lanes = [...trustedWorkflow.matchAll(/command: test:typescript:vitest --shard=(\d+)\/(\d+)/g)]+ .map((match) => [Number(match[1]), Number(match[2])]);+ assert.ok(lanes.length > 0, "expected sharded test:typescript:vitest lanes in pr-trusted.yml");+ assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "vitest lanes must agree on the shard count");+ const shardCount = lanes[0][1];+ assert.deepEqual(+ lanes.map(([index]) => index).sort((left, right) => left - right),+ Array.from({ length: shardCount }, (_, index) => index + 1),+ "vitest lanes must cover every shard exactly once",+ );+ assert.equal(lanes.filter(([index, count]) => index === count).length, 1,+ "exactly one final vitest shard carries the native-runner group");++ const runnerPackage = JSON.parse(+ readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8"),+ );+ assert.equal(runnerPackage.scripts["test:typescript:vitest"], "node ./scripts/run-pr-vitest-lane.mjs");+});++const laneWrapper = path.join(repoRoot, "packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs");++function wrapperPlan(args, envOverrides = {}) {+ const result = spawnSync(process.execPath, [laneWrapper, ...args, "--dry-run"], {+ cwd: repoRoot,+ encoding: "utf8",+ env: workflowEnv(envOverrides),+ });+ assert.equal(result.status, 0, `expected wrapper dry run to succeed for ${args.join(" ")}: ${result.stderr}`);+ return JSON.parse(result.stdout);+}++test("the PR vitest lane wrapper runs the native-runner group exactly on the final PR shard", () => {+ for (const [args, envOverrides, expected] of [+ [["--shard=1/2"], { GITHUB_WORKFLOW: "PR" }, false],+ [["--shard=2/2"], { GITHUB_WORKFLOW: "PR" }, true],+ [[], { GITHUB_WORKFLOW: "PR" }, true],+ [["--shard=2/2"], { GITHUB_WORKFLOW: "Release" }, false],+ [["--shard=2/2"], {}, false],+ [[], {}, false],+ ]) {+ const plan = wrapperPlan(args, envOverrides);+ assert.equal(plan.runNativeRunnerGroup, expected,+ `args ${JSON.stringify(args)} env ${JSON.stringify(envOverrides)}`);+ const commands = plan.plannedCommands.map((planned) => planned.args.join(" "));+ assert.ok(commands.some((command) => command.startsWith(`exec vitest run${args.length ? ` ${args.join(" ")}` : ""}`)),+ "the wrapper must pass shard flags through to vitest");+ assert.equal(+ commands.some((command) => command.includes("--group general-server-native-runner")),+ expected,+ );+ }++ const malformed = spawnSync(process.execPath, [laneWrapper, "--shard=nonsense", "--dry-run"], {+ cwd: repoRoot,+ encoding: "utf8",+ env: workflowEnv({ GITHUB_WORKFLOW: "PR" }),+ });+ assert.notEqual(malformed.status, 0, "a malformed shard flag must fail rather than guess a lane");+});+const lineShardFile = path.join(repoRoot, "server/src/__tests__/chat-channels.integration.test.ts");const caseAt = (line, name) => ({ name, file: lineShardFile, projectName: "@paperclipai/server", location: { line, column: 3 } });diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjsindex 1e7a91df35..d98a814a12 100644--- a/scripts/run-vitest-stable.mjs+++ b/scripts/run-vitest-stable.mjs@@ -70,13 +70,39 @@ const allModeName = "all";const generalServerGroupName = "general-server";const generalServerWithoutChatGroupName = "general-server-without-chat";const generalChatGroupName = "general-chat";+const generalServerNativeRunnerGroupName = "general-server-native-runner";const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts";+// This suite rebuilds the Runner release binaries with cargo in beforeAll.+// Inside the PR workflow's plain server shards, which carry no Rust cache,+// that build was a ~4m30s cold compile of every third-party crate on each run+// (277s of a 291s shard vitest step, actions run 35246999382, 2026-09-17).+const nativeRunnerSuite =+ "server/src/services/native-runtime/native-codex-runner.integration.test.ts";+// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable+// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip+// Runner vitest shard runs the native-runner group instead, because those+// lanes restore the shared release-runner-v1 Rust cache (see+// packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs). Every other+// caller — local runs, release-verify.yml under the Release and Cloud+// readiness workflows — keeps the suite in the server shards, so a renamed or+// unknown workflow degrades to today's slower-but-covered behavior rather+// than dropping the suite.+const prWorkflowName = "PR";+const nativeRunnerSuiteRunsInRustCachedLane = process.env.GITHUB_WORKFLOW === prWorkflowName;+const withoutChatExcludedSuites = nativeRunnerSuiteRunsInRustCachedLane+ ? [chatSuite, nativeRunnerSuite]+ : [chatSuite];const generalWorkspacesAGroupName = "general-workspaces-a";const generalWorkspacesBGroupName = "general-workspaces-b";const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"];const generalWorkspacesBProjects = nonServerProjects.filter((project) => !generalWorkspacesAProjects.includes(project));const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, generalWorkspacesBGroupName];-const allowedGeneralGroupNames = [...generalGroupNames, generalServerWithoutChatGroupName, generalChatGroupName];+const allowedGeneralGroupNames = [+ ...generalGroupNames,+ generalServerWithoutChatGroupName,+ generalChatGroupName,+ generalServerNativeRunnerGroupName,+];const serializedServerVitestArgs = ["--no-file-parallelism","--maxWorkers=1",@@ -355,9 +381,21 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount ="chat integration test shard", { index: shardIndex ?? 0, count: shardCount ?? 1 });return;}+ if (groupName === generalServerNativeRunnerGroupName) {+ runVitest(+ ["--project", "@paperclipai/server", ...serializedServerVitestArgs, nativeRunnerSuite],+ "native runner vertical-slice suite",+ );+ return;+ }if (groupName === generalServerGroupName || groupName === generalServerWithoutChatGroupName) {+ // In the PR workflow the without-chat group also leaves the native-runner+ // suite to the Rust-cached vitest lane; the full general-server group+ // (local runs) keeps both.const withoutChat = groupName === generalServerWithoutChatGroupName;- const files = withoutChat ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles;+ const files = withoutChat+ ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file))+ : generalServerTestFiles;if (shardCount !== null && shardCount > 1) {const shardFiles = selectGeneralServerShard(files,@@ -385,7 +423,11 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount =}const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]);- if (withoutChat) excludeRouteArgs.push("--exclude", "src/__tests__/chat-channels.integration.test.ts");+ if (withoutChat) {+ for (const suite of withoutChatExcludedSuites) {+ excludeRouteArgs.push("--exclude", suite.replace(/^server\//, ""));+ }+ }runVitest(["--project",@@ -474,16 +516,20 @@ if (options.dryRun) {selectedSerializedSuites: serializedSuites.map((routeTest) => routeTest.repoPath),generalServerSuiteCount: generalServerTestFiles.length,selectedGeneralServerSuites:- options.mode === generalModeName &&- [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) &&- options.shardCount !== null- ? selectGeneralServerShard(- options.group === generalServerWithoutChatGroupName ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles,- options.shardIndex,- options.shardCount,- generalServerShardDurations,- )- : null,+ options.mode === generalModeName && options.group === generalServerNativeRunnerGroupName+ ? [nativeRunnerSuite]+ : options.mode === generalModeName &&+ [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) &&+ options.shardCount !== null+ ? selectGeneralServerShard(+ options.group === generalServerWithoutChatGroupName+ ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file))+ : generalServerTestFiles,+ options.shardIndex,+ options.shardCount,+ generalServerShardDurations,+ )+ : null,workspaceProjects:options.group === generalWorkspacesAGroupName? generalWorkspacesAProjects