Skip to content

The open-source app everyone uses to manage agents at work

perf(ci): build standalone public packages concurrently (#8567)

Devin Foleyef37203a4805
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml
index 42c7c4ec13..af04adf61d 100644
--- a/.github/workflows/pr.yml
+++ b/.github/workflows/pr.yml
@@ -58,6 +58,9 @@ jobs:
- name: Test general-server shard partition
run: node --test ./scripts/__tests__/run-vitest-stable-shard.test.mjs
+ - name: Test standalone package build concurrency
+ run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
+
- name: Validate release package manifest
run: node ./scripts/release-package-map.mjs check
diff --git a/scripts/__tests__/build-standalone-concurrency.test.mjs b/scripts/__tests__/build-standalone-concurrency.test.mjs
new file mode 100644
index 0000000000..ef22755f57
--- /dev/null
+++ b/scripts/__tests__/build-standalone-concurrency.test.mjs
@@ -0,0 +1,131 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ resolveConcurrency,
+ runWithConcurrency,
+} from "../build-standalone-public-packages.mjs";
+
+function deferred() {
+ let resolve;
+ let reject;
+ const promise = new Promise((res, rej) => {
+ resolve = res;
+ reject = rej;
+ });
+ return { promise, resolve, reject };
+}
+
+test("runs every item and preserves result order", async () => {
+ const items = [1, 2, 3, 4, 5];
+ const results = await runWithConcurrency(items, 2, async (value) => value * 10);
+ assert.deepEqual(results, [10, 20, 30, 40, 50]);
+});
+
+test("never exceeds the concurrency limit", async () => {
+ const items = Array.from({ length: 8 }, (_, index) => index);
+ const limit = 3;
+ let active = 0;
+ let peak = 0;
+ const gates = items.map(() => deferred());
+
+ const run = runWithConcurrency(items, limit, async (value) => {
+ active += 1;
+ peak = Math.max(peak, active);
+ assert.ok(active <= limit, `active ${active} exceeded limit ${limit}`);
+ await gates[value].promise;
+ active -= 1;
+ return value;
+ });
+
+ // Release gates progressively; the pool must keep at most `limit` in flight.
+ for (const gate of gates) {
+ await Promise.resolve();
+ gate.resolve();
+ }
+
+ await run;
+ assert.equal(peak, limit);
+});
+
+test("aggregates failures and still runs remaining items", async () => {
+ const items = [0, 1, 2, 3];
+ const processed = [];
+
+ await assert.rejects(
+ () =>
+ runWithConcurrency(items, 2, async (value) => {
+ processed.push(value);
+ if (value === 1) {
+ throw new Error(`boom-${value}`);
+ }
+ return value;
+ }),
+ (error) => {
+ assert.match(error.message, /1 standalone package build\(s\) failed/);
+ assert.equal(error.failures.length, 1);
+ assert.equal(error.failures[0].index, 1);
+ assert.match(error.failures[0].error.message, /boom-1/);
+ return true;
+ },
+ );
+
+ // A single failure must not abort the rest of the queue.
+ assert.deepEqual(processed.sort((a, b) => a - b), [0, 1, 2, 3]);
+});
+
+test("reports failures sorted by original index", async () => {
+ const items = [0, 1, 2, 3, 4];
+
+ await assert.rejects(
+ () =>
+ runWithConcurrency(items, 5, async (value) => {
+ if (value === 3 || value === 1) {
+ throw new Error(`fail-${value}`);
+ }
+ return value;
+ }),
+ (error) => {
+ assert.deepEqual(
+ error.failures.map(({ index }) => index),
+ [1, 3],
+ );
+ return true;
+ },
+ );
+});
+
+test("resolveConcurrency honors a valid env override, capped by package count", () => {
+ const previous = process.env.STANDALONE_BUILD_CONCURRENCY;
+ try {
+ process.env.STANDALONE_BUILD_CONCURRENCY = "2";
+ assert.equal(resolveConcurrency(7), 2);
+ assert.equal(resolveConcurrency(1), 1);
+ assert.equal(resolveConcurrency(0), 1);
+
+ process.env.STANDALONE_BUILD_CONCURRENCY = "0";
+ assert.ok(resolveConcurrency(7) >= 1);
+
+ process.env.STANDALONE_BUILD_CONCURRENCY = "not-a-number";
+ assert.ok(resolveConcurrency(7) >= 1);
+ } finally {
+ if (previous === undefined) {
+ delete process.env.STANDALONE_BUILD_CONCURRENCY;
+ } else {
+ process.env.STANDALONE_BUILD_CONCURRENCY = previous;
+ }
+ }
+});
+
+test("resolveConcurrency never returns more than the package count", () => {
+ const previous = process.env.STANDALONE_BUILD_CONCURRENCY;
+ try {
+ delete process.env.STANDALONE_BUILD_CONCURRENCY;
+ assert.equal(resolveConcurrency(0), 1);
+ assert.ok(resolveConcurrency(3) <= 3);
+ } finally {
+ if (previous !== undefined) {
+ process.env.STANDALONE_BUILD_CONCURRENCY = previous;
+ }
+ }
+});
diff --git a/scripts/build-standalone-public-packages.mjs b/scripts/build-standalone-public-packages.mjs
index 7d004bb7ab..7b5e6b4902 100644
--- a/scripts/build-standalone-public-packages.mjs
+++ b/scripts/build-standalone-public-packages.mjs
@@ -1,13 +1,17 @@
#!/usr/bin/env node
-import { execFileSync } from "node:child_process";
+import { execFile, execFileSync } from "node:child_process";
import { existsSync, readFileSync, rmSync } from "node:fs";
+import { availableParallelism } from "node:os";
import path, { dirname } from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
+import { promisify } from "node:util";
import { linkSdkInto } from "./link-plugin-dev-sdk.mjs";
+const execFileAsync = promisify(execFile);
+
const scriptDir = dirname(fileURLToPath(import.meta.url));
const repoRoot = path.resolve(scriptDir, "..");
const workspacePath = path.join(repoRoot, "pnpm-workspace.yaml");
@@ -92,34 +96,41 @@ function readPackageJson(pkgDir) {
);
}
-function run(command, args, cwd) {
- execFileSync(command, args, {
- cwd,
- env: {
- ...process.env,
- CI: "true",
- },
- stdio: "inherit",
- });
+async function runCaptured(command, args, cwd, log) {
+ try {
+ const { stdout, stderr } = await execFileAsync(command, args, {
+ cwd,
+ env: {
+ ...process.env,
+ CI: "true",
+ },
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ if (stdout?.trim()) log(stdout.trimEnd());
+ if (stderr?.trim()) log(stderr.trimEnd());
+ } catch (error) {
+ if (error.stdout?.toString().trim()) log(error.stdout.toString().trimEnd());
+ if (error.stderr?.toString().trim()) log(error.stderr.toString().trimEnd());
+ throw error;
+ }
}
-function main() {
- const workspaceEntries = parseWorkspaceEntries(readFileSync(workspacePath, "utf8"));
- const standalonePackages = listPublicPackages()
- .filter(({ dir }) => !isWorkspacePackage(dir, workspaceEntries));
-
- if (standalonePackages.length === 0) {
- console.log(" i No standalone public packages detected outside the pnpm workspace");
- return;
- }
+// Each standalone package installs into its own directory (`--ignore-workspace`)
+// and builds into its own `dist`, so there is no shared mutable state between
+// packages beyond pnpm's content-addressable global store, which is safe for
+// concurrent access. Buffer each package's output and flush it as one block so
+// interleaved parallel logs stay readable.
+async function prepareAndBuildPackage(pkg) {
+ const logs = [];
+ const log = (line) => logs.push(line);
- for (const pkg of standalonePackages) {
- const pkgDir = path.join(repoRoot, pkg.dir);
- const pkgJson = readPackageJson(pkg.dir);
- const nodeModulesDir = path.join(pkgDir, "node_modules");
- const packageLockfilePath = path.join(pkgDir, "pnpm-lock.yaml");
+ const pkgDir = path.join(repoRoot, pkg.dir);
+ const pkgJson = readPackageJson(pkg.dir);
+ const nodeModulesDir = path.join(pkgDir, "node_modules");
+ const packageLockfilePath = path.join(pkgDir, "pnpm-lock.yaml");
- console.log(` Preparing standalone package ${pkg.name} (${pkg.dir})`);
+ log(` Preparing standalone package ${pkg.name} (${pkg.dir})`);
+ try {
if (existsSync(nodeModulesDir)) {
rmSync(nodeModulesDir, { force: true, recursive: true });
}
@@ -133,7 +144,7 @@ function main() {
// Standalone packages intentionally avoid committed lockfile churn in the repo.
];
- run("pnpm", installArgs, pkgDir);
+ await runCaptured("pnpm", installArgs, pkgDir, log);
// The fresh install above wipes node_modules and no longer fires a
// per-plugin postinstall (removed for supply-chain safety), so link the
@@ -141,11 +152,113 @@ function main() {
linkSdkInto(pkgDir);
if (pkgJson.scripts?.build) {
- run("pnpm", ["run", "build"], pkgDir);
+ await runCaptured("pnpm", ["run", "build"], pkgDir, log);
} else {
- console.log(" i No build script; skipped build");
+ log(" i No build script; skipped build");
+ }
+ } finally {
+ if (logs.length > 0) {
+ console.log(logs.join("\n"));
+ }
+ }
+}
+
+export function resolveConcurrency(packageCount) {
+ const raw = process.env.STANDALONE_BUILD_CONCURRENCY;
+ if (raw !== undefined && raw !== "") {
+ const parsed = Number.parseInt(raw, 10);
+ if (Number.isFinite(parsed) && parsed > 0) {
+ return Math.max(1, Math.min(parsed, packageCount));
+ }
+ }
+
+ let cpus = 4;
+ try {
+ cpus = availableParallelism();
+ } catch {
+ // Fall back to the default when parallelism cannot be determined.
+ }
+ return Math.max(1, Math.min(cpus, packageCount));
+}
+
+// Bounded-concurrency task pool. Workers pull from a shared cursor so no more
+// than `limit` tasks run at once. Every task is awaited even if an earlier one
+// fails, and failures are aggregated (sorted by original index) so a single bad
+// package neither aborts the others mid-flight nor hides which one broke.
+export async function runWithConcurrency(items, limit, worker) {
+ const results = new Array(items.length);
+ const failures = [];
+ let nextIndex = 0;
+
+ async function runWorker() {
+ while (true) {
+ const current = nextIndex;
+ nextIndex += 1;
+ if (current >= items.length) {
+ return;
+ }
+ try {
+ results[current] = await worker(items[current], current);
+ } catch (error) {
+ failures.push({ index: current, error });
+ }
}
}
+
+ const poolSize = Math.max(1, Math.min(limit, items.length));
+ await Promise.all(Array.from({ length: poolSize }, () => runWorker()));
+
+ if (failures.length > 0) {
+ failures.sort((a, b) => a.index - b.index);
+ const aggregate = new Error(
+ `${failures.length} standalone package build(s) failed`,
+ );
+ aggregate.failures = failures;
+ throw aggregate;
+ }
+
+ return results;
}
-main();
+async function main() {
+ const workspaceEntries = parseWorkspaceEntries(readFileSync(workspacePath, "utf8"));
+ const standalonePackages = listPublicPackages()
+ .filter(({ dir }) => !isWorkspacePackage(dir, workspaceEntries));
+
+ if (standalonePackages.length === 0) {
+ console.log(" i No standalone public packages detected outside the pnpm workspace");
+ return;
+ }
+
+ const concurrency = resolveConcurrency(standalonePackages.length);
+ console.log(
+ ` Building ${standalonePackages.length} standalone package(s) with concurrency ${concurrency}`,
+ );
+
+ try {
+ await runWithConcurrency(
+ standalonePackages,
+ concurrency,
+ prepareAndBuildPackage,
+ );
+ } catch (error) {
+ if (Array.isArray(error.failures)) {
+ const names = error.failures
+ .map(({ index }) => standalonePackages[index]?.name)
+ .filter(Boolean)
+ .join(", ");
+ console.error(` ✗ Failed standalone package build(s): ${names}`);
+ }
+ throw error;
+ }
+}
+
+const invokedDirectly =
+ process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
+
+if (invokedDirectly) {
+ main().catch((error) => {
+ console.error(error);
+ process.exit(1);
+ });
+}