The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(ci): build standalone public packages concurrently (#8567)
Devin Foleyef37203a4805
diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.ymlindex 42c7c4ec13..af04adf61d 100644--- a/.github/workflows/pr.yml+++ b/.github/workflows/pr.yml@@ -58,6 +58,9 @@ jobs:- name: Test general-server shard partitionrun: node --test ./scripts/__tests__/run-vitest-stable-shard.test.mjs+ - name: Test standalone package build concurrency+ run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs+- name: Validate release package manifestrun: node ./scripts/release-package-map.mjs checkdiff --git a/scripts/__tests__/build-standalone-concurrency.test.mjs b/scripts/__tests__/build-standalone-concurrency.test.mjsnew file mode 100644index 0000000000..ef22755f57--- /dev/null+++ b/scripts/__tests__/build-standalone-concurrency.test.mjs@@ -0,0 +1,131 @@+import assert from "node:assert/strict";+import test from "node:test";++import {+ resolveConcurrency,+ runWithConcurrency,+} from "../build-standalone-public-packages.mjs";++function deferred() {+ let resolve;+ let reject;+ const promise = new Promise((res, rej) => {+ resolve = res;+ reject = rej;+ });+ return { promise, resolve, reject };+}++test("runs every item and preserves result order", async () => {+ const items = [1, 2, 3, 4, 5];+ const results = await runWithConcurrency(items, 2, async (value) => value * 10);+ assert.deepEqual(results, [10, 20, 30, 40, 50]);+});++test("never exceeds the concurrency limit", async () => {+ const items = Array.from({ length: 8 }, (_, index) => index);+ const limit = 3;+ let active = 0;+ let peak = 0;+ const gates = items.map(() => deferred());++ const run = runWithConcurrency(items, limit, async (value) => {+ active += 1;+ peak = Math.max(peak, active);+ assert.ok(active <= limit, `active ${active} exceeded limit ${limit}`);+ await gates[value].promise;+ active -= 1;+ return value;+ });++ // Release gates progressively; the pool must keep at most `limit` in flight.+ for (const gate of gates) {+ await Promise.resolve();+ gate.resolve();+ }++ await run;+ assert.equal(peak, limit);+});++test("aggregates failures and still runs remaining items", async () => {+ const items = [0, 1, 2, 3];+ const processed = [];++ await assert.rejects(+ () =>+ runWithConcurrency(items, 2, async (value) => {+ processed.push(value);+ if (value === 1) {+ throw new Error(`boom-${value}`);+ }+ return value;+ }),+ (error) => {+ assert.match(error.message, /1 standalone package build\(s\) failed/);+ assert.equal(error.failures.length, 1);+ assert.equal(error.failures[0].index, 1);+ assert.match(error.failures[0].error.message, /boom-1/);+ return true;+ },+ );++ // A single failure must not abort the rest of the queue.+ assert.deepEqual(processed.sort((a, b) => a - b), [0, 1, 2, 3]);+});++test("reports failures sorted by original index", async () => {+ const items = [0, 1, 2, 3, 4];++ await assert.rejects(+ () =>+ runWithConcurrency(items, 5, async (value) => {+ if (value === 3 || value === 1) {+ throw new Error(`fail-${value}`);+ }+ return value;+ }),+ (error) => {+ assert.deepEqual(+ error.failures.map(({ index }) => index),+ [1, 3],+ );+ return true;+ },+ );+});++test("resolveConcurrency honors a valid env override, capped by package count", () => {+ const previous = process.env.STANDALONE_BUILD_CONCURRENCY;+ try {+ process.env.STANDALONE_BUILD_CONCURRENCY = "2";+ assert.equal(resolveConcurrency(7), 2);+ assert.equal(resolveConcurrency(1), 1);+ assert.equal(resolveConcurrency(0), 1);++ process.env.STANDALONE_BUILD_CONCURRENCY = "0";+ assert.ok(resolveConcurrency(7) >= 1);++ process.env.STANDALONE_BUILD_CONCURRENCY = "not-a-number";+ assert.ok(resolveConcurrency(7) >= 1);+ } finally {+ if (previous === undefined) {+ delete process.env.STANDALONE_BUILD_CONCURRENCY;+ } else {+ process.env.STANDALONE_BUILD_CONCURRENCY = previous;+ }+ }+});++test("resolveConcurrency never returns more than the package count", () => {+ const previous = process.env.STANDALONE_BUILD_CONCURRENCY;+ try {+ delete process.env.STANDALONE_BUILD_CONCURRENCY;+ assert.equal(resolveConcurrency(0), 1);+ assert.ok(resolveConcurrency(3) <= 3);+ } finally {+ if (previous !== undefined) {+ process.env.STANDALONE_BUILD_CONCURRENCY = previous;+ }+ }+});diff --git a/scripts/build-standalone-public-packages.mjs b/scripts/build-standalone-public-packages.mjsindex 7d004bb7ab..7b5e6b4902 100644--- a/scripts/build-standalone-public-packages.mjs+++ b/scripts/build-standalone-public-packages.mjs@@ -1,13 +1,17 @@#!/usr/bin/env node-import { execFileSync } from "node:child_process";+import { execFile, execFileSync } from "node:child_process";import { existsSync, readFileSync, rmSync } from "node:fs";+import { availableParallelism } from "node:os";import path, { dirname } from "node:path";import process from "node:process";import { fileURLToPath } from "node:url";+import { promisify } from "node:util";import { linkSdkInto } from "./link-plugin-dev-sdk.mjs";+const execFileAsync = promisify(execFile);+const scriptDir = dirname(fileURLToPath(import.meta.url));const repoRoot = path.resolve(scriptDir, "..");const workspacePath = path.join(repoRoot, "pnpm-workspace.yaml");@@ -92,34 +96,41 @@ function readPackageJson(pkgDir) {);}-function run(command, args, cwd) {- execFileSync(command, args, {- cwd,- env: {- ...process.env,- CI: "true",- },- stdio: "inherit",- });+async function runCaptured(command, args, cwd, log) {+ try {+ const { stdout, stderr } = await execFileAsync(command, args, {+ cwd,+ env: {+ ...process.env,+ CI: "true",+ },+ maxBuffer: 64 * 1024 * 1024,+ });+ if (stdout?.trim()) log(stdout.trimEnd());+ if (stderr?.trim()) log(stderr.trimEnd());+ } catch (error) {+ if (error.stdout?.toString().trim()) log(error.stdout.toString().trimEnd());+ if (error.stderr?.toString().trim()) log(error.stderr.toString().trimEnd());+ throw error;+ }}-function main() {- const workspaceEntries = parseWorkspaceEntries(readFileSync(workspacePath, "utf8"));- const standalonePackages = listPublicPackages()- .filter(({ dir }) => !isWorkspacePackage(dir, workspaceEntries));-- if (standalonePackages.length === 0) {- console.log(" i No standalone public packages detected outside the pnpm workspace");- return;- }+// Each standalone package installs into its own directory (`--ignore-workspace`)+// and builds into its own `dist`, so there is no shared mutable state between+// packages beyond pnpm's content-addressable global store, which is safe for+// concurrent access. Buffer each package's output and flush it as one block so+// interleaved parallel logs stay readable.+async function prepareAndBuildPackage(pkg) {+ const logs = [];+ const log = (line) => logs.push(line);- for (const pkg of standalonePackages) {- const pkgDir = path.join(repoRoot, pkg.dir);- const pkgJson = readPackageJson(pkg.dir);- const nodeModulesDir = path.join(pkgDir, "node_modules");- const packageLockfilePath = path.join(pkgDir, "pnpm-lock.yaml");+ const pkgDir = path.join(repoRoot, pkg.dir);+ const pkgJson = readPackageJson(pkg.dir);+ const nodeModulesDir = path.join(pkgDir, "node_modules");+ const packageLockfilePath = path.join(pkgDir, "pnpm-lock.yaml");- console.log(` Preparing standalone package ${pkg.name} (${pkg.dir})`);+ log(` Preparing standalone package ${pkg.name} (${pkg.dir})`);+ try {if (existsSync(nodeModulesDir)) {rmSync(nodeModulesDir, { force: true, recursive: true });}@@ -133,7 +144,7 @@ function main() {// Standalone packages intentionally avoid committed lockfile churn in the repo.];- run("pnpm", installArgs, pkgDir);+ await runCaptured("pnpm", installArgs, pkgDir, log);// The fresh install above wipes node_modules and no longer fires a// per-plugin postinstall (removed for supply-chain safety), so link the@@ -141,11 +152,113 @@ function main() {linkSdkInto(pkgDir);if (pkgJson.scripts?.build) {- run("pnpm", ["run", "build"], pkgDir);+ await runCaptured("pnpm", ["run", "build"], pkgDir, log);} else {- console.log(" i No build script; skipped build");+ log(" i No build script; skipped build");+ }+ } finally {+ if (logs.length > 0) {+ console.log(logs.join("\n"));+ }+ }+}++export function resolveConcurrency(packageCount) {+ const raw = process.env.STANDALONE_BUILD_CONCURRENCY;+ if (raw !== undefined && raw !== "") {+ const parsed = Number.parseInt(raw, 10);+ if (Number.isFinite(parsed) && parsed > 0) {+ return Math.max(1, Math.min(parsed, packageCount));+ }+ }++ let cpus = 4;+ try {+ cpus = availableParallelism();+ } catch {+ // Fall back to the default when parallelism cannot be determined.+ }+ return Math.max(1, Math.min(cpus, packageCount));+}++// Bounded-concurrency task pool. Workers pull from a shared cursor so no more+// than `limit` tasks run at once. Every task is awaited even if an earlier one+// fails, and failures are aggregated (sorted by original index) so a single bad+// package neither aborts the others mid-flight nor hides which one broke.+export async function runWithConcurrency(items, limit, worker) {+ const results = new Array(items.length);+ const failures = [];+ let nextIndex = 0;++ async function runWorker() {+ while (true) {+ const current = nextIndex;+ nextIndex += 1;+ if (current >= items.length) {+ return;+ }+ try {+ results[current] = await worker(items[current], current);+ } catch (error) {+ failures.push({ index: current, error });+ }}}++ const poolSize = Math.max(1, Math.min(limit, items.length));+ await Promise.all(Array.from({ length: poolSize }, () => runWorker()));++ if (failures.length > 0) {+ failures.sort((a, b) => a.index - b.index);+ const aggregate = new Error(+ `${failures.length} standalone package build(s) failed`,+ );+ aggregate.failures = failures;+ throw aggregate;+ }++ return results;}-main();+async function main() {+ const workspaceEntries = parseWorkspaceEntries(readFileSync(workspacePath, "utf8"));+ const standalonePackages = listPublicPackages()+ .filter(({ dir }) => !isWorkspacePackage(dir, workspaceEntries));++ if (standalonePackages.length === 0) {+ console.log(" i No standalone public packages detected outside the pnpm workspace");+ return;+ }++ const concurrency = resolveConcurrency(standalonePackages.length);+ console.log(+ ` Building ${standalonePackages.length} standalone package(s) with concurrency ${concurrency}`,+ );++ try {+ await runWithConcurrency(+ standalonePackages,+ concurrency,+ prepareAndBuildPackage,+ );+ } catch (error) {+ if (Array.isArray(error.failures)) {+ const names = error.failures+ .map(({ index }) => standalonePackages[index]?.name)+ .filter(Boolean)+ .join(", ");+ console.error(` ✗ Failed standalone package build(s): ${names}`);+ }+ throw error;+ }+}++const invokedDirectly =+ process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);++if (invokedDirectly) {+ main().catch((error) => {+ console.error(error);+ process.exit(1);+ });+}