Skip to content

The open-source app everyone uses to manage agents at work

perf(ci): pack the verifier's staged packages concurrently (#15888)

Devin Foley1ac48aff8dc1
diff --git a/package.json b/package.json
index 29b999b28e..1d8f7fb078 100644
--- a/package.json
+++ b/package.json
@@ -61,7 +61,7 @@
"smoke:posthog-live": "node scripts/smoke/posthog-live.mjs",
"smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh",
"smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",
- "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs",
+ "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs",
"storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs",
"test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts",
"test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",
diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs
index 11fde52433..3cf43712a1 100644
--- a/scripts/__tests__/grok-public-install-sandbox.test.mjs
+++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs
@@ -120,19 +120,28 @@ test('the bundled-dependency installs run together while the other packages are
// Only the bundled packages' sources are staged before the installs start;
// the installs are registered for cleanup at once, before anything else can
// fail; the other packages are copied while the installs run; the join comes
- // before the first pack; and the pack loop walks every package in order.
+ // before the packs start; and the packs run through the helper covered by
+ // pack-staged-packages.test.mjs, bundled packages first, each registered for
+ // cleanup as it starts, with the results put back into dependency order
+ // before every tarball is checked.
const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)');
const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })");
const registered = position('prefetches.push(...installs.map(({ handle }) => handle))');
const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))');
const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))");
- const packLoop = position('for (const { target } of entries)');
- const firstPack = position("run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target)");
- assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packLoop && packLoop < firstPack);
- assert.equal(source.match(/run\('npm', \['pack'/g).length, 2, 'one package pack loop plus the lifecycle sentinel');
- // The timing line reports the split staging phase and each install.
- for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms']) assert.ok(source.includes(`timing.${key} =`));
+ assert.ok(source.includes("import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs'"));
+ const packs = position("await packStagedPackages([...entries].sort((a, b) => Number(b.bundled) - Number(a.bundled)), { logDir: prefetched, env, concurrency, register: handle => prefetches.push(handle) })");
+ const reordered = position('packs.sort((a, b) => a.index - b.index)');
+ const checks = position("for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball])");
+ const copied = position('for (const tarball of tarballs) {');
+ assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packs && packs < reordered && reordered < checks && checks < copied);
+ assert.equal(source.match(/run\('npm', \['pack'/g).length, 1, 'only the lifecycle sentinel is packed in-process');
+ assert.ok(source.includes("destination: join(root, `pack-${index}`)"), 'each pack writes into its own directory');
+ // The timing line reports the split staging phase, each install, and the
+ // pack step with what the packs would have cost one after another.
+ for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms', 'stage_pack_check_ms']) assert.ok(source.includes(`timing.${key} =`));
assert.ok(source.includes('...bundledInstallTimings(installs)'));
+ assert.ok(source.includes('Object.assign(timing, packTimings(packs, concurrency))'));
});
test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => {
diff --git a/scripts/__tests__/pack-staged-packages.test.mjs b/scripts/__tests__/pack-staged-packages.test.mjs
new file mode 100644
index 0000000000..09c19cf64a
--- /dev/null
+++ b/scripts/__tests__/pack-staged-packages.test.mjs
@@ -0,0 +1,107 @@
+import assert from 'node:assert/strict';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import test from 'node:test';
+import { stopPrefetches } from '../prefetch-child.mjs';
+import { PACK_ARGS, packConcurrency, packStagedPackages, packTimings } from '../pack-staged-packages.mjs';
+
+// A small local script stands in for npm; no test here needs npm or a network.
+// It is run as `node <script> pack --ignore-scripts --pack-destination <dir>`,
+// the same argument list the real npm receives.
+const withScratch = async body => {
+ const root = mkdtempSync(join(tmpdir(), 'pack-staged-packages-'));
+ try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); }
+};
+const entriesFor = (root, names) => names.map((name, index) => {
+ const target = join(root, `package-${index}`);
+ mkdirSync(target);
+ return { name, index, target, destination: join(root, `pack-${index}`) };
+});
+const fakeNpm = (root, body) => { const script = join(root, 'fake-npm.mjs'); writeFileSync(script, body); return [process.execPath, script]; };
+const settle = ms => new Promise(resolve => setTimeout(resolve, ms));
+
+test('the concurrency defaults to one child per core', () => {
+ assert.ok(Number.isInteger(packConcurrency()) && packConcurrency() >= 1);
+ assert.deepEqual(PACK_ARGS, ['pack', '--ignore-scripts', '--pack-destination']);
+});
+
+test('packs run together up to the concurrency, in the order given, and the tarballs come back in entry order', async () => withScratch(async root => {
+ // Each stand-in records when it started and finished, writes one tarball
+ // named after its package, and prints the tarball name as npm does. The
+ // first two packs are slow and the last two fast, so with two children at
+ // a time the slow pair runs together and the fast pair starts only once one
+ // of them has finished; the fast packs then finish out of entry order.
+ const command = fakeNpm(root, `
+ import { basename, join } from 'node:path';
+ import { writeFileSync } from 'node:fs';
+ const args = process.argv.slice(2);
+ const destination = args[args.length - 1];
+ const name = basename(process.cwd());
+ const started = Date.now();
+ const delay = name === 'package-0' || name === 'package-1' ? 400 : 50;
+ Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay);
+ writeFileSync(join(destination, name + '-1.0.0.tgz'), 'tarball');
+ writeFileSync(join(destination, '..', 'span-' + name), started + ' ' + Date.now());
+ console.log(args.join(' '));
+ console.log(name + '-1.0.0.tgz');
+ `);
+ const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db', '@paperclipai/shared', 'paperclipai']);
+ const registered = [];
+ const packs = await packStagedPackages(entries, { logDir: root, concurrency: 2, register: handle => registered.push(handle), command });
+ assert.deepEqual(packs.map(({ name }) => name), entries.map(({ name }) => name));
+ assert.deepEqual(packs.map(({ index }) => index), [0, 1, 2, 3]);
+ for (const [index, pack] of packs.entries()) {
+ assert.equal(pack.tarball, join(root, `pack-${index}`, `package-${index}-1.0.0.tgz`));
+ assert.equal(readFileSync(pack.tarball, 'utf8'), 'tarball');
+ assert.ok(Number.isInteger(pack.durationMs) && pack.durationMs >= 0);
+ const log = readFileSync(join(root, `pack-${index}.log`), 'utf8');
+ assert.equal(log.split('\n')[0], `pack --ignore-scripts --pack-destination ${join(root, `pack-${index}`)}`);
+ }
+ assert.equal(registered.length, 4, 'every pack is registered for cleanup as it starts');
+ for (const handle of registered) assert.equal(handle.child.exitCode, 0);
+ const span = name => readFileSync(join(root, `span-${name}`), 'utf8').split(' ').map(Number);
+ const [slowA, slowB, fastA, fastB] = ['package-0', 'package-1', 'package-2', 'package-3'].map(span);
+ assert.ok(slowA[0] < slowB[1] && slowB[0] < slowA[1], 'the two slow packs ran at the same time');
+ const firstSlowEnd = Math.min(slowA[1], slowB[1]);
+ assert.ok(fastA[0] >= firstSlowEnd - 50 && fastB[0] >= firstSlowEnd - 50, 'a third pack starts only once a slot is free');
+ const timings = packTimings(packs, 2);
+ assert.equal(timings.pack_concurrency, 2);
+ assert.equal(timings.pack_serial_ms, packs.reduce((total, { durationMs }) => total + durationMs, 0));
+ assert.equal(timings.pack_longest_ms, Math.max(...packs.map(({ durationMs }) => durationMs)));
+}));
+
+test('a failed pack rejects with the tail of its log, launches nothing more, and leaves the running pack to be stopped', async () => withScratch(async root => {
+ const command = fakeNpm(root, `
+ import { basename } from 'node:path';
+ if (basename(process.cwd()) === 'package-1') { console.error('pack exploded'); process.exit(2); }
+ setTimeout(() => {}, 30_000);
+ `);
+ const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db', '@paperclipai/shared', 'paperclipai']);
+ const registered = [];
+ await assert.rejects(packStagedPackages(entries, { logDir: root, concurrency: 2, register: handle => registered.push(handle), command }), /pack-1 prefetch failed \(exit 2\): pack exploded/);
+ await settle(100);
+ assert.equal(registered.length, 2, 'the two queued packs never start after the failure');
+ assert.equal(registered[0].child.exitCode, null, 'the other pack is still running when the failure surfaces');
+ assert.ok(!existsSync(join(root, 'pack-2')) && !existsSync(join(root, 'pack-3')));
+ await stopPrefetches(registered);
+ assert.notEqual(registered[0].child.signalCode ?? registered[0].child.exitCode, null);
+}));
+
+test('a pack that exits without producing exactly one tarball is a failure', async () => withScratch(async root => {
+ const command = fakeNpm(root, `
+ import { basename, join } from 'node:path';
+ import { writeFileSync } from 'node:fs';
+ const destination = process.argv[process.argv.length - 1];
+ const name = basename(process.cwd());
+ if (name === 'package-0') { writeFileSync(join(destination, 'a.tgz'), ''); writeFileSync(join(destination, 'b.tgz'), ''); }
+ `);
+ const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']);
+ await assert.rejects(packStagedPackages(entries, { logDir: root, concurrency: 1, command }), /@paperclipai\/server pack produced 2 tarballs/);
+ assert.deepEqual(readdirSync(root).filter(name => name.startsWith('pack-') && !name.endsWith('.log')), ['pack-0'], 'the second pack never started');
+ await assert.rejects(packStagedPackages([entries[1]], { logDir: root, concurrency: 1, command }), /@paperclipai\/db pack produced 0 tarballs/);
+}));
+
+test('no entries resolve at once', async () => withScratch(async root => {
+ assert.deepEqual(await packStagedPackages([], { logDir: root, concurrency: 2, command: [process.execPath, '-e', ''] }), []);
+}));
diff --git a/scripts/pack-staged-packages.mjs b/scripts/pack-staged-packages.mjs
new file mode 100644
index 0000000000..2167ecdd0e
--- /dev/null
+++ b/scripts/pack-staged-packages.mjs
@@ -0,0 +1,64 @@
+// The public-install verifier packs every staged public package with
+// `npm pack`. Each pack reads only its own `package-<index>` directory and
+// writes one tarball, and the work is a Node start-up plus CPU-bound gzip per
+// package, so the packs run as concurrent children through the prefetch
+// helper, at most one per core, instead of one after another. Each child
+// packs into its own destination directory, so the tarball a package produced
+// is read from there rather than guessed from a shared directory, and the
+// results come back in the order the entries were given.
+import { mkdirSync, readdirSync } from 'node:fs';
+import { availableParallelism } from 'node:os';
+import { join } from 'node:path';
+import { startPrefetch } from './prefetch-child.mjs';
+
+export const PACK_ARGS = ['pack', '--ignore-scripts', '--pack-destination'];
+
+export const packConcurrency = () => Math.max(1, availableParallelism());
+
+// Packs `entries` (`{ name, target, destination }`) with at most `concurrency`
+// children at once, launched in the order given, and resolves with each entry
+// extended by its `tarball` path and `durationMs`, in the same order. Every
+// child is passed to `register` as it starts, so the caller can stop the ones
+// still running when something fails. The first failure rejects with the tail
+// of that child's log, and nothing further is launched after it.
+export function packStagedPackages(entries, { logDir, env = process.env, concurrency = packConcurrency(), register = () => {}, command = ['npm'] }) {
+ const [cmd, ...commandArgs] = command;
+ const results = new Array(entries.length);
+ const queue = entries.map((entry, index) => ({ entry, index }));
+ let active = 0;
+ let failed = false;
+ return new Promise((resolve, reject) => {
+ const fail = error => { failed = true; reject(error); };
+ const launch = () => {
+ if (!failed && active === 0 && queue.length === 0) return resolve(results);
+ while (!failed && active < concurrency && queue.length > 0) {
+ const { entry, index } = queue.shift();
+ active += 1;
+ let handle;
+ try {
+ mkdirSync(entry.destination, { recursive: true });
+ handle = startPrefetch({ name: `pack-${index}`, cmd, args: [...commandArgs, ...PACK_ARGS, entry.destination], cwd: entry.target, env, logPath: join(logDir, `pack-${index}.log`) });
+ register(handle);
+ } catch (error) {
+ return fail(error);
+ }
+ handle.done.then(() => {
+ const packed = readdirSync(entry.destination).filter(file => file.endsWith('.tgz'));
+ if (packed.length !== 1) throw new Error(`${entry.name} pack produced ${packed.length} tarballs in ${entry.destination}`);
+ results[index] = { ...entry, tarball: join(entry.destination, packed[0]), durationMs: handle.durationMs };
+ active -= 1;
+ launch();
+ }).catch(fail);
+ }
+ };
+ launch();
+ });
+}
+
+// Timing-line keys: the concurrency used, the sum of the per-pack times (what
+// the packs would have cost one after another) and the longest single pack.
+export const packTimings = (packs, concurrency) => ({
+ pack_concurrency: concurrency,
+ pack_serial_ms: packs.reduce((total, { durationMs }) => total + durationMs, 0),
+ pack_longest_ms: packs.reduce((longest, { durationMs }) => Math.max(longest, durationMs), 0),
+});
diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs
index a5844b220b..c7276a60b5 100644
--- a/scripts/verify-grok-npm-install.mjs
+++ b/scripts/verify-grok-npm-install.mjs
@@ -8,6 +8,7 @@ import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, re
import { tmpdir } from 'node:os';
import { basename, dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
+import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs';
import { materializePublishManifest } from './prepare-bundled-package.mjs';
import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs';
import { startPrefetch, stopPrefetches } from './prefetch-child.mjs';
@@ -73,12 +74,14 @@ try {
// each need a host-side `npm install` of their bundled closure; those installs
// write only their own `package-<index>` and read nothing another package
// produces, so they start first and run together while the other packages
- // are copied, instead of one after another inside one loop. Packing stays
- // serial and in order, because each tarball is checked as it is produced.
+ // are copied, instead of one after another inside one loop. Packing is a
+ // Node start-up plus CPU-bound gzip per package that reads only the package's
+ // own directory, so the packs also run together, each into its own
+ // `pack-<index>`, and the tarballs are checked in order once all are made.
const entries = [...needed].map((name, index) => {
const { dir, manifest } = packages.get(name);
const bundled = (manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length > 0;
- return { name, dir, manifest, bundled, target: join(root, `package-${index}`), stagedSource: join(root, `source-${index}`) };
+ return { name, index, dir, manifest, bundled, target: join(root, `package-${index}`), stagedSource: join(root, `source-${index}`), destination: join(root, `pack-${index}`) };
});
const stageSource = ({ dir, manifest, stagedSource }) => {
mkdirSync(stagedSource);
@@ -109,16 +112,20 @@ try {
await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)));
timing.stage_bundled_ms = Math.round(performance.now() - installsStarted);
const packStarted = performance.now();
- const tarballs = [];
- for (const { target } of entries) {
- run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target);
- const packed = readdirSync(root).filter(f => f.endsWith('.tgz') && !tarballs.includes(join(root, f)));
- assert.equal(packed.length, 1);
- const tarball = join(root, packed[0]);
- assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n'));
- tarballs.push(tarball);
- }
+ // The bundled packages make the largest tarballs, so they are launched first
+ // and the short packs fill the remaining slots behind them. Each pack is
+ // registered for cleanup as it starts, so a failure stops the ones still
+ // running before the scratch root is removed. The results are put back into
+ // dependency order before the tarballs are checked and copied.
+ const concurrency = packConcurrency();
+ const packs = await packStagedPackages([...entries].sort((a, b) => Number(b.bundled) - Number(a.bundled)), { logDir: prefetched, env, concurrency, register: handle => prefetches.push(handle) });
+ packs.sort((a, b) => a.index - b.index);
+ const checksStarted = performance.now();
+ const tarballs = packs.map(({ tarball }) => tarball);
+ for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n'));
+ timing.stage_pack_check_ms = Math.round(performance.now() - checksStarted);
timing.stage_pack_ms = Math.round(performance.now() - packStarted);
+ Object.assign(timing, packTimings(packs, concurrency));
const assets = join(root, 'assets'); mkdirSync(assets, { mode: 0o755 });
const consumer = join(root, 'consumer'); mkdirSync(consumer);
const cache = join(root, 'cache'); mkdirSync(cache);