The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(ci): prefetch the consumer image and Grok binary during package staging (#15849)
Devin Foley42ecc96556e7
diff --git a/package.json b/package.jsonindex 3e026aa985..5a5000234b 100644--- a/package.json+++ b/package.json@@ -61,7 +61,7 @@"smoke:posthog-live": "node scripts/smoke/posthog-live.mjs","smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh","smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",- "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs",+ "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs","storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs","test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts","test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjsindex d084ee9be8..5a3dc487e3 100644--- a/scripts/__tests__/grok-public-install-sandbox.test.mjs+++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs@@ -81,6 +81,35 @@ test('verification never elevates PR-controlled provisioning or cleanup on the hassert.ok(source.includes("const prerequisite = join(root, 'native/grok')"));});+test('the image and Grok downloads start before staging and join without weakening the npm checks', () => {+ const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8');+ const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; };+ // Exactly the pinned consumer image is pulled, for the platform the probes run on.+ const imagePull = position("prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE])");+ // Grok is provisioned into the prefetch directory, never straight into `native/grok`.+ const grokProvision = position("prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok])");+ assert.ok(source.includes("const stagedGrok = join(prefetched, 'grok')"));+ assert.doesNotMatch(source, /provision-grok\.mjs'\), prerequisite\]/);+ // Both downloads start before any package is staged and before the first sandbox runs.+ const staging = position("visit('@paperclipai/server')");+ const firstSandbox = position("isolated(['npm', 'install', '--ignore-scripts', '--omit=dev'");+ assert.ok(imagePull < staging && grokProvision < staging);+ assert.ok(position("await awaited('image', imageReady)") < firstSandbox);+ // The binary moves to the canonical host path only after npm is proven not to+ // have provisioned it, after the negative probe, and before the positive probe.+ const npmCheck = position("'npm must not provision Grok'");+ const negativeProbe = position("isolated(['node', '/packages/probe.mjs', 'missing'])");+ const grokJoin = position("await awaited('grok', grokReady)");+ const move = position('renameSync(stagedGrok, prerequisite)');+ const positiveProbe = position("isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite })");+ assert.ok(npmCheck < negativeProbe && negativeProbe < grokJoin && grokJoin < move && move < positiveProbe);+ // The children run through the helper covered by prefetch-child.test.mjs, and+ // cleanup waits for them before removing the directory they write into.+ assert.ok(source.includes("import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'"));+ assert.ok(position('await stopPrefetches(prefetches)') < position('rmSync(root, { recursive: true, force: true })'));+ assert.equal(source.match(/rmSync\(root/g).length, 1);+});+test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => {const root = realpathSync(mkdtempSync(join(tmpdir(), 'installed-codex-probe-')));try {diff --git a/scripts/__tests__/prefetch-child.test.mjs b/scripts/__tests__/prefetch-child.test.mjsnew file mode 100644index 0000000000..bd3d9864be--- /dev/null+++ b/scripts/__tests__/prefetch-child.test.mjs@@ -0,0 +1,93 @@+import assert from 'node:assert/strict';+import { spawnSync } from 'node:child_process';+import { mkdtempSync, readFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import test from 'node:test';+import { startPrefetch, stopPrefetches } from '../prefetch-child.mjs';++// Small local Node children stand in for the image pull and the Grok download;+// none of these tests needs Docker or a network.+const withScratch = async body => {+ const root = mkdtempSync(join(tmpdir(), 'prefetch-child-'));+ try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); }+};+const start = (root, name, script, cmd = process.execPath, args = ['-e', script]) =>+ startPrefetch({ name, cmd, args, cwd: root, env: process.env, logPath: join(root, `${name}.log`) });+const blockFor = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);+const untilLogHas = async (path, text) => {+ for (let attempt = 0; attempt < 400; attempt++) {+ try { if (readFileSync(path, 'utf8').includes(text)) return; } catch {}+ await new Promise(resolve => setTimeout(resolve, 25));+ }+ throw new Error(`log never contained ${text}`);+};++test('a successful prefetch resolves, keeps its output, and is left alone by stop', async () => withScratch(async root => {+ const handle = start(root, 'ok', "setTimeout(() => process.stdout.write('fetched\\n'), 150)");+ await handle.done;+ assert.equal(handle.child.exitCode, 0);+ assert.equal(readFileSync(join(root, 'ok.log'), 'utf8'), 'fetched\n');+ assert.ok(handle.durationMs >= 100 && handle.durationMs < 5000, `durationMs ${handle.durationMs}`);+ await stopPrefetches([handle]);+ assert.equal(handle.child.signalCode, null);+}));++test('the duration ends at the download, not when a blocked parent notices the exit', async () => withScratch(async root => {+ const handle = start(root, 'early', "setTimeout(() => process.stdout.write('Status: Downloaded\\n'), 100)");+ // Synchronous work in the parent, as package staging is in the verifier.+ blockFor(1500);+ const noticed = Date.now();+ await handle.done;+ assert.ok(Date.now() - noticed < 500, 'the exit was pending by the time the parent yielded');+ assert.ok(handle.durationMs < 900, `durationMs ${handle.durationMs} must not include the 1500ms the parent was blocked`);+}));++test('a nonzero exit rejects with the log tail where the result is awaited', async () => withScratch(async root => {+ const handle = start(root, 'bad', "process.stderr.write('registry unreachable\\n'); process.exit(3)");+ await assert.rejects(handle.done, /^Error: bad prefetch failed \(exit 3\): registry unreachable$/);+ await stopPrefetches([handle]);+}));++test('a command that cannot start rejects instead of hanging the join or the stop', async () => withScratch(async root => {+ const handle = start(root, 'missing', '', join(root, 'no-such-command'), []);+ await assert.rejects(handle.done, /missing prefetch failed \(.*ENOENT/);+ await stopPrefetches([handle]);+}));++test('stopping a running prefetch waits for it to exit so its directory can be removed safely', async () => withScratch(async root => {+ const handle = start(root, 'slow', "process.stdout.write('started\\n'); setInterval(() => {}, 1000)");+ await untilLogHas(join(root, 'slow.log'), 'started');+ const stopping = Date.now();+ await stopPrefetches([handle]);+ assert.ok(Date.now() - stopping < 4000);+ assert.equal(handle.child.signalCode, 'SIGTERM');+ assert.equal(handle.child.exitCode, null);+ await assert.rejects(handle.done, /slow prefetch failed \(SIGTERM\): started/);+}));++test('a prefetch that ignores SIGTERM is killed after the grace period', async () => withScratch(async root => {+ const handle = start(root, 'stubborn', "process.on('SIGTERM', () => {}); process.stdout.write('ignoring\\n'); setInterval(() => {}, 1000)");+ await untilLogHas(join(root, 'stubborn.log'), 'ignoring');+ const stopping = Date.now();+ await stopPrefetches([handle], { graceMs: 200 });+ assert.ok(Date.now() - stopping >= 200 && Date.now() - stopping < 4000);+ assert.equal(handle.child.signalCode, 'SIGKILL');+}));++test('an unawaited failure never becomes an unhandled rejection', () => {+ const script = `+ import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)};+ import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path';+ const root = mkdtempSync(join(tmpdir(), 'prefetch-unawaited-'));+ try {+ startPrefetch({ name: 'ignored', cmd: process.execPath, args: ['-e', 'process.exit(2)'], cwd: root, env: process.env, logPath: join(root, 'ignored.log') });+ await new Promise(resolve => setTimeout(resolve, 500));+ console.log('still running');+ } finally { rmSync(root, { recursive: true, force: true }); }+ `;+ const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { encoding: 'utf8', timeout: 10_000 });+ assert.equal(result.status, 0, result.stderr);+ assert.match(result.stdout, /still running/);+ assert.doesNotMatch(result.stderr, /unhandled/i);+});diff --git a/scripts/prefetch-child.mjs b/scripts/prefetch-child.mjsnew file mode 100644index 0000000000..bdaf3d25a8--- /dev/null+++ b/scripts/prefetch-child.mjs@@ -0,0 +1,60 @@+// Background downloads for verification scripts: start a child early with its+// output in a log file, join it where the result is needed, and stop it before+// the scratch directory it writes into is removed. No shell, no inherited+// terminal, and a failure is reported where the caller awaits it.+import { spawn } from 'node:child_process';+import { closeSync, openSync, readFileSync, statSync } from 'node:fs';++const running = child => child.pid !== undefined && child.exitCode === null && child.signalCode === null;++// Returns { name, child, done, durationMs }. `done` resolves on exit code 0+// and otherwise rejects with the tail of the log; the rejection is observed+// here so an unawaited failure never surfaces as an unhandled rejection.+export function startPrefetch({ name, cmd, args, cwd, env, logPath }) {+ const startedAt = Date.now();+ const log = openSync(logPath, 'w');+ let child;+ try {+ child = spawn(cmd, args, { cwd, env, stdio: ['ignore', log, log] });+ } finally {+ closeSync(log); // the child holds its own descriptor+ }+ const handle = { name, child, done: null, durationMs: null };+ let settled = false;+ handle.done = new Promise((settle, reject) => {+ const finish = failure => {+ if (settled) return;+ settled = true;+ // The caller may be blocked in synchronous work when the child exits, so+ // this event can arrive long after the fact. When the child wrote+ // anything, its last write is the end time; both downloads this serves+ // print their final status line as they complete. A silent child falls+ // back to the time the exit was observed.+ let finishedAt = Date.now();+ try {+ const { mtimeMs, size } = statSync(logPath);+ if (size > 0) finishedAt = Math.min(finishedAt, mtimeMs);+ } catch {}+ handle.durationMs = Math.max(0, Math.round(finishedAt - startedAt));+ if (failure === null) return settle();+ let output = '';+ try { output = readFileSync(logPath, 'utf8').trim().slice(-2000); } catch {}+ reject(new Error(`${name} prefetch failed (${failure}): ${output}`));+ };+ child.once('error', error => finish(error.message));+ child.once('exit', (code, signal) => finish(code === 0 ? null : (signal ?? `exit ${code}`)));+ });+ handle.done.catch(() => {});+ return handle;+}++// Stops every prefetch that is still running and resolves only once all of+// them have exited, so the caller can remove their working directory without+// racing a write. A child that ignores SIGTERM is killed after `graceMs`.+export async function stopPrefetches(prefetches, { graceMs = 5000 } = {}) {+ await Promise.all(prefetches.map(async ({ child, done }) => {+ if (running(child)) child.kill('SIGTERM');+ const escalate = setTimeout(() => { if (running(child)) child.kill('SIGKILL'); }, graceMs);+ try { await done; } catch {} finally { clearTimeout(escalate); }+ }));+}diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjsindex b3c2db21d2..d35c40e818 100644--- a/scripts/verify-grok-npm-install.mjs+++ b/scripts/verify-grok-npm-install.mjs@@ -4,12 +4,13 @@// explicit and separate from npm installation, and is removed in finally.import assert from 'node:assert/strict';import { execFileSync } from 'node:child_process';-import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';+import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';import { tmpdir } from 'node:os';import { basename, dirname, join, resolve } from 'node:path';import { fileURLToPath } from 'node:url';import { materializePublishManifest, prepareBundledPackage } from './prepare-bundled-package.mjs';import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs';+import { startPrefetch, stopPrefetches } from './prefetch-child.mjs';import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs';const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host');@@ -19,7 +20,31 @@ const env = { ...process.env, NODE_PATH: '', PAPERCLIP_RELEASE_REUSE_UI_DIST: '1const run = (cmd, args, cwd = root) => execFileSync(cmd, args, { cwd, env, stdio: 'pipe', maxBuffer: 32 * 1024 * 1024 });const sourceRevision = run('git', ['rev-parse', 'HEAD'], repo).toString().trim();const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`;+// The two network downloads the probes need do not depend on anything packed+// below, so they overlap the CPU-bound staging instead of running after it.+// The consumer image is pinned by digest and the Grok binary is checked against+// its pinned sha256 by the provisioning script, so an early start changes no+// verified property. Neither touches the consumer, the assets, nor the+// `native/grok` path that the post-install assertion inspects.+const prefetched = join(root, 'prefetch'); mkdirSync(prefetched);+const stagedGrok = join(prefetched, 'grok');+const prefetches = [];+const timing = {};+// A prefetch failure surfaces where its result is awaited, never as an+// unhandled rejection that would mask an earlier staging error.+const prefetch = (name, cmd, args) => {+ const handle = startPrefetch({ name, cmd, args, cwd: root, env, logPath: join(prefetched, `${name}.log`) });+ prefetches.push(handle);+ return handle.done;+};+let phaseStarted = performance.now();+const phase = name => { const now = performance.now(); timing[`${name}_ms`] = Math.round(now - phaseStarted); phaseStarted = now; };+const awaited = async (name, promise) => { const started = performance.now(); await promise; timing[`${name}_wait_ms`] = Math.round(performance.now() - started); };try {+ const imageReady = prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE]);+ // Provision as the unprivileged verification user, never into the host's /opt.+ // Only the positive probe sees this file at the canonical sandbox path.+ const grokReady = prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok]);const listing = run(process.execPath, [join(repo, 'scripts/release-package-map.mjs'), 'list'], repo).toString().trim().split('\n').map(line => line.split('\t'));const packages = new Map(listing.map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(join(repo, dir, 'package.json'), 'utf8')) }]));const needed = new Set();@@ -81,9 +106,12 @@ try {const sentinel = join(consumer, 'node_modules/paperclip-verification-lifecycle-sentinel/lifecycle-ran');const consumerUid = process.getuid();const isolated = (command, options = {}) => run('docker', grokConsumerDockerArgs({ assets, consumer, cache, uid: consumerUid, gid: process.getgid(), command, ...options }));+ phase('stage_packages');+ await awaited('image', imageReady);// npm resolution is intentionally the public consumer graph, not the pnpm// workspace graph. Freeze that result before any lifecycle script can run.isolated(['npm', 'install', '--ignore-scripts', '--omit=dev', ...readdirSync(assets).filter(file => file.endsWith('.tgz')).map(file => `/packages/${file}`)], { download: true });+ phase('consumer_install');assert.equal(existsSync(sentinel), false, 'Dependency download must not run lifecycle scripts');const consumerLock = readFileSync(join(consumer, 'package-lock.json'), 'utf8');// npm ci rejects bundled optional platform dependencies absent from its own@@ -92,6 +120,7 @@ try {isolated(GROK_PUBLIC_INSTALL_LIFECYCLE);assert.equal(readFileSync(sentinel, 'utf8'), 'ok', 'Offline lifecycle scripts must actually execute');assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Lifecycle execution must preserve the resolved consumer lock');+ phase('lifecycle_rebuild');for (const name of needed) {const installedManifest = JSON.parse(readFileSync(join(consumer, 'node_modules', name, 'package.json'), 'utf8'));assert.equal(installedManifest.version, releaseVersion, `Installed release version for ${name}`);@@ -104,6 +133,7 @@ try {writeFileSync(join(assets, 'codex-probe.mjs'), installedCodexProbeSource('/consumer/node_modules/@paperclipai/server/dist/vendor/paperclip-runner/index.js', '/consumer', codexVersion), { mode: 0o644 });const codex = JSON.parse(isolated(['node', '/packages/codex-probe.mjs']).toString().trim());+ phase('codex_probe');assert.equal(codex.codexConsumerHostOnly, true);assert.equal(codex.codexPlatformPackageSource, 'official npm');assert.ok(existsSync(join(installed, 'providers/grok/launcher.cjs')));@@ -122,10 +152,14 @@ try {`;writeFileSync(join(assets, 'probe.mjs'), probe, { mode: 0o644 });isolated(['node', '/packages/probe.mjs', 'missing']);- // Provision as the unprivileged verification user, never into the host's /opt.- // Only the positive probe sees this file at the canonical sandbox path.- run(process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), prerequisite]);+ // The binary was provisioned outside `native/grok` while the packages were+ // staged, so the assertion above still proves npm put nothing there. It moves+ // into place only now, so only the positive probe sees it.+ await awaited('grok', grokReady);+ mkdirSync(dirname(prerequisite), { recursive: true });+ renameSync(stagedGrok, prerequisite);isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite });+ phase('grok_probes');console.log(JSON.stringify({ schema: 'paperclip.grok.public-npm-install.v1', sourceRevision, releaseVersion, lifecycleScriptsEnabled: true, lifecycleSentinelVerified: true, lifecycleNetwork: 'none', consumerImage: GROK_PUBLIC_INSTALL_IMAGE, consumerUid, consumerLockPreserved: true, cleanNpmInstall: true, packageCount: needed.size, codexPackedPayloadsOmitted: true, codexPackedPackagesChecked: tarballs.length, ...codex, builtinLauncherPresent: true, separateGrokPackage: false, npmProvisionedBinary: false, missingPrerequisiteRejected: true, provisionedBinaryVerified: true, commandLeaseVerified: true, providerCalls: 0 }));// Exercise Pi's public CLI and installed server, never a private workspace// package or binary override. Public dependency downloads are explicit and@@ -137,6 +171,7 @@ try {// temporary capacity than the smaller offline lifecycle probes.const setup = isolated(['node', '/consumer/node_modules/paperclipai/dist/index.js', 'runtime', 'setup', 'pi'], { download: true, temporarySizeMiB: 2048 }).toString();const receipt = JSON.parse(setup.trim());+ phase('pi_setup');assert.equal(receipt.status, 'installed_verified');assert.equal(receipt.target, 'linux-x64');assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Pi setup must preserve the consumer dependency graph');@@ -144,6 +179,13 @@ try {// Docker defaults tmpfs to noexec. The offline admission probe must execute// its verified private snapshot while keeping lifecycle/download scratch noexec.console.log(isolated(['node', '/packages/pi-public-install-probe.mjs', '/consumer/node_modules/@paperclipai/server'], { temporarySizeMiB: 2048, temporaryExecutable: true }).toString().trim());+ phase('pi_probe');+ // Wall time per phase, so the slow part of this step is visible in the CI log+ // without re-running it. `*_prefetch_ms` runs from a download's start to its+ // last output, and `*_wait_ms` is how long the main flow waited for a+ // prefetch that had not finished yet; zero means the overlap hid it entirely.+ const downloads = Object.fromEntries(prefetches.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs]));+ console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloads }));if (process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR) {const retained = retainRunnerQualificationPackages({ repo, output: process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR, sourceRevision, releaseVersion, env,publicArchives: [...needed].map((name, index) => ({ name, file: tarballs[index] })),@@ -151,5 +193,8 @@ try {console.log(JSON.stringify(retained));}} finally {+ // An earlier failure must not leave a prefetch running, and scratch is+ // removed only once both children have stopped writing into it.+ await stopPrefetches(prefetches);rmSync(root, { recursive: true, force: true });}