The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
perf(release): wait for npm registry visibility of all packages concurrently (#13495)
Devin Foleybfb4ceabbbb0
diff --git a/scripts/release-lib.sh b/scripts/release-lib.shindex 5aa5431e90..da160af004 100644--- a/scripts/release-lib.sh+++ b/scripts/release-lib.sh@@ -410,22 +410,80 @@ publish_package_to_npm() {return 1}-publish_package_to_npm_and_wait() {- local dist_tag="$1"- local package_name="$2"- local package_version="$3"- local publish_tool="${4:-pnpm}"- local attempts="${5:-12}"- local delay_seconds="${6:-5}"-- publish_package_to_npm "$dist_tag" "$package_name" "$package_version" "$publish_tool" || return 1+# Wait for every already-published package to become registry-visible,+# polling all of them concurrently. npm accepts a publish in seconds, but+# packument propagation through the registry CDN can lag minutes per package;+# waiting on each package before publishing the next made the total wait the+# SUM of every package's lag (~2 hours on a bad day for the full set). Every+# publish has already been accepted by the time this runs, so the polls can+# race: the wall-clock cost becomes the single slowest package's lag. Each+# package keeps its own attempts x delay budget, and a package that never+# becomes visible still fails the release, naming every straggler.+#+# $3 is the list_public_package_info tuple list (pkg_dir<TAB>name<TAB>version+# lines); the directory field is ignored.+wait_for_npm_package_versions() {+ local attempts="${1:-12}"+ local delay_seconds="${2:-5}"+ local package_info="$3"++ # The polling phase runs in a subshell that owns its own EXIT trap: a+ # cancelled or signalled release reaps every in-flight poller and the+ # scratch directory instead of leaking one npm poll per package for the+ # rest of its budget. The subshell also keeps this trap from clobbering+ # the caller's cleanup trap.+ (+ local status_dir+ local pids=()+ local specs=()+ local failures=()+ local index=0+ local pkg_name+ local pkg_version+ local i++ status_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-release-visibility.XXXXXX")"++ # shellcheck disable=SC2329 # invoked via the trap below+ reap_visibility_pollers() {+ local pid+ for pid in ${pids[@]+"${pids[@]}"}; do+ kill "$pid" 2>/dev/null || true+ done+ rm -rf "$status_dir"+ }+ trap reap_visibility_pollers EXIT INT TERM++ while IFS=$'\t' read -r _pkg_dir pkg_name pkg_version; do+ [ -z "$pkg_name" ] && continue+ (+ if wait_for_npm_package_version "$pkg_name" "$pkg_version" "$attempts" "$delay_seconds"; then+ : > "$status_dir/$index.ok"+ fi+ ) &+ pids+=("$!")+ specs+=("${pkg_name}@${pkg_version}")+ index=$((index + 1))+ done <<< "$package_info"++ if [ "${#pids[@]}" -gt 0 ]; then+ for i in "${!pids[@]}"; do+ wait "${pids[$i]}" || true+ if [ -e "$status_dir/$i.ok" ]; then+ release_info " ✓ ${specs[$i]} is registry-visible"+ else+ failures+=("${specs[$i]}")+ fi+ done+ fi- if wait_for_npm_package_version "$package_name" "$package_version" "$attempts" "$delay_seconds"; then- return 0- fi+ if [ "${#failures[@]}" -gt 0 ]; then+ release_warn "npm accepted every publish, but these versions did not become registry-visible: ${failures[*]}"+ exit 1+ fi- release_warn "npm accepted ${package_name}@${package_version}, but the version did not become registry-visible."- return 1+ exit 0+ )}verify_npm_installable() {diff --git a/scripts/release-lib.test.mjs b/scripts/release-lib.test.mjsindex 4d8da63ab2..93f3272359 100644--- a/scripts/release-lib.test.mjs+++ b/scripts/release-lib.test.mjs@@ -29,6 +29,7 @@ function runPublishHelper({npmVersionExistsAfterChecks = 0,verifyAttempts = 1,verifyDelaySeconds = 0,+ visibilityPackages = null,}) {const fixtureDir = mkdtempSync(join(tmpdir(), "paperclip-release-lib-"));const binDir = join(fixtureDir, "bin");@@ -86,6 +87,26 @@ exit 1set -euo pipefailprintf 'npm %s\\n' "$*" >> "$FAKE_CALL_LOG"if [ "$1" = "view" ]; then+ if [ "\${NPM_VIEW_CROSS:-}" = "true" ]; then+ # Cross-visibility mode: a package resolves only after every OTHER+ # package has been polled at least once. Mutually dependent visibility+ # can only converge when the polls run concurrently.+ spec="$2"+ name="\${spec%@*}"+ safe="$(printf '%s' "$name" | tr '/@' '__')"+ touch "$FAKE_STATE_DIR/seen-$safe"+ all_seen=true+ for other in $CROSS_PACKAGES; do+ osafe="$(printf '%s' "$other" | tr '/@' '__')"+ [ "$osafe" = "$safe" ] && continue+ [ -f "$FAKE_STATE_DIR/seen-$osafe" ] || all_seen=false+ done+ if [ "$all_seen" = "true" ]; then+ echo "1.2.3"+ exit 0+ fi+ exit 1+ fichecks=0if [ -f "$FAKE_STATE_DIR/view-checks" ]; thenread -r checks < "$FAKE_STATE_DIR/view-checks"@@ -147,15 +168,30 @@ exec npm "$@");const shellOptions = callerPipefail ? "set -euo pipefail" : "set -eu";+ // The cross-visibility concurrency test needs real (fractional-second)+ // sleeps so slower-starting sibling polls get a chance to run; every other+ // test records virtual waiting so registry-delay tests stay fast/offline.+ const sleepStub = visibilityPackages+ ? ""+ : `sleep() { printf 'sleep %s\\n' "$*" >> "$FAKE_CALL_LOG"; }`;+ const packageInfo = (+ visibilityPackages ?? ["@paperclipai/example"]+ )+ .map((name) => `packages/example\\t${name}\\t1.2.3`)+ .join("\\n");const script = `${shellOptions}source "${repoRoot}/scripts/release-lib.sh"-# Record virtual waiting so registry-delay tests remain fast and offline.-sleep() { printf 'sleep %s\\n' "$*" >> "$FAKE_CALL_LOG"; }+${sleepStub}${- waitForRegistry- ? `publish_package_to_npm_and_wait ${distTag} @paperclipai/example 1.2.3 ${publishTool} "$VERIFY_ATTEMPTS" "$VERIFY_DELAY_SECONDS"`- : `publish_package_to_npm ${distTag} @paperclipai/example 1.2.3 ${publishTool}`+ visibilityPackages+ ? `PACKAGE_INFO="$(printf '${packageInfo}')"+wait_for_npm_package_versions "$VERIFY_ATTEMPTS" "$VERIFY_DELAY_SECONDS" "$PACKAGE_INFO"`+ : waitForRegistry+ ? `publish_package_to_npm ${distTag} @paperclipai/example 1.2.3 ${publishTool}+PACKAGE_INFO="$(printf '${packageInfo}')"+wait_for_npm_package_versions "$VERIFY_ATTEMPTS" "$VERIFY_DELAY_SECONDS" "$PACKAGE_INFO"`+ : `publish_package_to_npm ${distTag} @paperclipai/example 1.2.3 ${publishTool}`}`;@@ -174,6 +210,8 @@ ${NPM_VERSION_EXISTS_AFTER_CHECKS: String(npmVersionExistsAfterChecks),VERIFY_ATTEMPTS: String(verifyAttempts),VERIFY_DELAY_SECONDS: String(verifyDelaySeconds),+ NPM_VIEW_CROSS: visibilityPackages ? "true" : "false",+ CROSS_PACKAGES: (visibilityPackages ?? []).join(" "),PNPM_MODE: pnpmMode,REPO_ROOT: fixtureDir,},@@ -270,7 +308,7 @@ test("publish_package_to_npm does not retry stable publishes without provenance"assert.doesNotMatch(result.calls, /--provenance=false/);});-test("publish_package_to_npm_and_wait confirms registry visibility before returning", () => {+test("wait_for_npm_package_versions confirms registry visibility after a publish", () => {const result = runPublishHelper({pnpmMode: "success",npmVersionExists: true,@@ -282,12 +320,29 @@ test("publish_package_to_npm_and_wait confirms registry visibility before returnassert.match(result.calls, /^npm view @paperclipai\/example@1\.2\.3 version$/m);});-test("publish_package_to_npm_and_wait blocks the release when registry visibility lags", () => {+test("wait_for_npm_package_versions blocks the release and names the straggler", () => {const result = runPublishHelper({ pnpmMode: "success", waitForRegistry: true });assert.notEqual(result.status, 0);assert.match(result.calls, /^npm view @paperclipai\/example@1\.2\.3 version$/m);- assert.match(result.output, /did not become registry-visible/);+ assert.match(result.output, /did not become registry-visible: @paperclipai\/example@1\.2\.3/);+});++test("wait_for_npm_package_versions polls every package concurrently", () => {+ // In cross-visibility mode each fake package resolves only after the OTHER+ // package has been polled at least once. Waiting out one package's full+ // budget before polling the next can never satisfy the first package;+ // only concurrent polling converges.+ const result = runPublishHelper({+ pnpmMode: "success",+ visibilityPackages: ["@paperclipai/alpha", "@paperclipai/beta"],+ verifyAttempts: 50,+ verifyDelaySeconds: 0.2,+ });++ assert.equal(result.status, 0, result.output);+ assert.match(result.output, /@paperclipai\/alpha@1\.2\.3 is registry-visible/);+ assert.match(result.output, /@paperclipai\/beta@1\.2\.3 is registry-visible/);});test("the workflow budget tolerates the observed 15-minute 20-second registry delay", () => {diff --git a/scripts/release.sh b/scripts/release.shindex 9c94f467e6..c6720e9962 100755--- a/scripts/release.sh+++ b/scripts/release.sh@@ -342,23 +342,29 @@ elsenode "$REPO_ROOT/scripts/prepare-bundled-package.mjs" "$REPO_ROOT/$pkg_dir" "$publish_dir"cd "$publish_dir"fi- if ! publish_package_to_npm_and_wait \+ if ! publish_package_to_npm \"$DIST_TAG" \"$pkg_name" \"$pkg_version" \- "$publish_tool" \- "$VERIFY_ATTEMPTS" \- "$VERIFY_DELAY_SECONDS"; then+ "$publish_tool"; thenif [ "$publish_tool" = "npm" ]; thenrm -rf "$publish_dir"fi- release_fail "stopping release: npm did not publish and expose ${pkg_name}@${pkg_version}"+ release_fail "stopping release: npm did not accept the publish of ${pkg_name}@${pkg_version}"fiif [ "$publish_tool" = "npm" ]; thenrm -rf "$publish_dir"fi- release_info " ✓ Published version is registry-visible"+ release_info " ✓ npm accepted the publish"done <<< "$VERSIONED_PACKAGE_INFO"+ release_info " ✓ npm accepted every package publish under dist-tag $DIST_TAG"+ release_info " Waiting for registry visibility of all packages concurrently..."+ if ! wait_for_npm_package_versions \+ "$VERIFY_ATTEMPTS" \+ "$VERIFY_DELAY_SECONDS" \+ "$VERSIONED_PACKAGE_INFO"; then+ release_fail "stopping release: npm did not expose every published version under dist-tag $DIST_TAG"+ firelease_info " ✓ Published the full package set under dist-tag $DIST_TAG"fi@@ -369,7 +375,7 @@ elserelease_info "==> Step 6/7: Confirming npm package availability and dist-tag integrity..."REGISTRY_STATE_VERIFY_ATTEMPTS="${NPM_REGISTRY_STATE_VERIFY_ATTEMPTS:-12}"REGISTRY_STATE_VERIFY_DELAY_SECONDS="${NPM_REGISTRY_STATE_VERIFY_DELAY_SECONDS:-5}"- release_info " ✓ Every version was registry-visible before the next package publish"+ release_info " ✓ Every version was registry-visible before tagging began"verify_args=(--channel "$channel"