Skip to content

The open-source app everyone uses to manage agents at work

fix(scripts): self-heal isolated workspace provisioning when the base CLI is broken (#10574)

Dotta79eff0aea146
diff --git a/scripts/__tests__/provision-worktree-self-heal.test.mjs b/scripts/__tests__/provision-worktree-self-heal.test.mjs
new file mode 100644
index 0000000000..9f44379501
--- /dev/null
+++ b/scripts/__tests__/provision-worktree-self-heal.test.mjs
@@ -0,0 +1,204 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import test from "node:test";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+const script = new URL("../provision-worktree.sh", import.meta.url).pathname;
+
+// Keep the PATH minimal so the fallback ladder is deterministic: node must be
+// reachable, but a globally installed `paperclipai` must not shadow the paths
+// under test.
+const testPath = [path.dirname(process.execPath), "/usr/bin", "/bin"].join(":");
+
+const cleanupDirs = [];
+
+function makeTempDir(prefix) {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
+ cleanupDirs.push(dir);
+ return dir;
+}
+
+test.after(() => {
+ for (const dir of cleanupDirs) {
+ fs.rmSync(dir, { recursive: true, force: true });
+ }
+});
+
+/**
+ * Writes a fake base workspace whose "tsx runner" is a plain node script, so
+ * the provision script's health check and init call can be steered per test.
+ *
+ * helpExit: exit code for `... index.ts --help` (the health check boot).
+ * initExit: exit code for `... index.ts worktree init ...`; on 0 the fake CLI
+ * writes a marker config so tests can tell CLI init from fallback.
+ */
+function makeBaseWorkspace({ helpExit, initExit }) {
+ const baseCwd = makeTempDir("paperclip-provision-base-");
+ const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
+ const entryPath = path.join(baseCwd, "cli", "src", "index.ts");
+ fs.mkdirSync(path.dirname(runnerPath), { recursive: true });
+ fs.mkdirSync(path.dirname(entryPath), { recursive: true });
+ fs.writeFileSync(entryPath, "// fake CLI entry\n");
+ fs.writeFileSync(
+ runnerPath,
+ `
+import fs from "node:fs";
+const cliArgs = process.argv.slice(3);
+if (cliArgs.includes("--help")) {
+ if (${helpExit} !== 0) console.error("ERR_MODULE_NOT_FOUND: drizzle-orm");
+ process.exit(${helpExit});
+}
+if (cliArgs[0] === "worktree" && cliArgs[1] === "init") {
+ if (${initExit} !== 0) {
+ console.error("fake worktree init failure");
+ process.exit(${initExit});
+ }
+ fs.mkdirSync(".paperclip", { recursive: true });
+ fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } }));
+ fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n");
+ process.exit(0);
+}
+process.exit(0);
+`,
+ );
+ return baseCwd;
+}
+
+function runProvision(baseCwd, { pathPrefix } = {}) {
+ const worktreeCwd = makeTempDir("paperclip-provision-worktree-");
+ const worktreesHome = makeTempDir("paperclip-provision-home-");
+ const result = spawnSync("bash", [script], {
+ cwd: worktreeCwd,
+ encoding: "utf8",
+ env: {
+ PATH: pathPrefix ? `${pathPrefix}:${testPath}` : testPath,
+ HOME: os.homedir(),
+ PAPERCLIP_WORKSPACE_BASE_CWD: baseCwd,
+ PAPERCLIP_WORKSPACE_CWD: worktreeCwd,
+ PAPERCLIP_WORKSPACE_BRANCH: "feature/provision-test",
+ PAPERCLIP_WORKTREES_DIR: worktreesHome,
+ PAPERCLIP_HOME: path.join(worktreesHome, "no-such-instance-home"),
+ },
+ });
+ return { result, worktreeCwd, worktreesHome };
+}
+
+function readWorktreeConfig(worktreeCwd) {
+ const configPath = path.join(worktreeCwd, ".paperclip", "config.json");
+ assert.ok(fs.existsSync(configPath), `expected ${configPath} to exist`);
+ return JSON.parse(fs.readFileSync(configPath, "utf8"));
+}
+
+test("uses the base CLI when its import graph boots", () => {
+ const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0 });
+ const { result, worktreeCwd } = runProvision(baseCwd);
+
+ assert.equal(result.status, 0, result.stderr);
+ const config = readWorktreeConfig(worktreeCwd);
+ assert.equal(config.$meta.source, "fake-cli");
+});
+
+test("falls back to an isolated config when the base CLI cannot boot", () => {
+ // Simulates the dangling pnpm symlink incident: the runner and entry files
+ // exist, but booting the CLI fails ESM resolution. The base has no
+ // package.json/pnpm-lock.yaml, so the repair install is not possible and the
+ // script must degrade to the no-CLI fallback config instead of failing.
+ const baseCwd = makeBaseWorkspace({ helpExit: 1, initExit: 0 });
+ const { result, worktreeCwd, worktreesHome } = runProvision(baseCwd);
+
+ assert.equal(result.status, 0, result.stderr);
+ assert.match(result.stderr, /writing isolated fallback config/);
+ const config = readWorktreeConfig(worktreeCwd);
+ assert.equal(config.$meta.source, "configure");
+ const dataDir = config.database.embeddedPostgresDataDir;
+ assert.ok(
+ !path.relative(worktreesHome, dataDir).startsWith(".."),
+ `expected ${dataDir} to live under ${worktreesHome}`,
+ );
+ const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8");
+ assert.match(env, /PAPERCLIP_IN_WORKTREE=true/);
+});
+
+test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => {
+ const hasTools = ["flock", "git"].every(
+ (tool) => spawnSync("bash", ["-lc", `command -v ${tool}`], { env: { PATH: testPath } }).status === 0,
+ );
+ if (!hasTools) {
+ t.skip("flock or git not available on this host");
+ return;
+ }
+
+ // The CLI's health is controlled by a flag file, and a fake `pnpm install`
+ // creates that flag — modeling a forced reinstall that relinks the store.
+ const baseCwd = makeTempDir("paperclip-provision-repair-base-");
+ const healthFlag = path.join(baseCwd, "cli-healthy.flag");
+ const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");
+ const entryPath = path.join(baseCwd, "cli", "src", "index.ts");
+ fs.mkdirSync(path.dirname(runnerPath), { recursive: true });
+ fs.mkdirSync(path.dirname(entryPath), { recursive: true });
+ fs.writeFileSync(entryPath, "// fake CLI entry\n");
+ fs.writeFileSync(
+ runnerPath,
+ `
+import fs from "node:fs";
+const cliArgs = process.argv.slice(3);
+if (cliArgs.includes("--help")) {
+ process.exit(fs.existsSync(${JSON.stringify(healthFlag)}) ? 0 : 1);
+}
+if (cliArgs[0] === "worktree" && cliArgs[1] === "init") {
+ fs.mkdirSync(".paperclip", { recursive: true });
+ fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } }));
+ fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n");
+ process.exit(0);
+}
+process.exit(0);
+`,
+ );
+ fs.writeFileSync(path.join(baseCwd, "package.json"), "{}\n");
+ fs.writeFileSync(path.join(baseCwd, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n");
+ spawnSync("git", ["init", "-q", baseCwd], { env: { PATH: testPath } });
+
+ const fakeBin = makeTempDir("paperclip-provision-fakebin-");
+ const installLog = path.join(baseCwd, "pnpm-invocations.log");
+ fs.writeFileSync(
+ path.join(fakeBin, "pnpm"),
+ `#!/usr/bin/env bash
+if [[ "$1" == "install" ]]; then
+ echo "$@" >> ${JSON.stringify(installLog)}
+ touch ${JSON.stringify(healthFlag)}
+ exit 0
+fi
+exit 1
+`,
+ { mode: 0o755 },
+ );
+
+ const { result, worktreeCwd } = runProvision(baseCwd, { pathPrefix: fakeBin });
+
+ assert.equal(result.status, 0, result.stderr);
+ const config = readWorktreeConfig(worktreeCwd);
+ assert.equal(config.$meta.source, "fake-cli");
+ const installs = fs.readFileSync(installLog, "utf8").trim().split("\n");
+ assert.equal(installs.length, 1, `expected exactly one repair install, got: ${installs.join(" | ")}`);
+ assert.match(installs[0], /--force/);
+ assert.match(installs[0], /--frozen-lockfile/);
+ assert.ok(
+ fs.existsSync(path.join(baseCwd, ".git", "paperclip-provision-repair.lock")),
+ "expected the repair lock file inside the resolved git dir",
+ );
+});
+
+test("a failed CLI init fails provisioning instead of being masked as success", () => {
+ // Regression test for the masked `return 0` after the init subshell: a CLI
+ // that passes the health check but fails `worktree init` signals a real
+ // problem, so the script must propagate the failure rather than report
+ // success or write an unseeded fallback config over it.
+ const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 3 });
+ const { result, worktreeCwd } = runProvision(baseCwd);
+
+ assert.equal(result.status, 3, result.stderr);
+ assert.match(result.stderr, /fake worktree init failure/);
+ assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "config.json")));
+});
diff --git a/scripts/provision-worktree.sh b/scripts/provision-worktree.sh
index 2435ba4aa6..fbfdda97b8 100644
--- a/scripts/provision-worktree.sh
+++ b/scripts/provision-worktree.sh
@@ -31,32 +31,89 @@ source_env_path="$(dirname "$source_config_path")/.env"
mkdir -p "$paperclip_dir"
-run_isolated_worktree_init() {
- local base_cli_runner="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"
- local base_cli_entry="$base_cwd/cli/src/index.ts"
+base_cli_runner_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"
+base_cli_entry_path="$base_cwd/cli/src/index.ts"
+
+base_cli_files_present() {
+ [[ -f "$base_cli_runner_path" && -f "$base_cli_entry_path" ]]
+}
- if [[ -f "$base_cli_runner" && -f "$base_cli_entry" ]]; then
+# File existence is not enough: pnpm links package node_modules into the
+# versioned virtual store, so a lockfile change plus a partial/filtered install
+# in the base workspace leaves dangling symlinks that fail ESM resolution at
+# runtime. Actually boot the CLI to prove its import graph resolves.
+base_cli_healthy() {
+ base_cli_files_present || return 1
+ (cd "$base_cwd" && node "$base_cli_runner_path" "$base_cli_entry_path" --help >/dev/null 2>&1)
+}
+
+repair_base_workspace_install() {
+ command -v pnpm >/dev/null 2>&1 || return 1
+ [[ -f "$base_cwd/package.json" && -f "$base_cwd/pnpm-lock.yaml" ]] || return 1
+ echo "Base workspace CLI at $base_cli_entry_path failed its health check (typically dangling pnpm symlinks after a partial install); repairing with pnpm install in $base_cwd." >&2
+ # --force guarantees relinking even when pnpm's up-to-date heuristics would
+ # otherwise skip the dangling symlinks; --frozen-lockfile keeps the repair
+ # from mutating the shared base workspace's lockfile.
+ local repair_cmd=(pnpm install --prod=false --force --frozen-lockfile --config.confirmModulesPurge=false)
+ # Resolve the real git dir so locking also covers base workspaces that are
+ # linked worktrees, where "$base_cwd/.git" is a file rather than a directory.
+ local repair_lock_dir=""
+ if command -v git >/dev/null 2>&1; then
+ repair_lock_dir="$(git -C "$base_cwd" rev-parse --absolute-git-dir 2>/dev/null || true)"
+ fi
+ if [[ ! -d "$repair_lock_dir" && -d "$base_cwd/.git" ]]; then
+ repair_lock_dir="$base_cwd/.git"
+ fi
+ if command -v flock >/dev/null 2>&1 && [[ -d "$repair_lock_dir" ]]; then
+ # The post-repair verification must run under the same lock: a concurrent
+ # provision's forced install could be mid-relink during an unlocked check
+ # and fail a repair that actually succeeded. Holding the lock also means a
+ # process that queued behind a peer's repair can skip its own reinstall.
(
- cd "$worktree_cwd"
- node "$base_cli_runner" "$base_cli_entry" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
+ cd "$base_cwd" || exit 1
+ exec 9>"$repair_lock_dir/paperclip-provision-repair.lock"
+ flock 9
+ if base_cli_healthy; then
+ echo "Base workspace CLI became healthy while waiting for the repair lock; skipping reinstall." >&2
+ exit 0
+ fi
+ env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 || exit 1
+ base_cli_healthy
)
- return 0
+ else
+ (cd "$base_cwd" && env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 && base_cli_healthy)
+ fi
+}
+
+ensure_base_cli_healthy() {
+ base_cli_files_present || return 1
+ base_cli_healthy && return 0
+ repair_base_workspace_install
+}
+
+run_isolated_worktree_init() {
+ if ensure_base_cli_healthy; then
+ (
+ cd "$worktree_cwd" &&
+ node "$base_cli_runner_path" "$base_cli_entry_path" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
+ )
+ return
fi
if command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then
(
- cd "$worktree_cwd"
- pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
+ cd "$worktree_cwd" &&
+ pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
)
- return 0
+ return
fi
if command -v paperclipai >/dev/null 2>&1; then
(
- cd "$worktree_cwd"
- paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
+ cd "$worktree_cwd" &&
+ paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"
)
- return 0
+ return
fi
return 127
@@ -67,9 +124,7 @@ paperclipai_command_available() {
return 0
fi
- local base_cli_tsx_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"
- local base_cli_entry_path="$base_cwd/cli/src/index.ts"
- if command -v node >/dev/null 2>&1 && [[ -f "$base_cli_tsx_path" ]] && [[ -f "$base_cli_entry_path" ]]; then
+ if command -v node >/dev/null 2>&1 && base_cli_files_present; then
return 0
fi
@@ -429,9 +484,24 @@ else
echo "Existing isolated Paperclip worktree config is stale for this host; regenerating." >&2
fi
if paperclipai_command_available; then
- run_isolated_worktree_init
+ if run_isolated_worktree_init; then
+ :
+ else
+ init_exit_code=$?
+ if [[ "$init_exit_code" -eq 127 ]]; then
+ # Every CLI candidate was unusable (e.g. an unhealthy base install that
+ # the repair could not fix); degrade instead of stranding the run.
+ echo "No usable paperclipai CLI found; writing isolated fallback config without DB seeding." >&2
+ write_fallback_worktree_config
+ else
+ # A CLI that ran and failed signals a real problem; do not paper over
+ # it with an unseeded fallback config.
+ echo "paperclipai worktree init failed (exit $init_exit_code); failing provisioning instead of writing an unseeded fallback config." >&2
+ exit "$init_exit_code"
+ fi
+ fi
else
- echo "paperclipai CLI not available in this workspace; writing isolated fallback config without DB seeding." >&2
+ echo "paperclipai worktree init unavailable; writing isolated fallback config without DB seeding." >&2
write_fallback_worktree_config
fi
fi