The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(scripts): self-heal isolated workspace provisioning when the base CLI is broken (#10574)
Dotta79eff0aea146
diff --git a/scripts/__tests__/provision-worktree-self-heal.test.mjs b/scripts/__tests__/provision-worktree-self-heal.test.mjsnew file mode 100644index 0000000000..9f44379501--- /dev/null+++ b/scripts/__tests__/provision-worktree-self-heal.test.mjs@@ -0,0 +1,204 @@+import assert from "node:assert/strict";+import { spawnSync } from "node:child_process";+import test from "node:test";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";++const script = new URL("../provision-worktree.sh", import.meta.url).pathname;++// Keep the PATH minimal so the fallback ladder is deterministic: node must be+// reachable, but a globally installed `paperclipai` must not shadow the paths+// under test.+const testPath = [path.dirname(process.execPath), "/usr/bin", "/bin"].join(":");++const cleanupDirs = [];++function makeTempDir(prefix) {+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));+ cleanupDirs.push(dir);+ return dir;+}++test.after(() => {+ for (const dir of cleanupDirs) {+ fs.rmSync(dir, { recursive: true, force: true });+ }+});++/**+ * Writes a fake base workspace whose "tsx runner" is a plain node script, so+ * the provision script's health check and init call can be steered per test.+ *+ * helpExit: exit code for `... index.ts --help` (the health check boot).+ * initExit: exit code for `... index.ts worktree init ...`; on 0 the fake CLI+ * writes a marker config so tests can tell CLI init from fallback.+ */+function makeBaseWorkspace({ helpExit, initExit }) {+ const baseCwd = makeTempDir("paperclip-provision-base-");+ const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");+ const entryPath = path.join(baseCwd, "cli", "src", "index.ts");+ fs.mkdirSync(path.dirname(runnerPath), { recursive: true });+ fs.mkdirSync(path.dirname(entryPath), { recursive: true });+ fs.writeFileSync(entryPath, "// fake CLI entry\n");+ fs.writeFileSync(+ runnerPath,+ `+import fs from "node:fs";+const cliArgs = process.argv.slice(3);+if (cliArgs.includes("--help")) {+ if (${helpExit} !== 0) console.error("ERR_MODULE_NOT_FOUND: drizzle-orm");+ process.exit(${helpExit});+}+if (cliArgs[0] === "worktree" && cliArgs[1] === "init") {+ if (${initExit} !== 0) {+ console.error("fake worktree init failure");+ process.exit(${initExit});+ }+ fs.mkdirSync(".paperclip", { recursive: true });+ fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } }));+ fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n");+ process.exit(0);+}+process.exit(0);+`,+ );+ return baseCwd;+}++function runProvision(baseCwd, { pathPrefix } = {}) {+ const worktreeCwd = makeTempDir("paperclip-provision-worktree-");+ const worktreesHome = makeTempDir("paperclip-provision-home-");+ const result = spawnSync("bash", [script], {+ cwd: worktreeCwd,+ encoding: "utf8",+ env: {+ PATH: pathPrefix ? `${pathPrefix}:${testPath}` : testPath,+ HOME: os.homedir(),+ PAPERCLIP_WORKSPACE_BASE_CWD: baseCwd,+ PAPERCLIP_WORKSPACE_CWD: worktreeCwd,+ PAPERCLIP_WORKSPACE_BRANCH: "feature/provision-test",+ PAPERCLIP_WORKTREES_DIR: worktreesHome,+ PAPERCLIP_HOME: path.join(worktreesHome, "no-such-instance-home"),+ },+ });+ return { result, worktreeCwd, worktreesHome };+}++function readWorktreeConfig(worktreeCwd) {+ const configPath = path.join(worktreeCwd, ".paperclip", "config.json");+ assert.ok(fs.existsSync(configPath), `expected ${configPath} to exist`);+ return JSON.parse(fs.readFileSync(configPath, "utf8"));+}++test("uses the base CLI when its import graph boots", () => {+ const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 0 });+ const { result, worktreeCwd } = runProvision(baseCwd);++ assert.equal(result.status, 0, result.stderr);+ const config = readWorktreeConfig(worktreeCwd);+ assert.equal(config.$meta.source, "fake-cli");+});++test("falls back to an isolated config when the base CLI cannot boot", () => {+ // Simulates the dangling pnpm symlink incident: the runner and entry files+ // exist, but booting the CLI fails ESM resolution. The base has no+ // package.json/pnpm-lock.yaml, so the repair install is not possible and the+ // script must degrade to the no-CLI fallback config instead of failing.+ const baseCwd = makeBaseWorkspace({ helpExit: 1, initExit: 0 });+ const { result, worktreeCwd, worktreesHome } = runProvision(baseCwd);++ assert.equal(result.status, 0, result.stderr);+ assert.match(result.stderr, /writing isolated fallback config/);+ const config = readWorktreeConfig(worktreeCwd);+ assert.equal(config.$meta.source, "configure");+ const dataDir = config.database.embeddedPostgresDataDir;+ assert.ok(+ !path.relative(worktreesHome, dataDir).startsWith(".."),+ `expected ${dataDir} to live under ${worktreesHome}`,+ );+ const env = fs.readFileSync(path.join(worktreeCwd, ".paperclip", ".env"), "utf8");+ assert.match(env, /PAPERCLIP_IN_WORKTREE=true/);+});++test("repairs an unhealthy base install under the lock and then uses the CLI", (t) => {+ const hasTools = ["flock", "git"].every(+ (tool) => spawnSync("bash", ["-lc", `command -v ${tool}`], { env: { PATH: testPath } }).status === 0,+ );+ if (!hasTools) {+ t.skip("flock or git not available on this host");+ return;+ }++ // The CLI's health is controlled by a flag file, and a fake `pnpm install`+ // creates that flag — modeling a forced reinstall that relinks the store.+ const baseCwd = makeTempDir("paperclip-provision-repair-base-");+ const healthFlag = path.join(baseCwd, "cli-healthy.flag");+ const runnerPath = path.join(baseCwd, "cli", "node_modules", "tsx", "dist", "cli.mjs");+ const entryPath = path.join(baseCwd, "cli", "src", "index.ts");+ fs.mkdirSync(path.dirname(runnerPath), { recursive: true });+ fs.mkdirSync(path.dirname(entryPath), { recursive: true });+ fs.writeFileSync(entryPath, "// fake CLI entry\n");+ fs.writeFileSync(+ runnerPath,+ `+import fs from "node:fs";+const cliArgs = process.argv.slice(3);+if (cliArgs.includes("--help")) {+ process.exit(fs.existsSync(${JSON.stringify(healthFlag)}) ? 0 : 1);+}+if (cliArgs[0] === "worktree" && cliArgs[1] === "init") {+ fs.mkdirSync(".paperclip", { recursive: true });+ fs.writeFileSync(".paperclip/config.json", JSON.stringify({ $meta: { source: "fake-cli" } }));+ fs.writeFileSync(".paperclip/.env", "PAPERCLIP_IN_WORKTREE=true\\n");+ process.exit(0);+}+process.exit(0);+`,+ );+ fs.writeFileSync(path.join(baseCwd, "package.json"), "{}\n");+ fs.writeFileSync(path.join(baseCwd, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n");+ spawnSync("git", ["init", "-q", baseCwd], { env: { PATH: testPath } });++ const fakeBin = makeTempDir("paperclip-provision-fakebin-");+ const installLog = path.join(baseCwd, "pnpm-invocations.log");+ fs.writeFileSync(+ path.join(fakeBin, "pnpm"),+ `#!/usr/bin/env bash+if [[ "$1" == "install" ]]; then+ echo "$@" >> ${JSON.stringify(installLog)}+ touch ${JSON.stringify(healthFlag)}+ exit 0+fi+exit 1+`,+ { mode: 0o755 },+ );++ const { result, worktreeCwd } = runProvision(baseCwd, { pathPrefix: fakeBin });++ assert.equal(result.status, 0, result.stderr);+ const config = readWorktreeConfig(worktreeCwd);+ assert.equal(config.$meta.source, "fake-cli");+ const installs = fs.readFileSync(installLog, "utf8").trim().split("\n");+ assert.equal(installs.length, 1, `expected exactly one repair install, got: ${installs.join(" | ")}`);+ assert.match(installs[0], /--force/);+ assert.match(installs[0], /--frozen-lockfile/);+ assert.ok(+ fs.existsSync(path.join(baseCwd, ".git", "paperclip-provision-repair.lock")),+ "expected the repair lock file inside the resolved git dir",+ );+});++test("a failed CLI init fails provisioning instead of being masked as success", () => {+ // Regression test for the masked `return 0` after the init subshell: a CLI+ // that passes the health check but fails `worktree init` signals a real+ // problem, so the script must propagate the failure rather than report+ // success or write an unseeded fallback config over it.+ const baseCwd = makeBaseWorkspace({ helpExit: 0, initExit: 3 });+ const { result, worktreeCwd } = runProvision(baseCwd);++ assert.equal(result.status, 3, result.stderr);+ assert.match(result.stderr, /fake worktree init failure/);+ assert.ok(!fs.existsSync(path.join(worktreeCwd, ".paperclip", "config.json")));+});diff --git a/scripts/provision-worktree.sh b/scripts/provision-worktree.shindex 2435ba4aa6..fbfdda97b8 100644--- a/scripts/provision-worktree.sh+++ b/scripts/provision-worktree.sh@@ -31,32 +31,89 @@ source_env_path="$(dirname "$source_config_path")/.env"mkdir -p "$paperclip_dir"-run_isolated_worktree_init() {- local base_cli_runner="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"- local base_cli_entry="$base_cwd/cli/src/index.ts"+base_cli_runner_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"+base_cli_entry_path="$base_cwd/cli/src/index.ts"++base_cli_files_present() {+ [[ -f "$base_cli_runner_path" && -f "$base_cli_entry_path" ]]+}- if [[ -f "$base_cli_runner" && -f "$base_cli_entry" ]]; then+# File existence is not enough: pnpm links package node_modules into the+# versioned virtual store, so a lockfile change plus a partial/filtered install+# in the base workspace leaves dangling symlinks that fail ESM resolution at+# runtime. Actually boot the CLI to prove its import graph resolves.+base_cli_healthy() {+ base_cli_files_present || return 1+ (cd "$base_cwd" && node "$base_cli_runner_path" "$base_cli_entry_path" --help >/dev/null 2>&1)+}++repair_base_workspace_install() {+ command -v pnpm >/dev/null 2>&1 || return 1+ [[ -f "$base_cwd/package.json" && -f "$base_cwd/pnpm-lock.yaml" ]] || return 1+ echo "Base workspace CLI at $base_cli_entry_path failed its health check (typically dangling pnpm symlinks after a partial install); repairing with pnpm install in $base_cwd." >&2+ # --force guarantees relinking even when pnpm's up-to-date heuristics would+ # otherwise skip the dangling symlinks; --frozen-lockfile keeps the repair+ # from mutating the shared base workspace's lockfile.+ local repair_cmd=(pnpm install --prod=false --force --frozen-lockfile --config.confirmModulesPurge=false)+ # Resolve the real git dir so locking also covers base workspaces that are+ # linked worktrees, where "$base_cwd/.git" is a file rather than a directory.+ local repair_lock_dir=""+ if command -v git >/dev/null 2>&1; then+ repair_lock_dir="$(git -C "$base_cwd" rev-parse --absolute-git-dir 2>/dev/null || true)"+ fi+ if [[ ! -d "$repair_lock_dir" && -d "$base_cwd/.git" ]]; then+ repair_lock_dir="$base_cwd/.git"+ fi+ if command -v flock >/dev/null 2>&1 && [[ -d "$repair_lock_dir" ]]; then+ # The post-repair verification must run under the same lock: a concurrent+ # provision's forced install could be mid-relink during an unlocked check+ # and fail a repair that actually succeeded. Holding the lock also means a+ # process that queued behind a peer's repair can skip its own reinstall.(- cd "$worktree_cwd"- node "$base_cli_runner" "$base_cli_entry" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"+ cd "$base_cwd" || exit 1+ exec 9>"$repair_lock_dir/paperclip-provision-repair.lock"+ flock 9+ if base_cli_healthy; then+ echo "Base workspace CLI became healthy while waiting for the repair lock; skipping reinstall." >&2+ exit 0+ fi+ env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 || exit 1+ base_cli_healthy)- return 0+ else+ (cd "$base_cwd" && env -u NODE_ENV CI=true "${repair_cmd[@]}" >&2 && base_cli_healthy)+ fi+}++ensure_base_cli_healthy() {+ base_cli_files_present || return 1+ base_cli_healthy && return 0+ repair_base_workspace_install+}++run_isolated_worktree_init() {+ if ensure_base_cli_healthy; then+ (+ cd "$worktree_cwd" &&+ node "$base_cli_runner_path" "$base_cli_entry_path" worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"+ )+ returnfiif command -v pnpm >/dev/null 2>&1 && pnpm paperclipai --help >/dev/null 2>&1; then(- cd "$worktree_cwd"- pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"+ cd "$worktree_cwd" &&+ pnpm paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path")- return 0+ returnfiif command -v paperclipai >/dev/null 2>&1; then(- cd "$worktree_cwd"- paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path"+ cd "$worktree_cwd" &&+ paperclipai worktree init --force --seed-mode minimal --name "$worktree_name" --from-config "$source_config_path")- return 0+ returnfireturn 127@@ -67,9 +124,7 @@ paperclipai_command_available() {return 0fi- local base_cli_tsx_path="$base_cwd/cli/node_modules/tsx/dist/cli.mjs"- local base_cli_entry_path="$base_cwd/cli/src/index.ts"- if command -v node >/dev/null 2>&1 && [[ -f "$base_cli_tsx_path" ]] && [[ -f "$base_cli_entry_path" ]]; then+ if command -v node >/dev/null 2>&1 && base_cli_files_present; thenreturn 0fi@@ -429,9 +484,24 @@ elseecho "Existing isolated Paperclip worktree config is stale for this host; regenerating." >&2fiif paperclipai_command_available; then- run_isolated_worktree_init+ if run_isolated_worktree_init; then+ :+ else+ init_exit_code=$?+ if [[ "$init_exit_code" -eq 127 ]]; then+ # Every CLI candidate was unusable (e.g. an unhealthy base install that+ # the repair could not fix); degrade instead of stranding the run.+ echo "No usable paperclipai CLI found; writing isolated fallback config without DB seeding." >&2+ write_fallback_worktree_config+ else+ # A CLI that ran and failed signals a real problem; do not paper over+ # it with an unseeded fallback config.+ echo "paperclipai worktree init failed (exit $init_exit_code); failing provisioning instead of writing an unseeded fallback config." >&2+ exit "$init_exit_code"+ fi+ fielse- echo "paperclipai CLI not available in this workspace; writing isolated fallback config without DB seeding." >&2+ echo "paperclipai worktree init unavailable; writing isolated fallback config without DB seeding." >&2write_fallback_worktree_configfifi