The open-source app everyone uses to manage agents at work
TypeScript acpxagent-orchestrationagent-skillsagentic-aiai-agentsai-engineeringai-governanceai-toolsautonomous-agentsclaudeclaude-codeclaude-skillscodexhermesllmmcpmulti-agentopenclawproject-managementworkflow-automation
fix(cli): recover abandoned workspace build locks (#13288)
Dotta30c63af0e619
diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.mdindex 8927e66d41..3983005523 100644--- a/doc/DEVELOPING.md+++ b/doc/DEVELOPING.md@@ -417,6 +417,19 @@ configs with `database.mode: postgres`, suppresses the invocation directory'sguard. The selected instance's own environment file still loads. The commandselects the first available loopback port at or above `3100`.+Source-checkout startup builds the shared and plugin SDK packages when needed.+It prints build progress and any wait for another build. Interrupted builds+release their lock after the compiler stops; later startups recover locks whose+owner and compiler have exited. Empty locks from older versions are recovered+once they are at least two minutes old. The command remains in the foreground+after printing its ready URL to serve the instance; use Ctrl-C to stop it.+Each package gets a completion marker only after a successful build. A hard+kill leaves that marker absent, so the next startup rebuilds partial output.+The marker records source and output content fingerprints, so recovery does+not depend on filesystem timestamp precision. Direct `tsc` builds that produce+identical output reuse the marker. Changed or partial output is rebuilt once+before later startups reuse the completed build.+Claude uses `ANTHROPIC_API_KEY`; Codex uses `OPENAI_API_KEY`; OpenCode uses`OPENROUTER_API_KEY` and requires an `openrouter/...` model. `--api-key-env`can name a different source variable while the agent still receives thediff --git a/scripts/__tests__/ensure-plugin-build-deps.test.mjs b/scripts/__tests__/ensure-plugin-build-deps.test.mjsnew file mode 100644index 0000000000..99778f75f3--- /dev/null+++ b/scripts/__tests__/ensure-plugin-build-deps.test.mjs@@ -0,0 +1,238 @@+import assert from "node:assert/strict";+import { spawn, spawnSync } from "node:child_process";+import { once } from "node:events";+import fs from "node:fs";+import os from "node:os";+import path from "node:path";+import { setTimeout as sleep } from "node:timers/promises";+import test from "node:test";++function fixture(t) {+ const root = fs.mkdtempSync(path.join(os.tmpdir(), "paperclip-build-lock-"));+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));+ fs.mkdirSync(path.join(root, "scripts"));+ fs.copyFileSync(new URL("../ensure-plugin-build-deps.mjs", import.meta.url), path.join(root, "scripts/ensure-plugin-build-deps.mjs"));+ const compiler = path.join(root, "node_modules/typescript/bin/tsc");+ fs.mkdirSync(path.dirname(compiler), { recursive: true });+ fs.writeFileSync(compiler, `+const fs = require("node:fs");+const path = require("node:path");+const target = path.dirname(process.argv[3]);+const active = path.resolve("compiler-active");+try { fs.mkdirSync(active); } catch { process.exit(42); }+process.on("exit", () => fs.rmSync(active, { recursive: true, force: true }));+process.on("SIGTERM", () => process.exit(143));+process.on("SIGINT", () => process.exit(130));+fs.appendFileSync("builds", target + "\\n");+fs.mkdirSync(path.join(target, "dist"), { recursive: true });+// Deliberately write index.js before the compiler finishes emitting the rest.+fs.writeFileSync(path.join(target, "dist/index.js"), "export {};\\n");+setTimeout(() => {+ if (fs.existsSync("fail")) process.exit(2);+ fs.writeFileSync(path.join(target, "dist/complete"), "done");+}, Number(process.env.BUILD_DELAY ?? 20));+`);+ for (const target of ["packages/shared", "packages/plugins/sdk"]) {+ fs.mkdirSync(path.join(root, target, "src"), { recursive: true });+ fs.writeFileSync(path.join(root, target, "src/index.ts"), "export {};\n");+ fs.writeFileSync(path.join(root, target, "tsconfig.json"), "{}");+ }+ const lock = path.join(root, "node_modules/.cache/paperclip-plugin-build-deps.lock");+ const launch = (env = {}) => {+ const child = spawn(process.execPath, ["scripts/ensure-plugin-build-deps.mjs"], {+ cwd: root, env: { ...process.env, ...env }, stdio: ["ignore", "pipe", "pipe"],+ });+ let output = "";+ child.stdout.on("data", (data) => { output += data; });+ child.stderr.on("data", (data) => { output += data; });+ const done = once(child, "close").then(([code]) => ({ code, output }));+ t.after(() => { if (child.exitCode === null) child.kill("SIGTERM"); });+ return { child, done };+ };+ return { root, lock, launch };+}++async function until(predicate) {+ const deadline = Date.now() + 5000;+ while (!predicate()) {+ assert.ok(Date.now() < deadline, "condition timed out");+ await sleep(10);+ }+}++test("recovers the old empty lock left by interrupted startup", async (t) => {+ const f = fixture(t);+ fs.mkdirSync(f.lock, { recursive: true });+ const old = new Date(Date.now() - 180_000);+ fs.utimesSync(f.lock, old, old);+ const result = await f.launch().done;+ assert.equal(result.code, 0, result.output);+ assert.match(result.output, /Recovered abandoned/);+ assert.match(result.output, /Building @paperclipai\/shared/);+ assert.equal(fs.existsSync(f.lock), false);+});++test("concurrent startups recover a dead owner and build only once", async (t) => {+ const f = fixture(t);+ const dead = spawnSync(process.execPath, ["-e", "" ]).pid;+ fs.mkdirSync(f.lock, { recursive: true });+ fs.writeFileSync(path.join(f.lock, `owner-${dead}-old.json`), JSON.stringify({ pid: dead }));+ const results = await Promise.all(Array.from({ length: 4 }, () => f.launch({ BUILD_DELAY: "150" }).done));+ for (const result of results) assert.equal(result.code, 0, result.output);+ assert.equal(fs.readFileSync(path.join(f.root, "builds"), "utf8").trim().split("\n").length, 2);+});++test("does not accept partially emitted output while another compiler holds the lock", async (t) => {+ const f = fixture(t);+ const first = f.launch({ BUILD_DELAY: "200" });+ await until(() => fs.existsSync(path.join(f.root, "packages/plugins/sdk/dist/index.js")));+ const second = await f.launch().done;+ assert.equal(second.code, 0, second.output);+ assert.match(second.output, /Waiting for another workspace build/);+ assert.equal(fs.existsSync(path.join(f.root, "packages/plugins/sdk/dist/complete")), true);+ assert.equal((await first.done).code, 0);+});++test("preserves a live compiler's lock even when its parent has exited", async (t) => {+ const f = fixture(t);+ const dead = spawnSync(process.execPath, ["-e", ""]).pid;+ fs.mkdirSync(f.lock, { recursive: true });+ fs.writeFileSync(path.join(f.lock, `owner-${dead}-old.json`), JSON.stringify({ pid: dead, childPid: process.pid }));+ const run = f.launch();+ await sleep(200);+ assert.equal(fs.existsSync(path.join(f.root, "builds")), false);+ run.child.kill("SIGTERM");+ assert.equal((await run.done).code, 143);+ assert.equal(fs.existsSync(f.lock), true);+});++test("termination stops the compiler and releases the lock for the next startup", async (t) => {+ const f = fixture(t);+ const run = f.launch({ BUILD_DELAY: "10000" });+ await until(() => fs.existsSync(path.join(f.root, "compiler-active")));+ run.child.kill("SIGTERM");+ assert.equal((await run.done).code, 143);+ assert.equal(fs.existsSync(f.lock), false);+ assert.equal(fs.existsSync(path.join(f.root, "compiler-active")), false);+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.equal(fs.existsSync(path.join(f.root, "packages/shared/dist/complete")), true);+});++test("failed compilation releases the lock and is rebuilt on retry", async (t) => {+ const f = fixture(t);+ fs.writeFileSync(path.join(f.root, "fail"), "");+ assert.equal((await f.launch().done).code, 2);+ assert.equal(fs.existsSync(f.lock), false);+ fs.unlinkSync(path.join(f.root, "fail"));+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.equal(fs.existsSync(path.join(f.root, "packages/shared/dist/complete")), true);+});++test("hard-killed compilation cannot leave partial output accepted on recovery", async (t) => {+ const f = fixture(t);+ // First establish valid completion markers, then start a rebuild.+ assert.equal((await f.launch().done).code, 0);+ const output = path.join(f.root, "packages/shared/dist/index.js");+ const complete = path.join(f.root, "packages/shared/dist/complete");+ fs.unlinkSync(output);+ fs.unlinkSync(complete);+ const run = f.launch({ BUILD_DELAY: "10000" });+ await until(() => fs.existsSync(output));+ const owner = JSON.parse(fs.readFileSync(path.join(f.lock, fs.readdirSync(f.lock)[0]), "utf8"));+ run.child.kill("SIGKILL");+ process.kill(owner.childPid, "SIGKILL");+ await run.done;+ await until(() => {+ try { process.kill(owner.childPid, 0); return false; }+ catch (error) { return error.code === "ESRCH"; }+ });+ // SIGKILL cannot run the fixture compiler's exit hook either.+ fs.rmSync(path.join(f.root, "compiler-active"), { recursive: true, force: true });+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.match(retry.output, /Recovered abandoned/);+ assert.equal(fs.existsSync(complete), true);+});++test("rebuilds changed direct output once, then reuses the completed build", async (t) => {+ const f = fixture(t);+ assert.equal((await f.launch().done).code, 0);+ const builds = fs.readFileSync(path.join(f.root, "builds"), "utf8");+ // A successful or interrupted direct tsc invocation updates index.js without+ // changing our marker. Neither can certify that all output was emitted.+ for (const target of ["packages/shared", "packages/plugins/sdk"]) {+ fs.appendFileSync(path.join(f.root, target, "dist/index.js"), "// direct build changed output\n");+ }+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ const rebuilt = fs.readFileSync(path.join(f.root, "builds"), "utf8");+ assert.equal(rebuilt.trim().split("\n").length, builds.trim().split("\n").length + 2);+ const next = await f.launch().done;+ assert.equal(next.code, 0, next.output);+ assert.equal(fs.readFileSync(path.join(f.root, "builds"), "utf8"), rebuilt);+ assert.doesNotMatch(next.output, /Building/);+});++test("rejects partial output from an interrupted direct compiler", async (t) => {+ const f = fixture(t);+ assert.equal((await f.launch().done).code, 0);+ const target = path.join(f.root, "packages/shared");+ fs.unlinkSync(path.join(target, "dist/complete"));+ const completionTime = fs.statSync(path.join(target, "dist/.paperclip-build-complete")).mtimeMs;+ await sleep(20);+ const compiler = spawn(process.execPath, ["node_modules/typescript/bin/tsc", "-p", path.join(target, "tsconfig.json")], {+ cwd: f.root, env: { ...process.env, BUILD_DELAY: "10000" }, stdio: "ignore",+ });+ const closed = once(compiler, "close");+ t.after(() => { if (compiler.exitCode === null) compiler.kill("SIGKILL"); });+ await until(() => fs.statSync(path.join(target, "dist/index.js")).mtimeMs > completionTime);+ compiler.kill("SIGKILL");+ await closed;+ fs.rmSync(path.join(f.root, "compiler-active"), { recursive: true, force: true });+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.equal(fs.existsSync(path.join(target, "dist/complete")), true);+});++test("detects partial output even when all modification times are unchanged", async (t) => {+ const f = fixture(t);+ assert.equal((await f.launch().done).code, 0);+ const target = path.join(f.root, "packages/shared");+ const output = path.join(target, "dist/index.js");+ const oldTime = fs.statSync(output).mtime;+ fs.writeFileSync(output, "// incomplete direct build\n");+ fs.utimesSync(output, oldTime, oldTime);+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.equal(fs.readFileSync(output, "utf8"), "export {};\n");+});++test("reuses identical direct output regardless of its timestamps", async (t) => {+ const f = fixture(t);+ assert.equal((await f.launch().done).code, 0);+ const builds = fs.readFileSync(path.join(f.root, "builds"), "utf8");+ const output = path.join(f.root, "packages/shared/dist/index.js");+ fs.writeFileSync(output, fs.readFileSync(output));+ const newer = new Date(Date.now() + 1000);+ fs.utimesSync(output, newer, newer);+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.equal(fs.readFileSync(path.join(f.root, "builds"), "utf8"), builds);+ assert.doesNotMatch(retry.output, /Building/);+});++test("shared source changes invalidate both shared and dependent SDK output", async (t) => {+ const f = fixture(t);+ assert.equal((await f.launch().done).code, 0);+ const source = path.join(f.root, "packages/shared/src/index.ts");+ const oldTime = fs.statSync(source).mtime;+ fs.appendFileSync(source, "export const changed = true;\n");+ fs.utimesSync(source, oldTime, oldTime);+ const retry = await f.launch().done;+ assert.equal(retry.code, 0, retry.output);+ assert.match(retry.output, /Building @paperclipai\/shared/);+ assert.match(retry.output, /Building @paperclipai\/plugin-sdk/);+ assert.equal(fs.readFileSync(path.join(f.root, "builds"), "utf8").trim().split("\n").length, 4);+});diff --git a/scripts/ensure-plugin-build-deps.mjs b/scripts/ensure-plugin-build-deps.mjsindex 5b19024dd9..356e0e0f80 100644--- a/scripts/ensure-plugin-build-deps.mjs+++ b/scripts/ensure-plugin-build-deps.mjs@@ -1,9 +1,11 @@#!/usr/bin/env node-import { spawnSync } from "node:child_process";+import { spawn } from "node:child_process";import fs from "node:fs";import path from "node:path";import { fileURLToPath } from "node:url";+import { createHash, randomUUID } from "node:crypto";+import { setTimeout as sleep } from "node:timers/promises";const scriptDir = path.dirname(fileURLToPath(import.meta.url));const rootDir = path.resolve(scriptDir, "..");@@ -16,14 +18,18 @@ const buildTargets = [{name: "@paperclipai/shared",output: path.join(rootDir, "packages/shared/dist/index.js"),+ completion: path.join(rootDir, "packages/shared/dist/.paperclip-build-complete"),sourceDir: path.join(rootDir, "packages/shared/src"),tsconfig: path.join(rootDir, "packages/shared/tsconfig.json"),+ dependencies: [],},{name: "@paperclipai/plugin-sdk",output: path.join(rootDir, "packages/plugins/sdk/dist/index.js"),+ completion: path.join(rootDir, "packages/plugins/sdk/dist/.paperclip-build-complete"),sourceDir: path.join(rootDir, "packages/plugins/sdk/src"),tsconfig: path.join(rootDir, "packages/plugins/sdk/tsconfig.json"),+ dependencies: [0],},];@@ -31,102 +37,212 @@ if (!fs.existsSync(tscCliPath)) {throw new Error(`TypeScript CLI not found at ${tscCliPath}`);}-function newestSourceMtimeMs(sourceDir) {- let newest = 0;-+function directoryFingerprint(directory, exclude) {+ const hash = createHash("sha256");function visit(dir) {- for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {+ for (const entry of fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {const entryPath = path.join(dir, entry.name);+ if (entryPath === exclude) continue;if (entry.isDirectory()) {visit(entryPath);- continue;+ } else if (entry.isFile()) {+ const content = fs.readFileSync(entryPath);+ hash.update(JSON.stringify([path.relative(directory, entryPath), content.length]));+ hash.update(content);}- if (!/\.(tsx?|json)$/.test(entry.name)) continue;- newest = Math.max(newest, fs.statSync(entryPath).mtimeMs);}}+ visit(directory);+ return hash.digest("hex");+}++function sourceFingerprint(target) {+ const hash = createHash("sha256");+ hash.update(directoryFingerprint(target.sourceDir));+ for (const config of [+ target.tsconfig,+ path.join(path.dirname(target.tsconfig), "package.json"),+ path.join(rootDir, "tsconfig.json"),+ path.join(rootDir, "tsconfig.base.json"),+ path.join(rootDir, "node_modules/typescript/package.json"),+ ]) {+ if (fs.existsSync(config)) hash.update(fs.readFileSync(config));+ }+ for (const dependency of target.dependencies) hash.update(sourceFingerprint(buildTargets[dependency]));+ return hash.digest("hex");+}- visit(sourceDir);- return newest;+function outputFingerprint(target) {+ return directoryFingerprint(path.dirname(target.output), target.completion);}function needsBuild(target) {if (!fs.existsSync(target.output)) return true;- const outputMtime = fs.statSync(target.output).mtimeMs;- return newestSourceMtimeMs(target.sourceDir) > outputMtime;+ try {+ const completed = JSON.parse(fs.readFileSync(target.completion, "utf8"));+ // Content fingerprints detect partial direct builds even on filesystems+ // with coarse timestamps, while identical successful direct builds reuse+ // the certified output without another compile.+ return completed.sources !== sourceFingerprint(target)+ || completed.outputs !== outputFingerprint(target);+ } catch (error) {+ if (error.code === "ENOENT" || error instanceof SyntaxError) return true;+ throw error;+ }}function allOutputsCurrent() {return buildTargets.every((target) => !needsBuild(target));}-function sleep(ms) {- Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);-}+// Publish an already-populated directory so another contender never mistakes a+// newly acquired lock for an abandoned, ownerless lock. Never recursively remove+// the shared path: another process may have acquired it since we last read it.+const ownerFile = `owner-${process.pid}-${randomUUID()}.json`;+let child = null;+let stoppingSignal = null;+let holdsLock = false;-function waitForLockRelease() {- const startedAt = Date.now();- while (Date.now() - startedAt < lockTimeoutMs) {- if (!fs.existsSync(lockDir)) {- return;- }- if (allOutputsCurrent()) {- return;- }- sleep(lockPollMs);+function processAlive(pid) {+ if (!Number.isInteger(pid) || pid <= 0) return true;+ try {+ process.kill(pid, 0);+ return true;+ } catch (error) {+ return error.code !== "ESRCH";}-- throw new Error(`Timed out waiting for plugin build dependency lock at ${lockDir}`);}-if (allOutputsCurrent()) {- process.exit(0);+function removeOwner(file) {+ try {+ fs.unlinkSync(path.join(lockDir, file));+ } catch (error) {+ if (error.code === "ENOENT") return;+ throw error;+ }+ try {+ fs.rmdirSync(lockDir);+ } catch (error) {+ if (!["ENOENT", "ENOTEMPTY", "EEXIST"].includes(error.code)) throw error;+ }}-fs.mkdirSync(path.dirname(lockDir), { recursive: true });+function releaseLock() {+ if (!holdsLock) return;+ removeOwner(ownerFile);+ holdsLock = false;+}-let holdsLock = false;-let exitCode = 0;-try {+function recoverAbandonedLock() {try {- fs.mkdirSync(lockDir);- holdsLock = true;- } catch (error) {- if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") {- waitForLockRelease();- if (!allOutputsCurrent()) {- throw new Error("Plugin build dependency lock released before all outputs were created");- }- process.exit(0);+ const entries = fs.readdirSync(lockDir);+ if (entries.length === 0) {+ // Older versions wrote no owner. Allow their bounded CLI build to finish+ // before reclaiming an empty directory left by interruption or timeout.+ if (Date.now() - fs.statSync(lockDir).mtimeMs < 120_000) return;+ fs.rmdirSync(lockDir);+ } else if (entries.length === 1 && /^owner-.*\.json$/.test(entries[0])) {+ const owner = JSON.parse(fs.readFileSync(path.join(lockDir, entries[0]), "utf8"));+ if (processAlive(owner.pid) || (owner.childPid && processAlive(owner.childPid))) return;+ removeOwner(entries[0]);+ } else {+ return;}+ console.log("[paperclip] Recovered abandoned workspace build lock.");+ } catch (error) {+ if (["ENOENT", "ENOTEMPTY", "EEXIST"].includes(error.code) || error instanceof SyntaxError) return;throw error;}+}- for (const target of buildTargets) {- if (!needsBuild(target)) {- continue;+async function acquireLock() {+ fs.mkdirSync(path.dirname(lockDir), { recursive: true });+ const candidate = fs.mkdtempSync(`${lockDir}.candidate-`);+ fs.writeFileSync(path.join(candidate, ownerFile), JSON.stringify({ pid: process.pid }));+ const startedAt = Date.now();+ let reportedWait = false;+ try {+ while (!stoppingSignal) {+ // Do not replace a fresh empty lock held by an older script.+ recoverAbandonedLock();+ if (!fs.existsSync(lockDir)) {+ try {+ fs.renameSync(candidate, lockDir);+ holdsLock = true;+ return;+ } catch (error) {+ if (!["ENOTEMPTY", "EEXIST", "EPERM"].includes(error.code)) throw error;+ }+ }+ if (!reportedWait) {+ console.log(`[paperclip] Waiting for another workspace build (${lockDir})...`);+ reportedWait = true;+ }+ if (Date.now() - startedAt >= lockTimeoutMs) {+ throw new Error(`Timed out waiting for workspace build lock at ${lockDir}. Another build may still be running.`);+ }+ await sleep(lockPollMs);}+ } finally {+ fs.rmSync(candidate, { recursive: true, force: true });+ }+}- const result = spawnSync(process.execPath, [tscCliPath, "-p", target.tsconfig], {+async function build(target) {+ console.log(`[paperclip] Building ${target.name}...`);+ // A hard kill bypasses cleanup. Only a completed compile may restore this+ // marker, so recovery never trusts index.js emitted partway through a build.+ fs.rmSync(target.completion, { force: true });+ const sources = sourceFingerprint(target);+ const code = await new Promise((resolve, reject) => {+ child = spawn(process.execPath, [tscCliPath, "-p", target.tsconfig], {cwd: rootDir,stdio: "inherit",});+ // A hard-killed parent must not let a successor race its surviving compiler.+ fs.writeFileSync(path.join(lockDir, ownerFile), JSON.stringify({ pid: process.pid, childPid: child.pid }));+ child.once("error", (error) => {+ fs.rmSync(target.output, { force: true });+ reject(error);+ });+ child.once("close", (code) => {+ child = null;+ resolve(code ?? 1);+ });+ });+ // tsc emits index.js before it finishes the package. A failed or interrupted+ // compile must not make the next startup accept that partial build as current.+ if (code !== 0) fs.rmSync(target.output, { force: true });+ else fs.writeFileSync(target.completion, JSON.stringify({ sources, outputs: outputFingerprint(target) }) + "\n");+ return code;+}- if (result.error) {- throw result.error;- }+if (allOutputsCurrent() && !fs.existsSync(lockDir)) {+ process.exit(0);+}++// Keep the lock until the compiler has stopped, including when the foreground+// CLI's build timeout terminates this helper.+for (const signal of ["SIGINT", "SIGTERM"]) {+ process.on(signal, () => {+ stoppingSignal = signal;+ child?.kill(signal);+ });+}+process.once("exit", releaseLock);- if (result.status !== 0) {- exitCode = result.status ?? 1;- break;+let exitCode = 0;+try {+ await acquireLock();+ if (holdsLock) {+ for (const target of buildTargets) {+ if (stoppingSignal) break;+ if (!needsBuild(target)) continue;+ exitCode = await build(target);+ if (exitCode !== 0) break;}}} finally {- if (holdsLock) {- fs.rmSync(lockDir, { recursive: true, force: true });- }-}--if (exitCode !== 0) {- process.exit(exitCode);+ releaseLock();}+process.exitCode = stoppingSignal === "SIGINT" ? 130 : stoppingSignal ? 143 : exitCode;