Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(policy): reclaim state.lock when its owner process is dead (#3362)

Mr. Makcf2177bb4b7a
diff --git a/v3/@claude-flow/cli/__tests__/policy-lock-dead-owner.test.ts b/v3/@claude-flow/cli/__tests__/policy-lock-dead-owner.test.ts
new file mode 100644
index 0000000000..2266e01a55
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/policy-lock-dead-owner.test.ts
@@ -0,0 +1,327 @@
+/**
+ * Regression guard: the ADR-324 policy lock (`.claude-flow/policy/state.lock`)
+ * must not outlive the process that owns it, and must never be taken from a
+ * process that is still alive.
+ *
+ * `acquireLock()` recorded `{ pid, acquiredAt }` but only ever reclaimed a
+ * lock by mtime age (LOCK_STALE_MS = 30 s). Every MCP tool call — reads such
+ * as `memory_retrieve` included — passes through `authorizeMcpTool()` →
+ * `withPolicyTransaction()` → `acquireLock()`. When a server is killed inside
+ * a transaction (SIGKILL, OOM, or a default-action SIGTERM) its
+ * `finally { release() }` never runs, so each call on a fresh server waited
+ * LOCK_WAIT_MS (5 s) and failed with `policy-state-lock-timeout` until the
+ * orphan's mtime aged past 30 s.
+ *
+ * The fix mirrors #1799 (`isPidAlive()` in swarm-tools.ts): a lock whose owner
+ * has exited is reclaimed at once. It trusts the pid only when the lock was
+ * written by this OS, on this kernel boot and in this PID namespace, treats
+ * only ESRCH as dead, reads only small regular files (no symlinks, FIFOs or
+ * devices), and answers only for the lock it read, by re-reading its bytes
+ * (not a lock a new owner created meanwhile). Every other lock keeps the old
+ * mtime rule.
+ */
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { execFileSync, spawn } from 'node:child_process';
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ rmSync,
+ symlinkSync,
+ unlinkSync,
+ utimesSync,
+ writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import {
+ authorizeMcpTool,
+ autoMigratePolicyStateIfNeeded,
+ classifyMcpTool,
+ withPolicyTransaction,
+} from '../src/services/policy-runtime.js';
+
+// Mirror LOCK_WAIT_MS / LOCK_STALE_MS / LOCK_MAX_BYTES in policy-runtime.ts.
+const LOCK_WAIT_MS = 5_000;
+const LOCK_STALE_MS = 30_000;
+const LOCK_MAX_BYTES = 1_024;
+
+type LockOwner = Record<string, unknown>;
+
+const roots: string[] = [];
+const children: Array<ReturnType<typeof spawn>> = [];
+
+afterEach(() => {
+ for (const child of children.splice(0)) child.kill('SIGKILL');
+ for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true });
+});
+
+function sleep(ms: number): Promise<void> {
+ return new Promise((r) => setTimeout(r, ms));
+}
+
+function signalProbe(pid: number): 'alive' | 'ESRCH' | 'EPERM' {
+ try {
+ process.kill(pid, 0);
+ return 'alive';
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code as 'ESRCH' | 'EPERM';
+ }
+}
+
+/** A pid that is guaranteed dead: spawn a child, wait for it to exit and be reaped. */
+async function deadPid(): Promise<number> {
+ const child = spawn(process.execPath, ['-e', ''], { stdio: 'ignore' });
+ const pid = child.pid!;
+ await new Promise((resolveExit) => child.once('exit', resolveExit));
+ expect(signalProbe(pid)).toBe('ESRCH');
+ return pid;
+}
+
+/** A pid that stays alive (same user) until afterEach kills it. */
+function livePid(): number {
+ const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });
+ children.push(child);
+ expect(signalProbe(child.pid!)).toBe('alive');
+ return child.pid!;
+}
+
+/**
+ * A migrated project plus the lock exactly as `acquireLock()` writes it in
+ * this process, so planted locks can never drift from the real format.
+ */
+async function setup(): Promise<{ root: string; lock: string; owner: LockOwner }> {
+ const root = mkdtempSync(join(tmpdir(), 'ruflo-policy-lock-dead-owner-'));
+ mkdirSync(join(root, '.claude-flow'), { recursive: true });
+ roots.push(root);
+ await autoMigratePolicyStateIfNeeded(root);
+ const lock = join(root, '.claude-flow', 'policy', 'state.lock');
+ let written = '';
+ await withPolicyTransaction(root, () => { written = readFileSync(lock, 'utf8'); });
+ const owner = JSON.parse(written) as LockOwner;
+ expect(owner.pid).toBe(process.pid);
+ return { root, lock, owner };
+}
+
+/** Leave behind what a crashed owner would: the lock with a fresh mtime. */
+function plantLock(lock: string, content: string): void {
+ writeFileSync(lock, content);
+}
+
+/** The MCP chokepoint every tool call (reads included) passes through. */
+function authorizeRead(root: string) {
+ return authorizeMcpTool(
+ 'memory_retrieve',
+ { key: 'k', namespace: 'default' },
+ { projectRoot: root },
+ classifyMcpTool('memory_retrieve'),
+ );
+}
+
+/**
+ * Start an authorization against a lock it must wait for, check it is still
+ * pending (and the lock untouched) well after the first poll, then release the
+ * lock the way its owner would and check the waiter proceeds. `midway` runs
+ * after the first poll, 150 ms in.
+ */
+async function expectWaitsForOwner(
+ root: string,
+ lock: string,
+ content: string,
+ midway?: () => void,
+): Promise<void> {
+ let settled = false;
+ const outcome = authorizeRead(root)
+ .then((decision) => ({ decision }), (error: unknown) => ({ error }))
+ .finally(() => { settled = true; });
+ await sleep(150);
+ midway?.();
+ await sleep(150);
+ expect(settled).toBe(false);
+ expect(readFileSync(lock, 'utf8')).toBe(content);
+ unlinkSync(lock); // the owner's release()
+ expect(await outcome).toMatchObject({ decision: { enforcedOutcome: 'allowed' } });
+}
+
+describe('policy state.lock owned by a dead process', () => {
+ it('reclaims the lock at once instead of stalling memory_retrieve until policy-state-lock-timeout', async () => {
+ const { root, lock, owner } = await setup();
+ plantLock(lock, JSON.stringify({ ...owner, pid: await deadPid() }));
+
+ const started = Date.now();
+ const decision = await authorizeRead(root);
+
+ expect(decision.enforcedOutcome).toBe('allowed');
+ // Before the fix this waited the full LOCK_WAIT_MS and threw.
+ expect(Date.now() - started).toBeLessThan(LOCK_WAIT_MS / 5);
+ // The transaction released its own lock afterwards.
+ expect(existsSync(lock)).toBe(false);
+ }, 15_000);
+
+ it('recovers on every call: five crashes in a row cost no timeouts', async () => {
+ const { root, lock, owner } = await setup();
+ const started = Date.now();
+ for (let i = 0; i < 5; i++) {
+ plantLock(lock, JSON.stringify({ ...owner, pid: await deadPid(), acquiredAt: Date.now() }));
+ expect((await authorizeRead(root)).enforcedOutcome).toBe('allowed');
+ }
+ expect(Date.now() - started).toBeLessThan(LOCK_WAIT_MS);
+ }, 40_000);
+
+ // The re-check after the probe asks "is this still the dead owner's lock?",
+ // not "is this the same file?", so it compares the lock's bytes. This pins
+ // that portably: the lock is replaced at every probe by a byte-identical
+ // file, which `dev`/`ino`/`ctime` reads as a different lock and stalls on
+ // forever. The same property is what makes the check sound on filesystems
+ // where the stat triple is not an identity — ext4 reuses a freed inode
+ // number at once, and before Linux 6.13 two files created in one clock tick
+ // share a ctime, so there the triple matches a *different* lock.
+ it('reclaims a dead owner whose lock file is replaced by an identical one while it is probed', async () => {
+ const { root, lock, owner } = await setup();
+ const dead = await deadPid();
+ const content = JSON.stringify({ ...owner, pid: dead });
+ plantLock(lock, content);
+ let replacements = 0;
+ const realKill = process.kill.bind(process);
+ const kill = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => {
+ if (pid === dead && signal === 0) {
+ replacements++;
+ unlinkSync(lock); // a different file (new inode, new ctime)...
+ writeFileSync(lock, content); // ...holding the same dead owner
+ }
+ return realKill(pid, signal);
+ }) as typeof process.kill);
+ try {
+ const started = Date.now();
+ expect((await authorizeRead(root)).enforcedOutcome).toBe('allowed');
+ expect(replacements).toBeGreaterThan(0);
+ expect(Date.now() - started).toBeLessThan(LOCK_WAIT_MS / 5);
+ } finally {
+ kill.mockRestore();
+ }
+ }, 15_000);
+});
+
+describe('policy state.lock that must still be waited for', () => {
+ it('a live owner', async () => {
+ const { root, lock, owner } = await setup();
+ const content = JSON.stringify({ ...owner, pid: livePid() });
+ plantLock(lock, content);
+ await expectWaitsForOwner(root, lock, content);
+ }, 15_000);
+
+ // POSIX only: Windows has no pid 1.
+ it.skipIf(process.platform === 'win32')('an owner signal 0 cannot reach (EPERM counts as alive)', async () => {
+ const { root, lock, owner } = await setup();
+ // pid 1 (init/launchd) always exists. As a non-root user signal 0 to it
+ // fails with EPERM, which must mean "alive", not "dead". Under root it
+ // simply succeeds, so the assertion below holds either way.
+ expect(['alive', 'EPERM']).toContain(signalProbe(1));
+ const content = JSON.stringify({ ...owner, pid: 1 });
+ plantLock(lock, content);
+ await expectWaitsForOwner(root, lock, content);
+ }, 15_000);
+
+ it('a new owner that acquired while the previous owner was being probed', async () => {
+ const { root, lock, owner } = await setup();
+ const dead = await deadPid();
+ plantLock(lock, JSON.stringify({ ...owner, pid: dead }));
+ const next = JSON.stringify({ ...owner, pid: livePid() });
+ let handedOver = false;
+ const handOver = () => {
+ if (handedOver) return;
+ handedOver = true;
+ unlinkSync(lock); // the old lock goes away...
+ writeFileSync(lock, next); // ...and a new owner creates a fresh one
+ };
+ // Hand the lock over exactly between the waiter reading it and probing
+ // its pid. A waiter that never probes pids gets the same handover midway.
+ const realKill = process.kill.bind(process);
+ const kill = vi.spyOn(process, 'kill').mockImplementation(((pid: number, signal?: string | number) => {
+ if (pid === dead && signal === 0) handOver();
+ return realKill(pid, signal);
+ }) as typeof process.kill);
+ try {
+ await expectWaitsForOwner(root, lock, next, handOver);
+ } finally {
+ kill.mockRestore();
+ }
+ }, 15_000);
+
+ // A pid is only meaningful where it was written. Each of these carries a
+ // dead (or impossible) pid, so only the identity/format check keeps it.
+ it.each<[string, (owner: LockOwner, dead: number) => string]>([
+ ['written on another host', (o, dead) => JSON.stringify({ ...o, pid: dead, host: 'another-host.invalid' })],
+ ['written by another OS (a WSL1 owner read from Windows)',
+ (o, dead) => JSON.stringify({
+ ...o, pid: dead, platform: process.platform === 'linux' ? 'win32' : 'linux',
+ })],
+ ['written in another PID namespace (bwrap --unshare-pid, containers)',
+ (o, dead) => JSON.stringify({ ...o, pid: dead, pidns: 'pid:[0]' })],
+ ['written by ruflo <= 3.42.4 (no identity)',
+ (_o, dead) => JSON.stringify({ pid: dead, acquiredAt: Date.now() })],
+ ['with a non-positive pid', (o) => JSON.stringify({ ...o, pid: -2147483000 })],
+ ['with a pid beyond 2^31 - 1', (o) => JSON.stringify({ ...o, pid: 2 ** 31 })],
+ // `process.kill` coerces a numeric string, so "1234" would be probed as a
+ // pid and answer ESRCH; only the `typeof pid !== 'number'` guard stops it.
+ ['with a pid that is not a number', (o, dead) => JSON.stringify({ ...o, pid: `${dead}` })],
+ ['larger than LOCK_MAX_BYTES',
+ (o, dead) => JSON.stringify({ ...o, pid: dead }).padEnd(LOCK_MAX_BYTES + 1, ' ')],
+ ['that is empty (crash between create and write)', () => ''],
+ ])('a lock %s', async (_name, build) => {
+ const { root, lock, owner } = await setup();
+ const content = build(owner, await deadPid());
+ plantLock(lock, content);
+ await expectWaitsForOwner(root, lock, content);
+ }, 15_000);
+
+ // Linux only: `boot` is only written where /proc/sys/kernel/random/boot_id
+ // exists. It is what tells two machines apart when they share a filesystem
+ // and report the same hostname.
+ it.skipIf(process.platform !== 'linux')('a lock written on another kernel boot (same hostname)', async () => {
+ const { root, lock, owner } = await setup();
+ expect(typeof owner.boot).toBe('string');
+ const content = JSON.stringify({ ...owner, pid: await deadPid(), boot: '00000000-0000-0000-0000-000000000000' });
+ plantLock(lock, content);
+ await expectWaitsForOwner(root, lock, content);
+ }, 15_000);
+});
+
+describe('policy state.lock mtime fallback', () => {
+ it('still reclaims an old lock with no parsable pid', async () => {
+ const { root, lock } = await setup();
+ plantLock(lock, '{"pid":');
+ const old = (Date.now() - LOCK_STALE_MS - 5_000) / 1000;
+ utimesSync(lock, old, old);
+
+ const started = Date.now();
+ expect((await authorizeRead(root)).enforcedOutcome).toBe('allowed');
+ expect(Date.now() - started).toBeLessThan(LOCK_WAIT_MS / 5);
+ }, 15_000);
+
+ // POSIX only: needs mkfifo and symlinks. Opening a FIFO for reading blocks
+ // the whole event loop until a writer appears, so the lock must never be
+ // read through a symlink or from a non-regular file.
+ it.skipIf(process.platform === 'win32')('never reads a symlinked FIFO lock, so it cannot block the event loop', async () => {
+ const { root, lock } = await setup();
+ const fifo = join(root, 'owner.fifo');
+ execFileSync('mkfifo', [fifo]);
+ symlinkSync(fifo, lock);
+ const old = (Date.now() - LOCK_STALE_MS - 5_000) / 1000;
+ utimesSync(fifo, old, old);
+ // A regression here blocks the event loop inside open(), which no test
+ // timeout can interrupt: the run would hang instead of failing. This
+ // writer opens the FIFO after 2 s, so a regressed reader unblocks in time
+ // to fail the 1 s assertion below. Correct code never opens the FIFO, so
+ // the writer stays blocked in its own open() until afterEach kills it.
+ children.push(spawn('sh', ['-c', 'sleep 2; : > "$1"', 'sh', fifo], { stdio: 'ignore' }));
+
+ const started = Date.now();
+ expect((await authorizeRead(root)).enforcedOutcome).toBe('allowed');
+ expect(Date.now() - started).toBeLessThan(1_000);
+ // The mtime rule removed the symlink, not its target.
+ expect(existsSync(fifo)).toBe(true);
+ }, 15_000);
+});
diff --git a/v3/@claude-flow/cli/src/services/policy-runtime.ts b/v3/@claude-flow/cli/src/services/policy-runtime.ts
index d68014d4a1..97876b9a90 100644
--- a/v3/@claude-flow/cli/src/services/policy-runtime.ts
+++ b/v3/@claude-flow/cli/src/services/policy-runtime.ts
@@ -18,10 +18,15 @@ import { createHash, createHmac, randomBytes, randomUUID, timingSafeEqual } from
import { execFileSync } from 'node:child_process';
import {
closeSync,
+ constants as fsConstants,
existsSync,
+ fstatSync,
+ lstatSync,
mkdirSync,
openSync,
readFileSync,
+ readSync,
+ readlinkSync,
realpathSync,
renameSync,
statSync,
@@ -29,13 +34,28 @@ import {
writeFileSync,
} from 'node:fs';
import { dirname, join, resolve } from 'node:path';
-import { userInfo } from 'node:os';
+import { hostname, userInfo } from 'node:os';
const POLICY_DIR = join('.claude-flow', 'policy');
const POLICY_FILE = 'state.json';
const LOCK_FILE = 'state.lock';
const LOCK_STALE_MS = 30_000;
const LOCK_WAIT_MS = 5_000;
+// A lock is ~170 bytes plus the hostname (at most 255 bytes).
+const LOCK_MAX_BYTES = 1_024;
+// Linux PID namespace of this process, e.g. `pid:[4026531836]`; undefined on
+// other platforms or without /proc. Recorded in the lock because a pid only
+// identifies a process inside the namespace that wrote it.
+const PID_NAMESPACE = (() => {
+ try { return readlinkSync('/proc/self/ns/pid'); } catch { return undefined; }
+})();
+// Linux boot id: one uuid per kernel boot, shared by every process on that
+// kernel and readable inside containers. Recorded because `host` is only a
+// name: two machines that share a project directory can report the same
+// hostname, and the init PID namespace has the same inode on every Linux host.
+const BOOT_ID = (() => {
+ try { return readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(); } catch { return undefined; }
+})();
function paths(projectRoot: string): { dir: string; state: string; lock: string } {
const root = resolve(projectRoot);
@@ -47,19 +67,102 @@ function sleep(ms: number): Promise<void> {
return new Promise((resolveSleep) => setTimeout(resolveSleep, ms));
}
+/**
+ * Signal-0 existence probe, as in #1799 (`isPidAlive` in swarm-tools.ts), but
+ * only ESRCH counts as dead (as `processIsAlive` in helper-refresh.ts): here a
+ * false "dead" breaks mutual exclusion, so EPERM (another user's process) and
+ * any unexpected error mean alive.
+ */
+function isPidAlive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (e) {
+ return (e as NodeJS.ErrnoException).code !== 'ESRCH';
+ }
+}
+
+/**
+ * Reads a lock only if it is a small regular file: never follows a symlink,
+ * never blocks on a FIFO or device, never reads more than LOCK_MAX_BYTES.
+ * Anything else returns undefined and the caller keeps the mtime rule.
+ */
+function readLockFile(lockPath: string): string | undefined {
+ if (!lstatSync(lockPath).isFile()) return undefined;
+ const fd = openSync(
+ lockPath,
+ fsConstants.O_RDONLY | (fsConstants.O_NOFOLLOW ?? 0) | (fsConstants.O_NONBLOCK ?? 0),
+ );
+ try {
+ const stat = fstatSync(fd);
+ if (!stat.isFile() || stat.size > LOCK_MAX_BYTES) return undefined;
+ const buffer = Buffer.alloc(LOCK_MAX_BYTES);
+ return buffer.toString('utf8', 0, readSync(fd, buffer, 0, LOCK_MAX_BYTES, 0));
+ } finally {
+ closeSync(fd);
+ }
+}
+
+/**
+ * A process killed inside a transaction (SIGKILL, OOM, or a signal whose
+ * default action terminates it) never runs `finally { release() }`, so every
+ * caller, MCP reads included, waited LOCK_WAIT_MS and failed with
+ * policy-state-lock-timeout until the orphan's mtime crossed LOCK_STALE_MS.
+ * The pid is trusted only if the lock was written by this OS, on this kernel
+ * boot and in this PID namespace: anywhere else (a bwrap --unshare-pid
+ * sandbox, a container, another machine on a shared filesystem) a live owner
+ * also answers ESRCH. Locks without that identity (older ruflo, a crash
+ * between create and write, anything unparsable) return false and keep the
+ * mtime rule.
+ */
+function lockOwnerIsDead(lockPath: string): boolean {
+ try {
+ const read = readLockFile(lockPath);
+ if (read === undefined) return false;
+ const owner = JSON.parse(read) as
+ { pid?: unknown; host?: unknown; platform?: unknown; pidns?: unknown; boot?: unknown } | null;
+ if (!owner || owner.host !== hostname() || owner.platform !== process.platform
+ || owner.pidns !== PID_NAMESPACE || owner.boot !== BOOT_ID) return false;
+ // Without /proc, one Linux kernel boot or PID namespace cannot be told
+ // from another, and `host` is only a name.
+ if (process.platform === 'linux'
+ && (PID_NAMESPACE === undefined || BOOT_ID === undefined)) return false;
+ const pid = owner.pid;
+ if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0 || pid > 0x7fffffff
+ || isPidAlive(pid)) return false;
+ // The lock can change hands while it is read and probed (its owner
+ // releases and exits, another process acquires). Re-read it and answer
+ // only if it is byte for byte the lock that was read: the next owner's
+ // lock always differs in `pid` or `acquiredAt`. `dev`/`ino`/`ctime` cannot
+ // stand in for that — ext4 reuses a freed inode number at once, and before
+ // Linux 6.13 two files created in one clock tick share a ctime.
+ return readLockFile(lockPath) === read;
+ } catch {
+ return false;
+ }
+}
+
async function acquireLock(lockPath: string): Promise<() => void> {
const started = Date.now();
while (Date.now() - started < LOCK_WAIT_MS) {
try {
const fd = openSync(lockPath, 'wx', 0o600);
- writeFileSync(fd, JSON.stringify({ pid: process.pid, acquiredAt: Date.now() }));
+ writeFileSync(fd, JSON.stringify({
+ pid: process.pid,
+ acquiredAt: Date.now(),
+ host: hostname(),
+ platform: process.platform,
+ pidns: PID_NAMESPACE,
+ boot: BOOT_ID,
+ }));
closeSync(fd);
return () => {
try { unlinkSync(lockPath); } catch { /* already released */ }
};
} catch {
try {
- if (Date.now() - statSync(lockPath).mtimeMs > LOCK_STALE_MS) unlinkSync(lockPath);
+ if (lockOwnerIsDead(lockPath)
+ || Date.now() - statSync(lockPath).mtimeMs > LOCK_STALE_MS) unlinkSync(lockPath);
} catch { /* another process changed the lock */ }
await sleep(10);
}
diff --git a/v3/docs/adr/ADR-324-agentic-policy-engine-codex-swarm.md b/v3/docs/adr/ADR-324-agentic-policy-engine-codex-swarm.md
index bf82b697b3..74649df7fa 100644
--- a/v3/docs/adr/ADR-324-agentic-policy-engine-codex-swarm.md
+++ b/v3/docs/adr/ADR-324-agentic-policy-engine-codex-swarm.md
@@ -249,8 +249,11 @@ The normative implementation is split across existing package boundaries:
The policy state lives at `.claude-flow/policy/state.json`. Mutations acquire
`.claude-flow/policy/state.lock` using exclusive creation and replace state by
atomic rename. Decision, approval consumption, budget usage, and receipt append
-therefore commit as one transaction. A stale lock may be recovered after 30
-seconds; lock acquisition otherwise fails after five seconds.
+therefore commit as one transaction. The lock records its owner's pid, host,
+platform, and (on Linux) PID namespace and kernel boot id. A lock whose owner
+no longer exists under that same identity, and whose bytes are unchanged since
+they were read, is recovered immediately; any other lock may be recovered after
+30 seconds. Lock acquisition otherwise fails after five seconds.
When enforcement is first enabled, an HMAC key and state authentication record
are stored outside the workspace under