Skip to content

armgit / ruflo

Public Imported from GitHub
0

🌊 The original agent harness. Deploy intelligent multi-player swarms, coordinate autonomous workflows, and build conversational AI systems. Features adaptive memory, self-learning intelligence, federation, vector RAG integration, and native Claude Code / Codex / Hermes and many more Integrated

fix(daemon): opt-in AI workers, global launch budget, cross-worktree dedup, stop --all (#2661) (#2662)

rUv215840536e9e
diff --git a/scripts/audit-env-var-precedence.mjs b/scripts/audit-env-var-precedence.mjs
index 724662c0e9..75f14fb332 100644
--- a/scripts/audit-env-var-precedence.mjs
+++ b/scripts/audit-env-var-precedence.mjs
@@ -64,6 +64,9 @@ const KNOWN_ESCAPE_HATCHES = new Set([
'RUFLO_FUNNEL_EVENTS_ENDPOINT', // Staging/self-hosted override for the funnel events endpoint β€” same deployment-config pattern as CLICK_ENDPOINT above
'RUFLO_FUNNEL_MESSAGES_ENDPOINT', // Staging/self-hosted override for the funnel messages endpoint β€” same deployment-config pattern as CLICK_ENDPOINT above
'RUFLO_STATE_DIR', // Test/CI isolation seam for the funnel state directory (defaults to ~/.ruflo) β€” background hook state, no CLI command reads it
+ 'RUFLO_AI_DEDUP_DISABLE', // #2661 β€” background daemon tuning knob (cross-worktree AI job dedup), no CLI command reads a running daemon's config
+ 'RUFLO_AI_DEDUP_WINDOW_SECS', // #2661 β€” same background daemon tuning context as RUFLO_AI_DEDUP_DISABLE above
+ 'RUFLO_DAEMON_AI_WORKERS', // #2661 β€” DOES have CLI-flag precedence (`daemon start --headless`), but it's wired via constructor-injected config in commands/daemon.ts, not a local check the audit's same-file heuristic can see from worker-daemon.ts where this read lives
// ── Embedding substrate toggles (3.25.x β€” opt-in tier + fail-closed ops flag) ─
'RUFLO_REQUIRE_REAL_EMBEDDINGS', // Fail-closed "no stubs" strict mode β€” deploy/CI ops toggle, not a per-invocation CLI flag (ADR-176)
diff --git a/v3/@claude-flow/cli/__tests__/daemon-ai-workers-optin-2661.test.ts b/v3/@claude-flow/cli/__tests__/daemon-ai-workers-optin-2661.test.ts
new file mode 100644
index 0000000000..b311de544a
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/daemon-ai-workers-optin-2661.test.ts
@@ -0,0 +1,152 @@
+/**
+ * Regression guard for ruvnet/ruflo#2661 β€” scheduled AI workers must be
+ * OPT-IN. Merely finding the Claude CLI on PATH must never authorize
+ * recurring `claude --print` launches: a default install produces ZERO
+ * autonomous Claude launches regardless of how many worktree daemons run.
+ *
+ * Consent gates (any one):
+ * - `daemon start --headless` β†’ constructor { aiWorkersEnabled: true }
+ * - `daemon.aiWorkers.enabled: true` β†’ .claude-flow/config.json
+ * - `RUFLO_DAEMON_AI_WORKERS=1` β†’ environment
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { WorkerDaemon } from '../src/services/worker-daemon.js';
+import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+
+describe('#2661 β€” AI workers are opt-in', () => {
+ let tempDir: string;
+
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), 'daemon-2661-test-'));
+ mkdirSync(join(tempDir, '.claude-flow', 'logs'), { recursive: true });
+ delete process.env.RUFLO_DAEMON_AI_WORKERS;
+ });
+
+ afterEach(() => {
+ delete process.env.RUFLO_DAEMON_AI_WORKERS;
+ rmSync(tempDir, { recursive: true, force: true });
+ process.removeAllListeners('SIGTERM');
+ process.removeAllListeners('SIGINT');
+ process.removeAllListeners('SIGHUP');
+ });
+
+ it('defaults to aiWorkersEnabled=false (invariant 1: zero autonomous launches)', () => {
+ const daemon = new WorkerDaemon(tempDir);
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(false);
+ });
+
+ it('never constructs a headless executor without consent β€” even with claude on PATH', async () => {
+ const daemon = new WorkerDaemon(tempDir);
+ // Let the (gated) init settle.
+ await new Promise((r) => setTimeout(r, 50));
+ expect(daemon.isHeadlessAvailable()).toBe(false);
+ expect(daemon.getHeadlessExecutor()).toBeNull();
+ });
+
+ it('enables via constructor config (the `--headless` flag path)', () => {
+ const daemon = new WorkerDaemon(tempDir, { aiWorkersEnabled: true });
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(true);
+ });
+
+ it('enables via RUFLO_DAEMON_AI_WORKERS=1', () => {
+ process.env.RUFLO_DAEMON_AI_WORKERS = '1';
+ const daemon = new WorkerDaemon(tempDir);
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(true);
+ });
+
+ it('enables via daemon.aiWorkers.enabled in .claude-flow/config.json', () => {
+ writeFileSync(
+ join(tempDir, '.claude-flow', 'config.json'),
+ JSON.stringify({ 'daemon.aiWorkers.enabled': true })
+ );
+ const daemon = new WorkerDaemon(tempDir);
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(true);
+ });
+
+ it('config.json false beats env opt-in precedence is flag > file > env', () => {
+ process.env.RUFLO_DAEMON_AI_WORKERS = '1';
+ writeFileSync(
+ join(tempDir, '.claude-flow', 'config.json'),
+ JSON.stringify({ 'daemon.aiWorkers.enabled': false })
+ );
+ const daemon = new WorkerDaemon(tempDir);
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(false);
+ });
+
+ it('constructor consent beats config.json opt-out', () => {
+ writeFileSync(
+ join(tempDir, '.claude-flow', 'config.json'),
+ JSON.stringify({ 'daemon.aiWorkers.enabled': false })
+ );
+ const daemon = new WorkerDaemon(tempDir, { aiWorkersEnabled: true });
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(true);
+ });
+
+ it('is NOT resurrected from a stale daemon-state.json (consent is never persisted state)', () => {
+ // Simulate an old state file written by a daemon that had AI enabled.
+ writeFileSync(
+ join(tempDir, '.claude-flow', 'daemon-state.json'),
+ JSON.stringify({
+ running: false,
+ config: { aiWorkersEnabled: true, workers: [] },
+ workers: {},
+ })
+ );
+ const daemon = new WorkerDaemon(tempDir);
+ expect(daemon.getStatus().config.aiWorkersEnabled).toBe(false);
+ });
+
+ it('a headless-type worker triggered without consent runs the LOCAL path', async () => {
+ const daemon = new WorkerDaemon(tempDir);
+ // `audit` is a headless-capable worker; without consent it must fall
+ // through to the $0 local implementation.
+ const result = await daemon.triggerWorker('audit');
+ expect(result.success).toBe(true);
+ expect((result.output as { mode?: string })?.mode).not.toBe('headless');
+ });
+});
+
+describe('#2661 β€” lifecycle: removed worktree shuts its daemon down', () => {
+ let tempDir: string;
+
+ beforeEach(() => {
+ tempDir = mkdtempSync(join(tmpdir(), 'daemon-2661-lc-'));
+ mkdirSync(join(tempDir, '.claude-flow', 'logs'), { recursive: true });
+ });
+
+ afterEach(() => {
+ rmSync(tempDir, { recursive: true, force: true });
+ process.removeAllListeners('SIGTERM');
+ process.removeAllListeners('SIGINT');
+ process.removeAllListeners('SIGHUP');
+ });
+
+ // Access the private predicate directly β€” the production path wraps it in
+ // a 60s interval + process.exit(), neither of which belongs in a test.
+ type WithPredicate = { lifecycleShutdownReason(now: number): string | null };
+
+ it('reports no shutdown reason while the workspace exists (ttl/idle disabled)', () => {
+ const daemon = new WorkerDaemon(tempDir, { ttlMs: 0, idleShutdownMs: 0 });
+ const reason = (daemon as unknown as WithPredicate).lifecycleShutdownReason(Date.now());
+ expect(reason).toBeNull();
+ });
+
+ it('requests shutdown once the workspace directory is gone β€” even with ttl/idle disabled', () => {
+ const daemon = new WorkerDaemon(tempDir, { ttlMs: 0, idleShutdownMs: 0 });
+ rmSync(tempDir, { recursive: true, force: true });
+ const reason = (daemon as unknown as WithPredicate).lifecycleShutdownReason(Date.now());
+ expect(reason).toMatch(/workspace directory removed/);
+ });
+
+ it('still enforces the TTL through the shared predicate', () => {
+ const daemon = new WorkerDaemon(tempDir, { ttlMs: 1000, idleShutdownMs: 0 });
+ // No startedAt (daemon not started) β†’ falls back to `now`, so a far
+ // future timestamp exceeds the 1s TTL deterministically.
+ (daemon as unknown as { startedAt?: Date }).startedAt = new Date(Date.now() - 5000);
+ const reason = (daemon as unknown as WithPredicate).lifecycleShutdownReason(Date.now());
+ expect(reason).toMatch(/max age/);
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/services/ai-job-dedup.test.ts b/v3/@claude-flow/cli/__tests__/services/ai-job-dedup.test.ts
new file mode 100644
index 0000000000..3e51438ac0
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/services/ai-job-dedup.test.ts
@@ -0,0 +1,97 @@
+/**
+ * #2661 invariant 5 β€” cross-worktree AI job dedup registry.
+ * Same repositoryId + HEAD + worker + config β†’ at most one run per
+ * freshness window, shared across registry instances (daemons).
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import {
+ AiJobDedupRegistry,
+ computeAiJobKey,
+ hashWorkerConfig,
+} from '../../src/services/ai-job-dedup.js';
+
+const PARTS = {
+ repositoryId: 'repo-abc',
+ head: 'deadbeef',
+ workerType: 'audit',
+ configHash: 'cfg-1',
+};
+
+describe('#2661 β€” AI job dedup registry', () => {
+ let dir: string;
+
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), 'ai-jobs-test-'));
+ delete process.env.RUFLO_AI_DEDUP_DISABLE;
+ });
+
+ afterEach(() => {
+ delete process.env.RUFLO_AI_DEDUP_DISABLE;
+ rmSync(dir, { recursive: true, force: true });
+ });
+
+ describe('computeAiJobKey', () => {
+ it('is stable for identical parts and distinct when any part changes', () => {
+ const base = computeAiJobKey(PARTS);
+ expect(computeAiJobKey({ ...PARTS })).toBe(base);
+ expect(computeAiJobKey({ ...PARTS, head: 'other' })).not.toBe(base);
+ expect(computeAiJobKey({ ...PARTS, workerType: 'optimize' })).not.toBe(base);
+ expect(computeAiJobKey({ ...PARTS, repositoryId: 'repo-xyz' })).not.toBe(base);
+ expect(computeAiJobKey({ ...PARTS, configHash: 'cfg-2' })).not.toBe(base);
+ });
+ });
+
+ describe('hashWorkerConfig', () => {
+ it('is insensitive to object key order', () => {
+ expect(hashWorkerConfig({ a: 1, b: { c: 2, d: 3 } }))
+ .toBe(hashWorkerConfig({ b: { d: 3, c: 2 }, a: 1 }));
+ });
+
+ it('changes when values change', () => {
+ expect(hashWorkerConfig({ a: 1 })).not.toBe(hashWorkerConfig({ a: 2 }));
+ });
+ });
+
+ describe('freshness', () => {
+ it('a recorded success is fresh within the window and shared across instances', () => {
+ const key = computeAiJobKey(PARTS);
+ const daemonA = new AiJobDedupRegistry({ baseDir: dir });
+ const daemonB = new AiJobDedupRegistry({ baseDir: dir });
+
+ expect(daemonA.isFresh(key, 60_000).fresh).toBe(false);
+ daemonA.recordSuccess(key, { workerType: 'audit', repositoryId: PARTS.repositoryId, workspace: '/tmp/wt-a' });
+
+ // Sibling worktree's daemon sees the same registry.
+ const seen = daemonB.isFresh(key, 60_000);
+ expect(seen.fresh).toBe(true);
+ expect(seen.lastRunAt).toBeTypeOf('number');
+ });
+
+ it('goes stale outside the freshness window', () => {
+ const key = computeAiJobKey(PARTS);
+ const reg = new AiJobDedupRegistry({ baseDir: dir });
+ reg.recordSuccess(key, { workerType: 'audit', repositoryId: PARTS.repositoryId, workspace: '/tmp/wt' });
+ expect(reg.isFresh(key, 0).fresh).toBe(false);
+ });
+
+ it('a different HEAD is a different job (never deduped)', () => {
+ const reg = new AiJobDedupRegistry({ baseDir: dir });
+ const keyA = computeAiJobKey(PARTS);
+ reg.recordSuccess(keyA, { workerType: 'audit', repositoryId: PARTS.repositoryId, workspace: '/tmp/wt' });
+ const keyB = computeAiJobKey({ ...PARTS, head: 'cafebabe' });
+ expect(reg.isFresh(keyB, 60_000).fresh).toBe(false);
+ });
+
+ it('RUFLO_AI_DEDUP_DISABLE=1 turns dedup off', () => {
+ const key = computeAiJobKey(PARTS);
+ const reg = new AiJobDedupRegistry({ baseDir: dir });
+ reg.recordSuccess(key, { workerType: 'audit', repositoryId: PARTS.repositoryId, workspace: '/tmp/wt' });
+ process.env.RUFLO_AI_DEDUP_DISABLE = '1';
+ expect(reg.isFresh(key, 60_000).fresh).toBe(false);
+ });
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/services/git-workspace-identity.test.ts b/v3/@claude-flow/cli/__tests__/services/git-workspace-identity.test.ts
new file mode 100644
index 0000000000..cfa665f16f
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/services/git-workspace-identity.test.ts
@@ -0,0 +1,104 @@
+/**
+ * #2661 β€” Git workspace identity: worktrees of one repository must resolve
+ * to the SAME repositoryId (the shared scheduler/dedup key), while the
+ * worktreeRoot stays per-worktree. Non-git directories degrade gracefully.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { execFileSync } from 'child_process';
+import { mkdtempSync, rmSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import {
+ resolveGitWorkspaceIdentity,
+ resetGitIdentityCacheForTests,
+} from '../../src/services/git-workspace-identity.js';
+
+const git = (cwd: string, ...args: string[]) =>
+ execFileSync('git', args, { cwd, encoding: 'utf-8', stdio: ['ignore', 'pipe', 'pipe'] }).trim();
+
+describe('#2661 β€” git workspace identity', () => {
+ let repo: string;
+ let worktree: string;
+
+ beforeEach(() => {
+ resetGitIdentityCacheForTests();
+ repo = mkdtempSync(join(tmpdir(), 'gwi-repo-'));
+ worktree = join(mkdtempSync(join(tmpdir(), 'gwi-wt-')), 'wt');
+ git(repo, 'init', '-q');
+ git(repo, 'config', 'user.email', '[email protected]');
+ git(repo, 'config', 'user.name', 'Test');
+ writeFileSync(join(repo, 'a.txt'), 'hello');
+ git(repo, 'add', 'a.txt');
+ git(repo, 'commit', '-q', '-m', 'init');
+ });
+
+ afterEach(() => {
+ resetGitIdentityCacheForTests();
+ rmSync(worktree, { recursive: true, force: true });
+ rmSync(repo, { recursive: true, force: true });
+ });
+
+ it('resolves root, common git dir, repositoryId, and HEAD for a plain clone', () => {
+ const id = resolveGitWorkspaceIdentity(repo);
+ expect(id.isGit).toBe(true);
+ expect(id.worktreeRoot).toBeTruthy();
+ expect(id.commonGitDir).toContain('.git');
+ expect(id.repositoryId).toMatch(/^[0-9a-f]{64}$/);
+ expect(id.head).toMatch(/^[0-9a-f]{40}$/);
+ });
+
+ it('two worktrees of one repository share repositoryId and HEAD', () => {
+ git(repo, 'worktree', 'add', '-q', worktree, '-b', 'wt-branch');
+
+ const a = resolveGitWorkspaceIdentity(repo);
+ const b = resolveGitWorkspaceIdentity(worktree);
+
+ // The fanout key insight: DIFFERENT worktree roots...
+ expect(a.worktreeRoot).not.toBe(b.worktreeRoot);
+ // ...but ONE repository identity (same commit β†’ same dedup scope).
+ expect(b.repositoryId).toBe(a.repositoryId);
+ expect(b.head).toBe(a.head);
+ });
+
+ it('different repositories get different repositoryIds', () => {
+ const other = mkdtempSync(join(tmpdir(), 'gwi-other-'));
+ try {
+ git(other, 'init', '-q');
+ const a = resolveGitWorkspaceIdentity(repo);
+ const b = resolveGitWorkspaceIdentity(other);
+ expect(b.repositoryId).not.toBe(a.repositoryId);
+ } finally {
+ rmSync(other, { recursive: true, force: true });
+ }
+ });
+
+ it('HEAD changes when a new commit lands (identity cache does not pin it)', () => {
+ const before = resolveGitWorkspaceIdentity(repo);
+ writeFileSync(join(repo, 'b.txt'), 'more');
+ git(repo, 'add', 'b.txt');
+ git(repo, 'commit', '-q', '-m', 'second');
+ const after = resolveGitWorkspaceIdentity(repo);
+ expect(after.repositoryId).toBe(before.repositoryId);
+ expect(after.head).not.toBe(before.head);
+ });
+
+ it('degrades gracefully for non-git directories', () => {
+ const plain = mkdtempSync(join(tmpdir(), 'gwi-plain-'));
+ try {
+ const id = resolveGitWorkspaceIdentity(plain);
+ expect(id.isGit).toBe(false);
+ expect(id.head).toBe('');
+ expect(id.repositoryId).toMatch(/^[0-9a-f]{64}$/);
+ // Distinct non-git dirs must not collide.
+ const other = mkdtempSync(join(tmpdir(), 'gwi-plain2-'));
+ try {
+ expect(resolveGitWorkspaceIdentity(other).repositoryId).not.toBe(id.repositoryId);
+ } finally {
+ rmSync(other, { recursive: true, force: true });
+ }
+ } finally {
+ rmSync(plain, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/services/global-ai-budget.test.ts b/v3/@claude-flow/cli/__tests__/services/global-ai-budget.test.ts
new file mode 100644
index 0000000000..1223069654
--- /dev/null
+++ b/v3/@claude-flow/cli/__tests__/services/global-ai-budget.test.ts
@@ -0,0 +1,201 @@
+/**
+ * #2661 β€” Global AI launch budget (emergency cost fuse).
+ *
+ * The budget is the user-global invariant that bounds autonomous
+ * `claude --print` launches independent of worktree/daemon count:
+ * - at most maxConcurrentGlobal active children
+ * - at most maxLaunchesPerHour / maxLaunchesPerDay launches
+ * - a quota error opens a circuit breaker that pauses everything
+ * - every denial carries a reason and is receipted
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, readFileSync, existsSync, writeFileSync, symlinkSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import {
+ GlobalAiBudget,
+ DEFAULT_AI_BUDGET_LIMITS,
+ isQuotaErrorText,
+} from '../../src/services/global-ai-budget.js';
+
+const REQ = { workerType: 'audit', model: 'haiku', workspace: '/tmp/wt-1' };
+
+describe('#2661 β€” GlobalAiBudget', () => {
+ let dir: string;
+
+ beforeEach(() => {
+ dir = mkdtempSync(join(tmpdir(), 'ai-budget-test-'));
+ });
+
+ afterEach(() => {
+ rmSync(dir, { recursive: true, force: true });
+ delete process.env.RUFLO_AI_BUDGET_DISABLE;
+ });
+
+ const makeBudget = (limits?: Partial<typeof DEFAULT_AI_BUDGET_LIMITS>) =>
+ new GlobalAiBudget({ baseDir: dir, limits });
+
+ describe('defaults', () => {
+ it('ships the #2661 containment limits', () => {
+ expect(DEFAULT_AI_BUDGET_LIMITS).toEqual({
+ maxConcurrentGlobal: 1,
+ maxLaunchesPerHour: 2,
+ maxLaunchesPerDay: 12,
+ pauseOnQuotaErrorMinutes: 60,
+ });
+ });
+ });
+
+ describe('reserve', () => {
+ it('grants a permit when under budget', async () => {
+ const budget = makeBudget();
+ const permit = await budget.reserve(REQ);
+ expect(permit.allowed).toBe(true);
+ expect(permit.permitId).toBeTruthy();
+ });
+
+ it('denies the second concurrent launch at maxConcurrentGlobal=1', async () => {
+ const budget = makeBudget({ maxLaunchesPerHour: 10, maxLaunchesPerDay: 10 });
+ const first = await budget.reserve(REQ);
+ expect(first.allowed).toBe(true);
+
+ const second = await budget.reserve({ ...REQ, workspace: '/tmp/wt-2' });
+ expect(second.allowed).toBe(false);
+ expect(second.reason).toMatch(/global-concurrency/);
+ });
+
+ it('allows a new launch after release() frees the concurrency slot', async () => {
+ const budget = makeBudget({ maxLaunchesPerHour: 10, maxLaunchesPerDay: 10 });
+ const first = await budget.reserve(REQ);
+ await budget.release(first.permitId);
+
+ const second = await budget.reserve(REQ);
+ expect(second.allowed).toBe(true);
+ });
+
+ it('enforces the hourly launch budget even after release', async () => {
+ const budget = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 2, maxLaunchesPerDay: 10 });
+ for (let i = 0; i < 2; i++) {
+ const p = await budget.reserve(REQ);
+ expect(p.allowed).toBe(true);
+ await budget.release(p.permitId);
+ }
+ const denied = await budget.reserve(REQ);
+ expect(denied.allowed).toBe(false);
+ expect(denied.reason).toMatch(/hourly-budget/);
+ });
+
+ it('enforces the daily budget across the 24h window', async () => {
+ const budget = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 10, maxLaunchesPerDay: 3 });
+ for (let i = 0; i < 3; i++) {
+ const p = await budget.reserve(REQ);
+ expect(p.allowed).toBe(true);
+ await budget.release(p.permitId);
+ }
+ const denied = await budget.reserve(REQ);
+ expect(denied.allowed).toBe(false);
+ expect(denied.reason).toMatch(/daily-budget/);
+ });
+
+ it('is shared across budget INSTANCES (simulating separate daemons/worktrees)', async () => {
+ // Two daemons in two worktrees each create their own GlobalAiBudget β€”
+ // the ledger on disk is the shared source of truth.
+ const daemonA = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 1, maxLaunchesPerDay: 10 });
+ const daemonB = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 1, maxLaunchesPerDay: 10 });
+
+ const a = await daemonA.reserve({ ...REQ, workspace: '/tmp/wt-a' });
+ expect(a.allowed).toBe(true);
+ await daemonA.release(a.permitId);
+
+ const b = await daemonB.reserve({ ...REQ, workspace: '/tmp/wt-b' });
+ expect(b.allowed).toBe(false);
+ expect(b.reason).toMatch(/hourly-budget/);
+ });
+
+ it('RUFLO_AI_BUDGET_DISABLE=1 bypasses the fuse (explicit escape hatch)', async () => {
+ process.env.RUFLO_AI_BUDGET_DISABLE = '1';
+ const budget = makeBudget({ maxLaunchesPerHour: 0, maxLaunchesPerDay: 0, maxConcurrentGlobal: 0 });
+ const p = await budget.reserve(REQ);
+ expect(p.allowed).toBe(true);
+ });
+
+ it('fails CLOSED when the ledger is a symlink (invariant 9)', async () => {
+ const budget = makeBudget();
+ writeFileSync(join(dir, 'evil-target.json'), '{}');
+ symlinkSync(join(dir, 'evil-target.json'), join(dir, 'ai-budget.json'));
+ const p = await budget.reserve(REQ);
+ expect(p.allowed).toBe(false);
+ expect(p.reason).toMatch(/budget-ledger-error/);
+ });
+ });
+
+ describe('quota circuit breaker', () => {
+ it('recordQuotaError pauses ALL subsequent launches for the cooldown', async () => {
+ const budget = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 10, maxLaunchesPerDay: 10 });
+ await budget.recordQuotaError('audit: 429 rate limit exceeded');
+
+ const denied = await budget.reserve(REQ);
+ expect(denied.allowed).toBe(false);
+ expect(denied.reason).toMatch(/circuit-open/);
+
+ const usage = budget.getUsage();
+ expect(usage.pausedUntil).toBeGreaterThan(Date.now());
+ });
+
+ it('circuit opened by one instance pauses another instance', async () => {
+ const daemonA = makeBudget();
+ const daemonB = makeBudget();
+ await daemonA.recordQuotaError('quota exhausted');
+ const p = await daemonB.reserve(REQ);
+ expect(p.allowed).toBe(false);
+ expect(p.reason).toMatch(/circuit-open/);
+ });
+ });
+
+ describe('receipts (invariant 10)', () => {
+ it('writes launch and deny receipts with reasons', async () => {
+ const budget = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 1, maxLaunchesPerDay: 10 });
+ const p = await budget.reserve(REQ);
+ await budget.release(p.permitId);
+ await budget.reserve(REQ); // denied
+
+ const receiptsFile = join(dir, 'ai-budget-receipts.jsonl');
+ expect(existsSync(receiptsFile)).toBe(true);
+ const lines = readFileSync(receiptsFile, 'utf-8').trim().split('\n').map((l) => JSON.parse(l));
+ expect(lines.some((r) => r.event === 'launch')).toBe(true);
+ const deny = lines.find((r) => r.event === 'deny');
+ expect(deny).toBeTruthy();
+ expect(deny.reason).toMatch(/hourly-budget/);
+ });
+ });
+
+ describe('getUsage', () => {
+ it('reports launches and active reservations', async () => {
+ const budget = makeBudget({ maxConcurrentGlobal: 10, maxLaunchesPerHour: 10, maxLaunchesPerDay: 10 });
+ await budget.reserve(REQ);
+ const p2 = await budget.reserve(REQ);
+ await budget.release(p2.permitId);
+
+ const usage = budget.getUsage();
+ expect(usage.lastHour).toBe(2);
+ expect(usage.lastDay).toBe(2);
+ expect(usage.active).toBe(1);
+ });
+ });
+
+ describe('isQuotaErrorText', () => {
+ it('matches quota / rate-limit failure signatures', () => {
+ expect(isQuotaErrorText('HTTP 429 Too Many Requests')).toBe(true);
+ expect(isQuotaErrorText('You have hit your usage limit')).toBe(true);
+ expect(isQuotaErrorText('rate_limit_error: quota exceeded')).toBe(true);
+ expect(isQuotaErrorText('overloaded_error')).toBe(true);
+ });
+
+ it('does not match unrelated errors', () => {
+ expect(isQuotaErrorText('Process exited with code 1')).toBe(false);
+ expect(isQuotaErrorText('ENOENT: claude not found')).toBe(false);
+ expect(isQuotaErrorText(undefined)).toBe(false);
+ });
+ });
+});
diff --git a/v3/@claude-flow/cli/__tests__/services/worker-daemon-resource-thresholds.test.ts b/v3/@claude-flow/cli/__tests__/services/worker-daemon-resource-thresholds.test.ts
index 4138a00ced..034e2e50ed 100644
--- a/v3/@claude-flow/cli/__tests__/services/worker-daemon-resource-thresholds.test.ts
+++ b/v3/@claude-flow/cli/__tests__/services/worker-daemon-resource-thresholds.test.ts
@@ -832,11 +832,18 @@ describe('WorkerDaemon resource thresholds', () => {
expect(config.ttlMs).toBe(1800 * 1000);
});
- it('does not arm the lifecycle monitor when both limits are disabled', () => {
+ it('arms the lifecycle monitor even when ttl/idle are disabled (#2661 workspace-removal check)', () => {
const daemon = new WorkerDaemon(tempDir, { ttlMs: 0, idleShutdownMs: 0 });
- // startLifecycleMonitor is a no-op when both are 0 β€” no timer is stored.
+ // Pre-#2661 this was a no-op with both limits at 0. The monitor now
+ // always runs so a daemon whose worktree is deleted shuts itself
+ // down; ttl/idle remain opt-in inside the shared predicate.
(daemon as any).startLifecycleMonitor();
- expect((daemon as any).lifecycleTimer).toBeUndefined();
+ expect((daemon as any).lifecycleTimer).toBeDefined();
+ // But with the workspace present and both limits off, the predicate
+ // must not request a shutdown.
+ expect((daemon as any).lifecycleShutdownReason(Date.now())).toBeNull();
+ clearInterval((daemon as any).lifecycleTimer);
+ (daemon as any).lifecycleTimer = undefined;
});
it('arms (and can clear) the lifecycle monitor when a TTL is set', () => {
diff --git a/v3/@claude-flow/cli/src/commands/daemon.ts b/v3/@claude-flow/cli/src/commands/daemon.ts
index ac674c0292..95ac446478 100644
--- a/v3/@claude-flow/cli/src/commands/daemon.ts
+++ b/v3/@claude-flow/cli/src/commands/daemon.ts
@@ -26,7 +26,11 @@ const startCommand: Command = {
{ name: 'quiet', short: 'Q', type: 'boolean', description: 'Suppress output' },
{ name: 'background', short: 'b', type: 'boolean', description: 'Run daemon in background (detached process)', default: true },
{ name: 'foreground', short: 'f', type: 'boolean', description: 'Run daemon in foreground (blocks terminal)' },
- { name: 'headless', type: 'boolean', description: 'Enable headless worker execution (E2B sandbox)' },
+ // #2661: --headless is the explicit consent gate for scheduled AI workers.
+ // Without it (or daemon.aiWorkers.enabled / RUFLO_DAEMON_AI_WORKERS=1),
+ // every worker runs its $0 local path β€” the daemon never spawns
+ // `claude --print` merely because the Claude CLI is on PATH.
+ { name: 'headless', type: 'boolean', description: 'Enable AI workers (scheduled `claude --print` execution, governed by the user-global AI budget). Default: off β€” workers run local-only' },
{ name: 'sandbox', type: 'string', description: 'Default sandbox mode for headless workers', choices: ['strict', 'permissive', 'disabled'] },
{ name: 'max-cpu-load', type: 'string', description: 'Override maxCpuLoad resource threshold (e.g. 4.0)' },
{ name: 'min-free-memory', type: 'string', description: 'Override minFreeMemoryPercent resource threshold (e.g. 15)' },
@@ -64,6 +68,15 @@ const startCommand: Command = {
// Parse resource threshold overrides from CLI flags
const config: Partial<DaemonConfig> = {};
+
+ // #2661: thread --headless into DaemonConfig so it actually gates the
+ // headless executor. Previously the flag was forwarded to the forked
+ // child but never consumed β€” AI workers auto-enabled whenever the
+ // Claude CLI was detected. Only set when true so config.json/env can
+ // still opt in when the flag is absent.
+ if (ctx.flags.headless === true) {
+ config.aiWorkersEnabled = true;
+ }
const rawMaxCpu = ctx.flags['max-cpu-load'] as string | undefined;
const rawMinMem = ctx.flags['min-free-memory'] as string | undefined;
@@ -277,6 +290,7 @@ const startCommand: Command = {
status.config.ttlMs > 0
? `TTL: ${Math.round(status.config.ttlMs / 3600000)}h (self-shutdown)`
: `TTL: off (runs until stopped)`,
+ `AI Workers: ${status.config.aiWorkersEnabled ? 'enabled (budget-capped)' : 'off (local-only, default)'}`,
`Workers: ${status.config.workers.filter(w => w.enabled).length} enabled`,
`Max Concurrent: ${status.config.maxConcurrent}`,
`Max CPU Load: ${status.config.resourceThresholds.maxCpuLoad}`,
@@ -568,6 +582,23 @@ async function startBackgroundDaemon(projectRoot: string, quiet: boolean, forwar
output.printSuccess(`Daemon started in background (PID: ${pid})`);
output.printInfo(`Logs: ${logFile}`);
output.printInfo(`Stop with: claude-flow daemon stop`);
+
+ // #2661: worktree-fanout warning. Each Git worktree gets its own daemon
+ // (per-workspace scope, #1914), so `init --start-daemon` across N
+ // worktrees quietly accumulates N daemons. Surface the fleet size at
+ // start time so the accumulation is visible where it happens.
+ try {
+ const fleet = await scanRunningDaemons();
+ if (fleet.length > 1) {
+ output.writeln();
+ output.printWarning(
+ `Found ${fleet.length} ruflo daemons running across workspaces/worktrees.`
+ );
+ output.printInfo('Scheduled AI workers are off by default and every AI launch is capped by the user-global budget.');
+ output.printInfo('Inspect: ruflo daemon status --all');
+ output.printInfo('Stop all: ruflo daemon stop --all');
+ }
+ } catch { /* best-effort visibility β€” never fail the start */ }
}
return { success: true };
@@ -579,14 +610,26 @@ const stopCommand: Command = {
description: 'Stop the worker daemon and all background workers',
options: [
{ name: 'quiet', short: 'Q', type: 'boolean', description: 'Suppress output' },
+ // #2661: emergency stop for worktree-daemon fleets. Stops every ruflo
+ // daemon owned by the current user across ALL workspaces/worktrees.
+ { name: 'all', short: 'a', type: 'boolean', description: 'Stop ruflo daemons in ALL workspaces/worktrees (not just the current one)' },
],
examples: [
- { command: 'claude-flow daemon stop', description: 'Stop the daemon' },
+ { command: 'claude-flow daemon stop', description: 'Stop the daemon in this workspace' },
+ { command: 'claude-flow daemon stop --all', description: 'Stop ruflo daemons in every workspace/worktree' },
],
action: async (ctx: CommandContext): Promise<CommandResult> => {
const quiet = ctx.flags.quiet as boolean;
const projectRoot = process.cwd();
+ // #2661: `stop --all` β€” the containment lever for daemon fanout across
+ // Git worktrees. Only processes positively identified as ruflo daemons
+ // (via their self-identifying argv) are touched; each receives SIGTERM
+ // so its own shutdown path reaps in-flight Claude process groups.
+ if (ctx.flags.all as boolean) {
+ return stopAllDaemons(quiet);
+ }
+
try {
if (!quiet) {
const spinner = output.createSpinner({ text: 'Stopping worker daemon...', spinner: 'dots' });
@@ -616,6 +659,77 @@ const stopCommand: Command = {
},
};
+/**
+ * #2661: stop every running ruflo daemon across all workspaces/worktrees.
+ *
+ * Reuses the same positive identification as `daemon status --all`
+ * (scanRunningDaemons): a process is only touched when its command line is
+ * self-identifying as a ruflo daemon (`daemon start --foreground` +
+ * claude-flow markers). Interactive Claude sessions and non-ruflo processes
+ * are never candidates. Each daemon gets SIGTERM first β€” its own shutdown
+ * handler cancels in-flight headless Claude process groups and removes its
+ * PID file β€” with a SIGKILL fallback for daemons that don't exit within 2s.
+ * Only ruflo-owned registry entries (each workspace's daemon.pid) are removed.
+ */
+async function stopAllDaemons(quiet: boolean): Promise<CommandResult> {
+ // Stop any in-process daemon plus this workspace's tracked daemon first,
+ // matching plain `daemon stop` semantics for the current directory.
+ try { await stopDaemon(); } catch { /* not running in-process */ }
+ await killBackgroundDaemon(process.cwd());
+
+ const daemons = await scanRunningDaemons();
+ if (daemons.length === 0) {
+ if (!quiet) {
+ output.printInfo('No ruflo daemons are running in any workspace.');
+ }
+ return { success: true, data: { stopped: 0 } };
+ }
+
+ const isWin = process.platform === 'win32';
+ let stopped = 0;
+
+ for (const d of daemons) {
+ try {
+ if (isWin) {
+ const { execFileSync } = await import('child_process');
+ // /t terminates the daemon's child tree too (no /f: graceful first).
+ execFileSync('taskkill', ['/pid', String(d.pid), '/t'], { encoding: 'utf-8', timeout: 5000 });
+ } else {
+ process.kill(d.pid, 'SIGTERM');
+ }
+ stopped++;
+ if (!quiet) {
+ output.printInfo(`Stopping daemon PID ${d.pid}${d.workspace ? ` (${d.workspace})` : ''}`);
+ }
+ } catch { /* exited between scan and kill */ }
+ }
+
+ // Give SIGTERM handlers time to reap children and clean up, then
+ // force-kill anything still alive (POSIX; taskkill /t already recursed).
+ await new Promise((r) => setTimeout(r, 2000));
+ for (const d of daemons) {
+ if (!isWin && isProcessRunning(d.pid)) {
+ try { process.kill(d.pid, 'SIGKILL'); } catch { /* already dead */ }
+ }
+ // Remove the ruflo-owned PID file for that workspace β€” but only when it
+ // still points at the daemon we just stopped (never clobber a newer one).
+ if (d.workspace) {
+ try {
+ const pidFile = join(d.workspace, '.claude-flow', 'daemon.pid');
+ if (fs.existsSync(pidFile)) {
+ const filePid = parseInt(fs.readFileSync(pidFile, 'utf-8').trim(), 10);
+ if (filePid === d.pid) fs.unlinkSync(pidFile);
+ }
+ } catch { /* workspace removed or unreadable β€” nothing to clean */ }
+ }
+ }
+
+ if (!quiet) {
+ output.printSuccess(`Stopped ${stopped} ruflo daemon(s) across all workspaces.`);
+ }
+ return { success: true, data: { stopped } };
+}
+
/**
* Kill background daemon process using PID file
*/
@@ -925,6 +1039,36 @@ async function renderAllDaemonsStatus(): Promise<CommandResult> {
} else {
output.printInfo(`${daemons.length} daemon(s) running, all within their TTL.`);
}
+ if (daemons.length > 1) {
+ output.printInfo('Stop all daemons across workspaces with: ruflo daemon stop --all');
+ }
+
+ // #2661: user-global AI launch usage β€” the shared budget every daemon
+ // draws from, independent of worktree count.
+ try {
+ const { getGlobalAiBudget } = await import('../services/global-ai-budget.js');
+ const budget = getGlobalAiBudget();
+ const usage = budget.getUsage();
+ const limits = budget.getLimits();
+ output.writeln();
+ // #2661: per-workspace 24h launch attribution β€” which worktree is
+ // actually spending the shared budget.
+ const byWs = usage.byWorkspace.slice(0, 5).map(
+ (w) => ` ${w.launches}Γ— ${w.workspace}`
+ );
+ output.printBox(
+ [
+ `Launches (last hour): ${usage.lastHour}/${limits.maxLaunchesPerHour}`,
+ `Launches (last 24h): ${usage.lastDay}/${limits.maxLaunchesPerDay}`,
+ `Active Claude children: ${usage.active}/${limits.maxConcurrentGlobal}`,
+ usage.pausedUntil
+ ? output.warning(`PAUSED until ${new Date(usage.pausedUntil).toISOString()} (${usage.pauseReason ?? 'quota error'})`)
+ : `Circuit breaker: ${output.dim('closed (normal)')}`,
+ ...(byWs.length > 0 ? ['Launches by workspace (24h):', ...byWs] : []),
+ ].join('\n'),
+ 'Global AI Budget (all workspaces)'
+ );
+ } catch { /* budget ledger unavailable β€” skip the panel */ }
return { success: true, data: { daemons: rows.length } };
}
@@ -967,6 +1111,21 @@ const statusCommand: Command = {
const isRunning = status.running || bgRunning;
const displayPid = bgPid || status.pid;
+ // #2661: this CLI process constructs its own (default-config) daemon
+ // instance, so status.config.aiWorkersEnabled reflects THIS process,
+ // not the running background daemon. The background daemon persists
+ // its real consent state into daemon-state.json β€” prefer that when it
+ // is the one running.
+ let aiWorkersEnabled = status.config.aiWorkersEnabled;
+ if (bgRunning) {
+ try {
+ const st = JSON.parse(fs.readFileSync(join(projectRoot, '.claude-flow', 'daemon-state.json'), 'utf-8'));
+ if (typeof st?.config?.aiWorkersEnabled === 'boolean') {
+ aiWorkersEnabled = st.config.aiWorkersEnabled;
+ }
+ } catch { /* no/partial state β€” fall back to in-process config */ }
+ }
+
output.writeln();
// Daemon status box
@@ -982,6 +1141,9 @@ const statusCommand: Command = {
status.config.ttlMs > 0
? `TTL: ${Math.round(status.config.ttlMs / 3600000)}h (self-shutdown)`
: `TTL: ${output.dim('off (runs until stopped)')}`,
+ // #2661: surface the AI-consent gate so "why is audit local-only?"
+ // is answerable from `daemon status` alone.
+ `AI Workers: ${aiWorkersEnabled ? output.warning('enabled (budget-capped)') : output.dim('off (local-only, default)')}`,
`Workers Enabled: ${status.config.workers.filter(w => w.enabled).length}`,
`Max Concurrent: ${status.config.maxConcurrent}`,
`Max CPU Load: ${status.config.resourceThresholds.maxCpuLoad}`,
@@ -1099,7 +1261,13 @@ const triggerCommand: Command = {
}
try {
- const daemon = getDaemon(process.cwd());
+ // #2661: an explicit `trigger --headless` is user consent for AI
+ // execution of THIS run (still governed by the global AI budget).
+ // Without the flag, config.json / env opt-in still applies.
+ const daemon = getDaemon(
+ process.cwd(),
+ ctx.flags.headless === true ? { aiWorkersEnabled: true } : undefined
+ );
const spinner = output.createSpinner({ text: `Running ${workerType} worker...`, spinner: 'dots' });
spinner.start();
diff --git a/v3/@claude-flow/cli/src/services/ai-job-dedup.ts b/v3/@claude-flow/cli/src/services/ai-job-dedup.ts
new file mode 100644
index 0000000000..7acb02c8be
--- /dev/null
+++ b/v3/@claude-flow/cli/src/services/ai-job-dedup.ts
@@ -0,0 +1,157 @@
+/**
+ * #2661 β€” Cross-worktree AI job dedup (issue invariant 5).
+ *
+ * N worktrees of one repository checked out at the same HEAD schedule the
+ * same analyses independently: N audits of identical content, N optimize
+ * passes, N testgap sweeps. Before a model launch, callers compute
+ *
+ * jobKey = sha256(repositoryId, head, workerType, workerConfigHash)
+ *
+ * and skip the launch when the same job succeeded within the freshness
+ * window. HEAD moves β†’ new key β†’ the job runs again.
+ *
+ * The registry lives next to the AI budget ledger under the user's home
+ * directory (owner-only, symlink-rejecting) so all daemons share it:
+ *
+ * ~/.claude-flow/ai-jobs.json
+ *
+ * This is a best-effort OPTIMIZATION layered under the budget: two daemons
+ * racing the same key may both miss and both attempt a launch, but the
+ * budget's atomic reservation (maxConcurrentGlobal / hourly cap) is the hard
+ * invariant that bounds actual launches. Only operational metadata is
+ * persisted β€” never prompts, outputs, or source content.
+ */
+
+import * as fs from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+import { createHash } from 'crypto';
+
+export interface AiJobKeyParts {
+ repositoryId: string;
+ head: string;
+ workerType: string;
+ /** Hash of the effective worker config (prompt/model/sandbox/patterns). */
+ configHash: string;
+}
+
+interface JobRecord {
+ at: number;
+ workerType: string;
+ repositoryId: string;
+ workspace: string;
+}
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+export function computeAiJobKey(parts: AiJobKeyParts): string {
+ return createHash('sha256')
+ .update([parts.repositoryId, parts.head, parts.workerType, parts.configHash].join('\n'))
+ .digest('hex');
+}
+
+/** Stable hash of an arbitrary config object (key-sorted JSON). */
+export function hashWorkerConfig(config: unknown): string {
+ const canonical = JSON.stringify(config, (_k, v) => {
+ if (v && typeof v === 'object' && !Array.isArray(v)) {
+ return Object.fromEntries(Object.entries(v as Record<string, unknown>).sort(([a], [b]) => a.localeCompare(b)));
+ }
+ return v;
+ });
+ return createHash('sha256').update(canonical ?? 'null').digest('hex');
+}
+
+/** Invariant 9: registry files must never be symlinks. */
+function assertNotSymlink(path: string): void {
+ try {
+ const st = fs.lstatSync(path);
+ if (st.isSymbolicLink()) {
+ throw new Error(`AI job registry is a symlink (refusing): ${path}`);
+ }
+ } catch (e) {
+ if ((e as NodeJS.ErrnoException).code === 'ENOENT') return;
+ throw e;
+ }
+}
+
+export class AiJobDedupRegistry {
+ private readonly dir: string;
+ private readonly file: string;
+
+ constructor(options?: { baseDir?: string }) {
+ this.dir = options?.baseDir
+ ?? process.env.RUFLO_AI_BUDGET_DIR
+ ?? join(homedir(), '.claude-flow');
+ this.file = join(this.dir, 'ai-jobs.json');
+ }
+
+ /**
+ * True when the job succeeded within `freshnessMs`. Any registry error
+ * reads as "not fresh" β€” dedup failing open only costs a (budget-capped)
+ * launch, never correctness.
+ */
+ isFresh(jobKey: string, freshnessMs: number): { fresh: boolean; lastRunAt?: number } {
+ if (process.env.RUFLO_AI_DEDUP_DISABLE === '1') return { fresh: false };
+ try {
+ const records = this.read();
+ const rec = records[jobKey];
+ if (rec && Date.now() - rec.at < freshnessMs) {
+ return { fresh: true, lastRunAt: rec.at };
+ }
+ return { fresh: false, lastRunAt: rec?.at };
+ } catch {
+ return { fresh: false };
+ }
+ }
+
+ /** Record a successful run of a job. Best-effort. */
+ recordSuccess(jobKey: string, meta: { workerType: string; repositoryId: string; workspace: string }): void {
+ try {
+ const records = this.read();
+ records[jobKey] = { at: Date.now(), ...meta };
+ this.write(records);
+ } catch { /* dedup is an optimization β€” never block on it */ }
+ }
+
+ private read(): Record<string, JobRecord> {
+ assertNotSymlink(this.file);
+ if (!fs.existsSync(this.file)) return {};
+ const raw = JSON.parse(fs.readFileSync(this.file, 'utf-8'));
+ if (!raw || typeof raw !== 'object') return {};
+ // Prune anything older than 24h β€” freshness windows are far shorter,
+ // and HEAD churn would otherwise grow the file without bound.
+ const now = Date.now();
+ const out: Record<string, JobRecord> = {};
+ for (const [key, rec] of Object.entries(raw as Record<string, JobRecord>)) {
+ if (rec && typeof rec.at === 'number' && now - rec.at < DAY_MS) {
+ out[key] = rec;
+ }
+ }
+ return out;
+ }
+
+ private write(records: Record<string, JobRecord>): void {
+ if (!fs.existsSync(this.dir)) {
+ fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+ }
+ assertNotSymlink(this.file);
+ const tmp = `${this.file}.tmp.${process.pid}`;
+ fs.writeFileSync(tmp, JSON.stringify(records), { mode: 0o600 });
+ fs.renameSync(tmp, this.file);
+ }
+}
+
+// Singleton β€” one registry per process.
+let registryInstance: AiJobDedupRegistry | null = null;
+
+export function getAiJobDedupRegistry(): AiJobDedupRegistry {
+ if (!registryInstance) {
+ registryInstance = new AiJobDedupRegistry();
+ }
+ return registryInstance;
+}
+
+/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
+export function resetAiJobDedupRegistryForTests(): void {
+ registryInstance = null;
+}
diff --git a/v3/@claude-flow/cli/src/services/git-workspace-identity.ts b/v3/@claude-flow/cli/src/services/git-workspace-identity.ts
new file mode 100644
index 0000000000..16a40c2c38
--- /dev/null
+++ b/v3/@claude-flow/cli/src/services/git-workspace-identity.ts
@@ -0,0 +1,118 @@
+/**
+ * #2661 β€” Git workspace identity: separate WORKTREE identity from
+ * REPOSITORY identity.
+ *
+ * Daemon dedup, state, and scheduling have historically been keyed on the
+ * worktree path (`process.cwd()`), so N Git worktrees of the same repository
+ * behave as N unrelated projects β€” the cardinality bug behind the worktree
+ * daemon fanout. This service resolves the identity that is SHARED across
+ * worktrees:
+ *
+ * worktreeRoot `git rev-parse --show-toplevel` β€” per-worktree
+ * commonGitDir `git rev-parse --git-common-dir` β€” shared by all worktrees
+ * repositoryId sha256(canonical commonGitDir) β€” stable repo key
+ * head `git rev-parse HEAD` β€” current commit
+ *
+ * Two worktrees of one repository resolve to the SAME repositoryId (and,
+ * when checked out at the same commit, the same head) β€” the key ingredient
+ * for cross-worktree job dedup (issue invariant 5).
+ *
+ * Non-git directories degrade gracefully: repositoryId falls back to a hash
+ * of the resolved directory path (prefixed `dir:`-style via isGit=false), so
+ * callers never need a special case.
+ */
+
+import { execFileSync } from 'child_process';
+import { createHash } from 'crypto';
+import { resolve } from 'path';
+import * as fs from 'fs';
+
+export interface GitWorkspaceIdentity {
+ /** Absolute root of this worktree (or the input dir when not a git repo). */
+ worktreeRoot: string;
+ /** Absolute path of the shared .git directory (equals worktreeRoot/.git for non-worktree clones). */
+ commonGitDir: string;
+ /** Stable id shared by ALL worktrees of one repository. */
+ repositoryId: string;
+ /** Current HEAD commit sha ('' when not a git repo or unborn HEAD). */
+ head: string;
+ /** False when the directory is not inside a git repository. */
+ isGit: boolean;
+}
+
+const GIT_TIMEOUT_MS = 3000;
+
+function git(cwd: string, ...args: string[]): string | null {
+ try {
+ return execFileSync('git', args, {
+ cwd,
+ encoding: 'utf-8',
+ timeout: GIT_TIMEOUT_MS,
+ stdio: ['ignore', 'pipe', 'ignore'],
+ windowsHide: true,
+ }).trim();
+ } catch {
+ return null;
+ }
+}
+
+function sha256(input: string): string {
+ return createHash('sha256').update(input).digest('hex');
+}
+
+// Identity is stable for the life of a process (repo location doesn't move),
+// but HEAD is not β€” cache only the expensive, stable parts per directory.
+const identityCache = new Map<string, Omit<GitWorkspaceIdentity, 'head'>>();
+
+/**
+ * Resolve the git workspace identity for a directory. Never throws.
+ */
+export function resolveGitWorkspaceIdentity(dir: string): GitWorkspaceIdentity {
+ const resolved = resolve(dir);
+
+ let stable = identityCache.get(resolved);
+ if (!stable) {
+ const worktreeRoot = git(resolved, 'rev-parse', '--show-toplevel');
+ if (!worktreeRoot) {
+ stable = {
+ worktreeRoot: resolved,
+ commonGitDir: '',
+ // 'dir:' prefix keeps non-git ids from ever colliding with repo ids.
+ repositoryId: sha256(`dir:${canonicalPath(resolved)}`),
+ isGit: false,
+ };
+ } else {
+ // --git-common-dir may be relative to the worktree root (git < 2.31
+ // and some invocation contexts) β€” resolve against it.
+ const rawCommon = git(worktreeRoot, 'rev-parse', '--git-common-dir') ?? '.git';
+ const commonGitDir = resolve(worktreeRoot, rawCommon);
+ stable = {
+ worktreeRoot,
+ commonGitDir,
+ repositoryId: sha256(`git:${canonicalPath(commonGitDir)}`),
+ isGit: true,
+ };
+ }
+ identityCache.set(resolved, stable);
+ }
+
+ const head = stable.isGit ? (git(stable.worktreeRoot, 'rev-parse', 'HEAD') ?? '') : '';
+ return { ...stable, head };
+}
+
+/**
+ * Canonicalize a path so the same repository yields the same repositoryId
+ * regardless of symlinked prefixes (/tmp vs /private/tmp on macOS, etc.).
+ */
+function canonicalPath(p: string): string {
+ try {
+ return fs.realpathSync(p);
+ } catch {
+ return p;
+ }
+}
+
+/** Test hook: clear the per-process identity cache. */
+export function resetGitIdentityCacheForTests(): void {
+ identityCache.clear();
+}
diff --git a/v3/@claude-flow/cli/src/services/global-ai-budget.ts b/v3/@claude-flow/cli/src/services/global-ai-budget.ts
new file mode 100644
index 0000000000..2399f28676
--- /dev/null
+++ b/v3/@claude-flow/cli/src/services/global-ai-budget.ts
@@ -0,0 +1,415 @@
+/**
+ * #2661 β€” Global AI launch budget (emergency cost fuse).
+ *
+ * Every autonomous `claude --print` launch across ALL ruflo daemons in ALL
+ * worktrees/workspaces owned by the current user must pass through this
+ * user-global budget before a process is created. Without it, N worktree
+ * daemons each schedule their own AI workers and aggregate launch volume
+ * scales linearly with worktree count β€” enough to exhaust a user's Claude
+ * hourly quota silently (13 launches/hour/daemon under the legacy schedule).
+ *
+ * The ledger lives under the user's home directory (NOT the workspace) so
+ * daemons started from different worktrees of the same repository β€” or from
+ * unrelated repositories β€” all share one budget:
+ *
+ * ~/.claude-flow/ai-budget.json launch ledger + circuit breaker
+ * ~/.claude-flow/ai-budget.lock O_EXCL mutation lock
+ * ~/.claude-flow/ai-budget-receipts.jsonl launch/deny/pause receipts
+ *
+ * Files are owner-only (0700 dir / 0600 files) and symlinks are rejected
+ * (invariant 9 of #2661). All checks happen BEFORE process creation and the
+ * ledger mutation is atomic under the lock, so two daemons racing for the
+ * last hourly slot cannot both win.
+ *
+ * Default limits (issue #2661 containment):
+ * maxConcurrentGlobal 1 at most one autonomous claude child, user-wide
+ * maxLaunchesPerHour 2
+ * maxLaunchesPerDay 12
+ * pauseOnQuotaErrorMinutes 60 circuit breaker on 429/quota responses
+ */
+
+import * as fs from 'fs';
+import { join } from 'path';
+import { homedir } from 'os';
+
+export interface AiBudgetLimits {
+ maxConcurrentGlobal: number;
+ maxLaunchesPerHour: number;
+ maxLaunchesPerDay: number;
+ pauseOnQuotaErrorMinutes: number;
+}
+
+export const DEFAULT_AI_BUDGET_LIMITS: AiBudgetLimits = {
+ maxConcurrentGlobal: 1,
+ maxLaunchesPerHour: 2,
+ maxLaunchesPerDay: 12,
+ pauseOnQuotaErrorMinutes: 60,
+};
+
+export interface AiBudgetRequest {
+ workerType: string;
+ model: string;
+ /** Worktree/workspace root requesting the launch (recorded for receipts only). */
+ workspace: string;
+}
+
+export interface AiBudgetPermit {
+ allowed: boolean;
+ permitId?: string;
+ reason?: string;
+}
+
+interface LaunchRecord {
+ at: number;
+ pid: number;
+ workerType: string;
+ model: string;
+ workspace: string;
+}
+
+interface ActiveRecord {
+ permitId: string;
+ at: number;
+ pid: number;
+ workerType: string;
+}
+
+interface Ledger {
+ version: 1;
+ launches: LaunchRecord[];
+ active: ActiveRecord[];
+ pausedUntil?: number;
+ pauseReason?: string;
+}
+
+const HOUR_MS = 60 * 60 * 1000;
+const DAY_MS = 24 * HOUR_MS;
+// Active reservations older than this are treated as abandoned (crashed
+// daemon). Must exceed the daemon's 16-min worker timeout with margin.
+const ACTIVE_STALE_MS = 30 * 60 * 1000;
+// A mutation lock older than this belongs to a crashed process β€” take it over.
+const LOCK_STALE_MS = 10_000;
+const RECEIPTS_MAX_BYTES = 512 * 1024;
+const RECEIPTS_KEEP_LINES = 200;
+
+/**
+ * Heuristic match for Anthropic quota / rate-limit failures. Only ever
+ * applied to ERROR output of a FAILED launch (never to successful analysis
+ * output, which may legitimately discuss "rate limiting" in the user's code).
+ */
+export function isQuotaErrorText(text: string | undefined): boolean {
+ if (!text) return false;
+ return /\b429\b|rate[\s_-]?limit|usage[\s_-]?limit|quota|too many requests|overloaded_error/i.test(text);
+}
+
+function envPositiveInt(name: string): number | undefined {
+ const raw = process.env[name];
+ if (raw === undefined || raw.trim() === '') return undefined;
+ const n = Number.parseInt(raw, 10);
+ return Number.isFinite(n) && n >= 0 ? n : undefined;
+}
+
+function isProcessAlive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Invariant 9: registry files must never be symlinks. */
+function assertNotSymlink(path: string): void {
+ try {
+ const st = fs.lstatSync(path);
+ if (st.isSymbolicLink()) {
+ throw new Error(`AI budget file is a symlink (refusing): ${path}`);
+ }
+ } catch (e) {
+ if ((e as NodeJS.ErrnoException).code === 'ENOENT') return;
+ throw e;
+ }
+}
+
+function delay(ms: number): Promise<void> {
+ return new Promise((r) => setTimeout(r, ms));
+}
+
+export class GlobalAiBudget {
+ private readonly dir: string;
+ private readonly ledgerFile: string;
+ private readonly lockFile: string;
+ private readonly receiptsFile: string;
+ private readonly limits: AiBudgetLimits;
+
+ constructor(options?: { baseDir?: string; limits?: Partial<AiBudgetLimits> }) {
+ this.dir = options?.baseDir
+ ?? process.env.RUFLO_AI_BUDGET_DIR
+ ?? join(homedir(), '.claude-flow');
+ this.ledgerFile = join(this.dir, 'ai-budget.json');
+ this.lockFile = join(this.dir, 'ai-budget.lock');
+ this.receiptsFile = join(this.dir, 'ai-budget-receipts.jsonl');
+ this.limits = {
+ maxConcurrentGlobal:
+ options?.limits?.maxConcurrentGlobal
+ ?? envPositiveInt('RUFLO_AI_MAX_CONCURRENT')
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxConcurrentGlobal,
+ maxLaunchesPerHour:
+ options?.limits?.maxLaunchesPerHour
+ ?? envPositiveInt('RUFLO_AI_MAX_PER_HOUR')
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerHour,
+ maxLaunchesPerDay:
+ options?.limits?.maxLaunchesPerDay
+ ?? envPositiveInt('RUFLO_AI_MAX_PER_DAY')
+ ?? DEFAULT_AI_BUDGET_LIMITS.maxLaunchesPerDay,
+ pauseOnQuotaErrorMinutes:
+ options?.limits?.pauseOnQuotaErrorMinutes
+ ?? envPositiveInt('RUFLO_AI_QUOTA_PAUSE_MINUTES')
+ ?? DEFAULT_AI_BUDGET_LIMITS.pauseOnQuotaErrorMinutes,
+ };
+ }
+
+ getLimits(): AiBudgetLimits {
+ return { ...this.limits };
+ }
+
+ /**
+ * Atomically reserve one launch slot. Denials carry a machine-readable
+ * reason and are receipted. The reservation is counted as a launch
+ * immediately (the hourly/daily invariant is on launches, not completions);
+ * `release()` only frees the concurrency slot.
+ *
+ * Fails CLOSED: if the ledger cannot be read or locked, the launch is
+ * denied β€” an unaccountable launch is exactly what this fuse exists to
+ * prevent. `RUFLO_AI_BUDGET_DISABLE=1` is the explicit escape hatch.
+ */
+ async reserve(req: AiBudgetRequest): Promise<AiBudgetPermit> {
+ if (process.env.RUFLO_AI_BUDGET_DISABLE === '1') {
+ return { allowed: true, permitId: `bypass_${Date.now()}_${process.pid}` };
+ }
+
+ let unlock: (() => void) | null = null;
+ try {
+ unlock = await this.acquireLock();
+ const now = Date.now();
+ const ledger = this.readLedger(now);
+
+ let reason: string | null = null;
+ if (ledger.pausedUntil && ledger.pausedUntil > now) {
+ reason = `circuit-open until ${new Date(ledger.pausedUntil).toISOString()} (${ledger.pauseReason ?? 'quota error'})`;
+ } else if (ledger.active.length >= this.limits.maxConcurrentGlobal) {
+ reason = `global-concurrency (${ledger.active.length}/${this.limits.maxConcurrentGlobal} active)`;
+ } else {
+ const lastHour = ledger.launches.filter((l) => now - l.at < HOUR_MS).length;
+ const lastDay = ledger.launches.length; // already pruned to 24h
+ if (lastHour >= this.limits.maxLaunchesPerHour) {
+ reason = `hourly-budget (${lastHour}/${this.limits.maxLaunchesPerHour} in last hour)`;
+ } else if (lastDay >= this.limits.maxLaunchesPerDay) {
+ reason = `daily-budget (${lastDay}/${this.limits.maxLaunchesPerDay} in last 24h)`;
+ }
+ }
+
+ if (reason) {
+ this.appendReceipt({ event: 'deny', at: now, reason, ...req });
+ return { allowed: false, reason };
+ }
+
+ const permitId = `permit_${now}_${process.pid}_${Math.random().toString(36).slice(2, 8)}`;
+ ledger.launches.push({ at: now, pid: process.pid, workerType: req.workerType, model: req.model, workspace: req.workspace });
+ ledger.active.push({ permitId, at: now, pid: process.pid, workerType: req.workerType });
+ this.writeLedger(ledger);
+ this.appendReceipt({ event: 'launch', at: now, permitId, ...req });
+ return { allowed: true, permitId };
+ } catch (e) {
+ // Fail closed β€” see docstring.
+ const reason = `budget-ledger-error: ${e instanceof Error ? e.message : String(e)}`;
+ try { this.appendReceipt({ event: 'deny', at: Date.now(), reason, ...req }); } catch { /* best-effort */ }
+ return { allowed: false, reason };
+ } finally {
+ unlock?.();
+ }
+ }
+
+ /** Free the concurrency slot held by a permit. Best-effort. */
+ async release(permitId: string | undefined): Promise<void> {
+ if (!permitId || permitId.startsWith('bypass_')) return;
+ let unlock: (() => void) | null = null;
+ try {
+ unlock = await this.acquireLock();
+ const ledger = this.readLedger(Date.now());
+ const before = ledger.active.length;
+ ledger.active = ledger.active.filter((a) => a.permitId !== permitId);
+ if (ledger.active.length !== before) this.writeLedger(ledger);
+ } catch {
+ // Abandoned reservations expire via ACTIVE_STALE_MS pruning.
+ } finally {
+ unlock?.();
+ }
+ }
+
+ /**
+ * Open the user-global circuit breaker: a quota/429 response from ANY
+ * daemon pauses ALL autonomous Claude launches for the cooldown window.
+ */
+ async recordQuotaError(detail: string): Promise<void> {
+ let unlock: (() => void) | null = null;
+ try {
+ unlock = await this.acquireLock();
+ const now = Date.now();
+ const ledger = this.readLedger(now);
+ ledger.pausedUntil = now + this.limits.pauseOnQuotaErrorMinutes * 60 * 1000;
+ ledger.pauseReason = detail.slice(0, 200);
+ this.writeLedger(ledger);
+ this.appendReceipt({ event: 'quota-pause', at: now, until: ledger.pausedUntil, detail: ledger.pauseReason });
+ } catch {
+ // best-effort β€” the hourly budget still bounds retries
+ } finally {
+ unlock?.();
+ }
+ }
+
+ /** Snapshot for `daemon status` / diagnostics. */
+ getUsage(): {
+ lastHour: number;
+ lastDay: number;
+ active: number;
+ pausedUntil?: number;
+ pauseReason?: string;
+ /** #2661 β€” 24h launch counts per worktree/workspace, most active first. */
+ byWorkspace: Array<{ workspace: string; launches: number }>;
+ } {
+ try {
+ const now = Date.now();
+ const ledger = this.readLedger(now);
+ const byWs = new Map<string, number>();
+ for (const l of ledger.launches) {
+ byWs.set(l.workspace, (byWs.get(l.workspace) ?? 0) + 1);
+ }
+ return {
+ lastHour: ledger.launches.filter((l) => now - l.at < HOUR_MS).length,
+ lastDay: ledger.launches.length,
+ active: ledger.active.length,
+ pausedUntil: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pausedUntil : undefined,
+ pauseReason: ledger.pausedUntil && ledger.pausedUntil > now ? ledger.pauseReason : undefined,
+ byWorkspace: Array.from(byWs.entries())
+ .map(([workspace, launches]) => ({ workspace, launches }))
+ .sort((a, b) => b.launches - a.launches),
+ };
+ } catch {
+ return { lastHour: 0, lastDay: 0, active: 0, byWorkspace: [] };
+ }
+ }
+
+ // -- internals ----------------------------------------------------------
+
+ private ensureDir(): void {
+ if (!fs.existsSync(this.dir)) {
+ fs.mkdirSync(this.dir, { recursive: true, mode: 0o700 });
+ }
+ }
+
+ private async acquireLock(): Promise<() => void> {
+ this.ensureDir();
+ const deadline = Date.now() + 2000;
+ for (;;) {
+ try {
+ const fd = fs.openSync(this.lockFile, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600);
+ fs.writeSync(fd, String(process.pid));
+ fs.closeSync(fd);
+ return () => {
+ try { fs.unlinkSync(this.lockFile); } catch { /* already gone */ }
+ };
+ } catch (e) {
+ if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e;
+ // Stale lock from a crashed process β€” take over.
+ try {
+ const st = fs.lstatSync(this.lockFile);
+ if (Date.now() - st.mtimeMs > LOCK_STALE_MS) {
+ fs.unlinkSync(this.lockFile);
+ continue;
+ }
+ } catch { /* raced β€” retry */ }
+ if (Date.now() > deadline) {
+ throw new Error('timed out acquiring ai-budget lock');
+ }
+ await delay(25);
+ }
+ }
+ }
+
+ /** Read + prune the ledger. Caller must hold the lock for read-modify-write. */
+ private readLedger(now: number): Ledger {
+ assertNotSymlink(this.ledgerFile);
+ let ledger: Ledger = { version: 1, launches: [], active: [] };
+ if (fs.existsSync(this.ledgerFile)) {
+ try {
+ const raw = JSON.parse(fs.readFileSync(this.ledgerFile, 'utf-8'));
+ if (raw && typeof raw === 'object') {
+ ledger = {
+ version: 1,
+ launches: Array.isArray(raw.launches) ? raw.launches.filter((l: LaunchRecord) => typeof l?.at === 'number') : [],
+ active: Array.isArray(raw.active) ? raw.active.filter((a: ActiveRecord) => typeof a?.at === 'number') : [],
+ pausedUntil: typeof raw.pausedUntil === 'number' ? raw.pausedUntil : undefined,
+ pauseReason: typeof raw.pauseReason === 'string' ? raw.pauseReason : undefined,
+ };
+ }
+ } catch {
+ // Corrupt ledger: start fresh rather than blocking forever. The next
+ // write re-establishes it; worst case one over-budget launch.
+ }
+ }
+ ledger.launches = ledger.launches.filter((l) => now - l.at < DAY_MS);
+ // Drop reservations whose process died (tolerating PID reuse via the
+ // staleness cutoff) or that outlived any plausible worker run.
+ ledger.active = ledger.active.filter(
+ (a) => now - a.at < ACTIVE_STALE_MS && isProcessAlive(a.pid)
+ );
+ if (ledger.pausedUntil && ledger.pausedUntil <= now) {
+ ledger.pausedUntil = undefined;
+ ledger.pauseReason = undefined;
+ }
+ return ledger;
+ }
+
+ private writeLedger(ledger: Ledger): void {
+ this.ensureDir();
+ assertNotSymlink(this.ledgerFile);
+ const tmp = `${this.ledgerFile}.tmp`;
+ fs.writeFileSync(tmp, JSON.stringify(ledger), { mode: 0o600 });
+ fs.renameSync(tmp, this.ledgerFile);
+ }
+
+ /**
+ * Invariant 10: every launch, denial, and pause emits a receipt. Only
+ * operational metadata is persisted β€” never prompts or source content.
+ */
+ private appendReceipt(receipt: Record<string, unknown>): void {
+ try {
+ this.ensureDir();
+ assertNotSymlink(this.receiptsFile);
+ fs.appendFileSync(this.receiptsFile, JSON.stringify(receipt) + '\n', { mode: 0o600 });
+ const st = fs.statSync(this.receiptsFile);
+ if (st.size > RECEIPTS_MAX_BYTES) {
+ const lines = fs.readFileSync(this.receiptsFile, 'utf-8').split('\n');
+ fs.writeFileSync(this.receiptsFile, lines.slice(-RECEIPTS_KEEP_LINES).join('\n'), { mode: 0o600 });
+ }
+ } catch {
+ // Receipts are best-effort; never block a decision on them.
+ }
+ }
+}
+
+// Singleton β€” one budget per process, shared by all executors.
+let budgetInstance: GlobalAiBudget | null = null;
+
+export function getGlobalAiBudget(): GlobalAiBudget {
+ if (!budgetInstance) {
+ budgetInstance = new GlobalAiBudget();
+ }
+ return budgetInstance;
+}
+
+/** Test hook: reset the singleton (e.g. after changing RUFLO_AI_BUDGET_DIR). */
+export function resetGlobalAiBudgetForTests(): void {
+ budgetInstance = null;
+}
diff --git a/v3/@claude-flow/cli/src/services/headless-worker-executor.ts b/v3/@claude-flow/cli/src/services/headless-worker-executor.ts
index 1e406dfe54..8e406bf2a6 100644
--- a/v3/@claude-flow/cli/src/services/headless-worker-executor.ts
+++ b/v3/@claude-flow/cli/src/services/headless-worker-executor.ts
@@ -24,6 +24,9 @@ import { EventEmitter } from 'events';
import { existsSync, readFileSync, readdirSync, mkdirSync, writeFileSync } from 'fs';
import { join, relative } from 'path';
import type { WorkerType } from './worker-daemon.js';
+import { getGlobalAiBudget, isQuotaErrorText } from './global-ai-budget.js';
+import { resolveGitWorkspaceIdentity } from './git-workspace-identity.js';
+import { getAiJobDedupRegistry, computeAiJobKey, hashWorkerConfig } from './ai-job-dedup.js';
// ============================================
// Type Definitions
@@ -186,6 +189,14 @@ export interface HeadlessExecutionResult {
/** Execution ID for tracking */
executionId: string;
+
+ /**
+ * #2661 β€” true when the launch was skipped because the same job
+ * (repositoryId + HEAD + worker + config) succeeded within the freshness
+ * window, e.g. in a sibling worktree. No model call happened; consumers
+ * must not overwrite persisted metrics with this result.
+ */
+ dedupSkipped?: boolean;
}
/**
@@ -774,6 +785,19 @@ export class HeadlessWorkerExecutor extends EventEmitter {
return this.processPool.size;
}
+ /**
+ * #2661 β€” signal a pool entry's whole process group, not just the head.
+ * Children are spawned `detached: true` on POSIX precisely so their MCP
+ * bridge grandchildren can be reaped with `kill(-pid)`; a head-only kill
+ * orphans them (#2098B).
+ */
+ private killEntryTree(proc: ChildProcess, signal: NodeJS.Signals): void {
+ if (process.platform !== 'win32' && typeof proc.pid === 'number') {
+ try { process.kill(-proc.pid, signal); return; } catch { /* fall through */ }
+ }
+ try { proc.kill(signal); } catch { /* already dead */ }
+ }
+
/**
* Cancel a running execution
*/
@@ -784,7 +808,7 @@ export class HeadlessWorkerExecutor extends EventEmitter {
}
clearTimeout(entry.timeout);
- entry.process.kill('SIGTERM');
+ this.killEntryTree(entry.process, 'SIGTERM');
this.processPool.delete(executionId);
this.emit('cancelled', { executionId });
@@ -804,10 +828,10 @@ export class HeadlessWorkerExecutor extends EventEmitter {
const entries = Array.from(this.processPool.entries());
for (const [executionId, entry] of entries) {
clearTimeout(entry.timeout);
- entry.process.kill('SIGTERM');
+ this.killEntryTree(entry.process, 'SIGTERM');
// SIGKILL fallback after 5s to prevent orphan processes (#1395 Bug 6)
setTimeout(() => {
- try { if (!entry.process.killed) entry.process.kill('SIGKILL'); } catch { /* already dead */ }
+ try { if (!entry.process.killed) this.killEntryTree(entry.process, 'SIGKILL'); } catch { /* already dead */ }
}, 5000).unref();
this.emit('cancelled', { executionId });
cancelled++;
@@ -883,6 +907,72 @@ export class HeadlessWorkerExecutor extends EventEmitter {
const startTime = Date.now();
const executionId = `${workerType}_${startTime}_${Math.random().toString(36).slice(2, 8)}`;
+ // #2661 invariant 5 β€” cross-worktree job dedup. Worktrees of one
+ // repository at the same HEAD would otherwise run identical analyses
+ // once per worktree. jobKey = sha256(repositoryId, HEAD, worker,
+ // configHash); a success within the freshness window (the worker's own
+ // interval, floor 10 min) skips the launch entirely β€” no budget spend,
+ // no process. HEAD moves β†’ new key β†’ the job runs again.
+ const identity = resolveGitWorkspaceIdentity(this.projectRoot);
+ const jobKey = computeAiJobKey({
+ repositoryId: identity.repositoryId,
+ head: identity.head,
+ workerType,
+ configHash: hashWorkerConfig(headless),
+ });
+ const dedup = getAiJobDedupRegistry();
+ const envWindowSecs = Number.parseInt(process.env.RUFLO_AI_DEDUP_WINDOW_SECS || '', 10);
+ const freshnessMs = Number.isFinite(envWindowSecs) && envWindowSecs >= 0
+ ? envWindowSecs * 1000
+ : Math.max(baseConfig.intervalMs || 0, 10 * 60 * 1000);
+ const freshness = dedup.isFresh(jobKey, freshnessMs);
+ if (freshness.fresh) {
+ const skipped: HeadlessExecutionResult = {
+ success: true,
+ dedupSkipped: true,
+ output: '',
+ parsedOutput: undefined,
+ durationMs: 0,
+ model: 'none',
+ sandboxMode: headless.sandbox,
+ workerType,
+ timestamp: new Date(),
+ executionId,
+ };
+ this.logExecution(
+ executionId,
+ 'result',
+ `dedup-skip: job ${jobKey.slice(0, 12)} succeeded ${Math.round((Date.now() - (freshness.lastRunAt ?? Date.now())) / 1000)}s ago (repo ${identity.repositoryId.slice(0, 12)}, head ${identity.head.slice(0, 12) || 'n/a'})`
+ );
+ this.emit('dedup:skipped', { executionId, workerType, jobKey, lastRunAt: freshness.lastRunAt });
+ this.processQueue();
+ return skipped;
+ }
+
+ // #2661 β€” every autonomous launch must reserve a slot in the USER-GLOBAL
+ // AI budget before any process is created. This is the hard invariant
+ // that bounds aggregate launches across all worktree daemons: per-daemon
+ // maxConcurrent limits multiply with worktree count, the global budget
+ // does not. Denials return an error result (with a receipted reason)
+ // instead of queueing, so denied work never piles up into a retry storm.
+ const budget = getGlobalAiBudget();
+ const model = headless.model || 'sonnet';
+ const permit = await budget.reserve({ workerType, model, workspace: this.projectRoot });
+ if (!permit.allowed) {
+ const denied = this.createErrorResult(
+ workerType,
+ `Denied by global AI budget: ${permit.reason}`
+ );
+ denied.executionId = executionId;
+ this.logExecution(executionId, 'error', `budget-denied: ${permit.reason}`);
+ // NOTE: deliberately no `emit('error', ...)` here β€” Node treats
+ // unlistened 'error' events as throws, and callers consume the
+ // returned error result; `budget:denied` is the observable signal.
+ this.emit('budget:denied', { executionId, workerType, reason: permit.reason });
+ this.processQueue();
+ return denied;
+ }
+
this.emit('start', { executionId, workerType, config: headless });
try {
@@ -929,6 +1019,25 @@ export class HeadlessWorkerExecutor extends EventEmitter {
// Log result
this.logExecution(executionId, 'result', JSON.stringify(executionResult, null, 2));
+ // #2661 invariant 5 β€” record the success so sibling worktrees at the
+ // same HEAD skip this job for the rest of the freshness window.
+ if (result.success) {
+ dedup.recordSuccess(jobKey, {
+ workerType,
+ repositoryId: identity.repositoryId,
+ workspace: this.projectRoot,
+ });
+ }
+
+ // #2661 β€” a quota/429/usage-limit failure opens the user-global
+ // circuit breaker so EVERY daemon stops launching for the cooldown
+ // window instead of retrying into an exhausted quota. Only inspected
+ // on failure: successful analysis output may legitimately discuss
+ // rate limiting in the user's own code.
+ if (!result.success && isQuotaErrorText(result.error)) {
+ await budget.recordQuotaError(`${workerType}: ${(result.error ?? '').slice(0, 200)}`);
+ }
+
this.emit('complete', executionResult);
return executionResult;
} catch (error) {
@@ -938,10 +1047,15 @@ export class HeadlessWorkerExecutor extends EventEmitter {
executionResult.durationMs = Date.now() - startTime;
this.logExecution(executionId, 'error', errorMessage);
+ if (isQuotaErrorText(errorMessage)) {
+ await budget.recordQuotaError(`${workerType}: ${errorMessage.slice(0, 200)}`);
+ }
this.emit('error', executionResult);
return executionResult;
} finally {
+ // #2661 β€” free the global concurrency slot (launch counts persist).
+ await budget.release(permit.permitId);
// Process next in queue
this.processQueue();
}
diff --git a/v3/@claude-flow/cli/src/services/worker-daemon.ts b/v3/@claude-flow/cli/src/services/worker-daemon.ts
index 84849e5d38..5966a26c33 100644
--- a/v3/@claude-flow/cli/src/services/worker-daemon.ts
+++ b/v3/@claude-flow/cli/src/services/worker-daemon.ts
@@ -107,6 +107,14 @@ export interface DaemonConfig {
// shutdown if no worker has run within this window (0 disables).
ttlMs: number;
idleShutdownMs: number;
+ // #2661 β€” explicit consent gate for scheduled AI workers. When false
+ // (the default), NO worker is ever promoted to headless `claude --print`
+ // execution, regardless of whether the Claude CLI is on PATH. Merely
+ // finding `claude` must not authorize recurring model calls: a default
+ // install produces zero autonomous Claude launches. Enable via
+ // `daemon start --headless`, `daemon.aiWorkers.enabled: true` in
+ // .claude-flow/config.json, or RUFLO_DAEMON_AI_WORKERS=1.
+ aiWorkersEnabled: boolean;
workers: WorkerConfig[];
}
@@ -261,6 +269,12 @@ export class WorkerDaemon extends EventEmitter {
// env-or-default and honors an explicit 0 (disable).
ttlMs: config?.ttlMs ?? fileConfig.ttlMs ?? readEnvSecsAsMs('RUFLO_DAEMON_TTL_SECS', DEFAULT_DAEMON_TTL_MS),
idleShutdownMs: config?.idleShutdownMs ?? fileConfig.idleShutdownMs ?? readEnvSecsAsMs('RUFLO_DAEMON_IDLE_SECS', DEFAULT_DAEMON_IDLE_SHUTDOWN_MS),
+ // #2661 β€” AI workers are opt-in: flag > config.json > env > OFF.
+ // Deliberately NOT restored from daemon-state.json (initializeWorkerStates
+ // whitelist) so a stale state file can never resurrect consent.
+ aiWorkersEnabled: config?.aiWorkersEnabled
+ ?? fileConfig.aiWorkersEnabled
+ ?? (process.env.RUFLO_DAEMON_AI_WORKERS === '1'),
workers: config?.workers ?? DEFAULT_WORKERS,
};
@@ -295,6 +309,17 @@ export class WorkerDaemon extends EventEmitter {
* Initialize headless executor if Claude Code is available
*/
private async initHeadlessExecutor(): Promise<void> {
+ // #2661 β€” scheduled AI workers require explicit consent. Without it,
+ // don't even probe `claude --version`: headlessAvailable stays false,
+ // every worker runs its $0 local path, and a default install produces
+ // zero autonomous Claude launches regardless of worktree count.
+ if (!this.config.aiWorkersEnabled) {
+ this.log(
+ 'info',
+ 'AI workers disabled (default) - all workers run local-only. Enable with `daemon start --headless`, daemon.aiWorkers.enabled=true, or RUFLO_DAEMON_AI_WORKERS=1 (#2661)'
+ );
+ return;
+ }
try {
this.headlessExecutor = new HeadlessWorkerExecutor(this.projectRoot, {
maxConcurrent: this.config.maxConcurrent,
@@ -415,6 +440,7 @@ export class WorkerDaemon extends EventEmitter {
minFreeMemoryPercent?: number;
ttlMs?: number;
idleShutdownMs?: number;
+ aiWorkersEnabled?: boolean;
} {
const jsonPath = join(claudeFlowDir, 'config.json');
const yamlPath = join(claudeFlowDir, 'config.yaml');
@@ -470,6 +496,8 @@ export class WorkerDaemon extends EventEmitter {
// and env var; stored internally as ms. An explicit 0 disables.
const rawTtl = cfg['daemon.ttlSecs'] ?? raw['daemon.ttlSecs'];
const rawIdle = cfg['daemon.idleSecs'] ?? raw['daemon.idleSecs'];
+ // #2661 β€” explicit opt-in for scheduled AI workers.
+ const rawAiEnabled = cfg['daemon.aiWorkers.enabled'] ?? raw['daemon.aiWorkers.enabled'];
return {
autoStart: typeof raw['daemon.autoStart'] === 'boolean' ? raw['daemon.autoStart'] : undefined,
maxConcurrent: (typeof rawMaxConcurrent === 'number' && rawMaxConcurrent > 0) ? rawMaxConcurrent : undefined,
@@ -478,6 +506,7 @@ export class WorkerDaemon extends EventEmitter {
minFreeMemoryPercent: (typeof rawMinMem === 'number' && rawMinMem >= 0 && rawMinMem <= 100) ? rawMinMem : undefined,
ttlMs: (typeof rawTtl === 'number' && rawTtl >= 0) ? rawTtl * 1000 : undefined,
idleShutdownMs: (typeof rawIdle === 'number' && rawIdle >= 0) ? rawIdle * 1000 : undefined,
+ aiWorkersEnabled: typeof rawAiEnabled === 'boolean' ? rawAiEnabled : undefined,
};
} catch {
return {};
@@ -965,6 +994,14 @@ export class WorkerDaemon extends EventEmitter {
this.lifecycleTimer = undefined;
}
+ // #2661 β€” reap in-flight headless `claude --print` children. They run
+ // detached (own process group on POSIX) and would otherwise outlive the
+ // daemon; `daemon stop --all` relies on SIGTERM β†’ this path to cancel
+ // active Claude process groups.
+ if (this.headlessExecutor) {
+ try { this.headlessExecutor.cancelAll(); } catch { /* best-effort */ }
+ }
+
this.running = false;
this.removePidFile();
this.saveState();
@@ -985,37 +1022,61 @@ export class WorkerDaemon extends EventEmitter {
private startLifecycleMonitor(): void {
const ttlMs = this.config.ttlMs;
const idleMs = this.config.idleShutdownMs;
- if ((!ttlMs || ttlMs <= 0) && (!idleMs || idleMs <= 0)) {
- return; // both limits disabled β€” preserve legacy run-until-stopped behavior
- }
+ // #2661 β€” unlike ttl/idle (both optional), the workspace-removal check
+ // always runs, so the monitor is no longer skipped when both limits are
+ // disabled. A daemon whose worktree was deleted must not keep running.
const CHECK_INTERVAL_MS = 60_000;
this.lifecycleTimer = setInterval(() => {
if (!this.running) return;
- const now = Date.now();
- const startedMs = this.startedAt?.getTime() ?? now;
-
- if (ttlMs > 0 && now - startedMs >= ttlMs) {
- void this.selfShutdown(`max age ${Math.round(ttlMs / 1000)}s reached`);
- return;
- }
- if (idleMs > 0) {
- const lastActivity = this.lastWorkerActivityMs() ?? startedMs;
- if (now - lastActivity >= idleMs) {
- void this.selfShutdown(`idle for ${Math.round(idleMs / 1000)}s (no worker activity)`);
- }
+ const reason = this.lifecycleShutdownReason(Date.now());
+ if (reason) {
+ void this.selfShutdown(reason);
}
}, CHECK_INTERVAL_MS);
if (typeof this.lifecycleTimer.unref === 'function') {
this.lifecycleTimer.unref();
}
- const parts: string[] = [];
+ const parts: string[] = ['workspace-removal'];
if (ttlMs > 0) parts.push(`ttl=${Math.round(ttlMs / 1000)}s`);
if (idleMs > 0) parts.push(`idle=${Math.round(idleMs / 1000)}s`);
this.log('info', `Lifecycle monitor active (${parts.join(', ')})`);
}
+ /**
+ * Decide whether the daemon should self-shutdown, and why. Extracted from
+ * the lifecycle timer so it is testable without racing a 60s interval or
+ * calling process.exit().
+ *
+ * #2661 (invariant 6, containment form): a removed worktree makes its
+ * daemon ineligible within one check interval β€” the daemon detects that
+ * its workspace directory is gone and shuts down instead of continuing to
+ * schedule jobs against a deleted tree. The full lease architecture
+ * (supervisor-dispatched jobs, heartbeats) is follow-up work; this stops
+ * the leak where recreated/removed worktrees leave schedulers behind.
+ */
+ private lifecycleShutdownReason(now: number): string | null {
+ if (!existsSync(this.projectRoot)) {
+ return 'workspace directory removed (#2661)';
+ }
+
+ const ttlMs = this.config.ttlMs;
+ const idleMs = this.config.idleShutdownMs;
+ const startedMs = this.startedAt?.getTime() ?? now;
+
+ if (ttlMs > 0 && now - startedMs >= ttlMs) {
+ return `max age ${Math.round(ttlMs / 1000)}s reached`;
+ }
+ if (idleMs > 0) {
+ const lastActivity = this.lastWorkerActivityMs() ?? startedMs;
+ if (now - lastActivity >= idleMs) {
+ return `idle for ${Math.round(idleMs / 1000)}s (no worker activity)`;
+ }
+ }
+ return null;
+ }
+
/**
* Most recent worker start/finish time across all workers (epoch ms), or
* null if no worker has ever started. Used for idle-shutdown detection.
@@ -1256,8 +1317,11 @@ export class WorkerDaemon extends EventEmitter {
* Run the actual worker logic
*/
private async runWorkerLogic(workerConfig: WorkerConfig): Promise<unknown> {
- // Check if this is a headless worker type and headless execution is available
- if (isHeadlessWorker(workerConfig.type) && this.headlessAvailable && this.headlessExecutor) {
+ // Check if this is a headless worker type and headless execution is available.
+ // #2661 β€” aiWorkersEnabled is re-checked here (not just at init) as
+ // defence in depth: no code path may promote a worker to `claude --print`
+ // without explicit consent.
+ if (this.config.aiWorkersEnabled && isHeadlessWorker(workerConfig.type) && this.headlessAvailable && this.headlessExecutor) {
try {
this.log('info', `Running ${workerConfig.type} in headless mode (Claude Code AI)`);
const result = await this.headlessExecutor.execute(workerConfig.type as HeadlessWorkerType);
@@ -1281,6 +1345,17 @@ export class WorkerDaemon extends EventEmitter {
error: String(reason).slice(0, 500),
});
// Fall through to local switch.
+ } else if (result.dedupSkipped) {
+ // #2661 invariant 5 β€” the same job (repositoryId + HEAD + worker +
+ // config) already succeeded within the freshness window, e.g. in a
+ // sibling worktree. No model call happened; do NOT overwrite the
+ // persisted metrics (which hold the real prior result) and do NOT
+ // fall back to local β€” the work is already done.
+ this.log('info', `Worker ${workerConfig.type} dedup-skipped (same repo+HEAD job ran recently in another worktree)`);
+ return {
+ mode: 'headless-dedup-skip',
+ ...result,
+ };
} else {
// #1793: persist the headless result to the same metrics files the
// local workers write to. Without this, AI-mode runs produced rich