Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: stop .agent-done sentinels lingering in agent worktrees and blocking cleanup

Adam Eivy3f229fdd8ecd
diff --git a/server/services/agentCompletionCleanup.js b/server/services/agentCompletionCleanup.js
index 0933c88161..8fbba05a57 100644
--- a/server/services/agentCompletionCleanup.js
+++ b/server/services/agentCompletionCleanup.js
@@ -370,7 +370,9 @@ async function runCompletionCleanupSteps(context, { onStepError }) {
// spawners run this from a `finally`, so they reach here on exactly that
// path. An unreadable record counts as not-over: leaving the file for the
// stale sweep is the recoverable direction, destroying a deliverable is not.
- if (context.agentState?.status === 'completed') {
+ // `failed` is an outcome too: left behind in a preserved worktree, the sentinel
+ // reads as uncommitted work and blocks every later merged-worktree cleanup.
+ if (['completed', 'failed'].includes(context.agentState?.status)) {
await removeCompletionSentinel(context)
.catch(err => emitLog('warn', `Completion sentinel removal failed for ${context.agentId}: ${err.message}`, { agentId: context.agentId }));
}
diff --git a/server/services/agentCompletionCleanup.test.js b/server/services/agentCompletionCleanup.test.js
index b96b8f13ac..7e8a42dbf4 100644
--- a/server/services/agentCompletionCleanup.test.js
+++ b/server/services/agentCompletionCleanup.test.js
@@ -861,9 +861,9 @@ describe('completion sentinel survives a finalize that recorded no outcome', ()
expect(await readFile(join(workspace, '.agent-done-a1'), 'utf8')).toContain('proposal');
});
- it('removes it once the record reached an outcome', async () => {
+ it.each(['completed', 'failed'])('removes it once the record reached a %s outcome', async (status) => {
await writeFile(join(workspace, '.agent-done-a1'), '{"summary":"x","payload":{"title":"proposal"}}');
- await cleanupWithStatus('completed');
+ await cleanupWithStatus(status);
await expect(readFile(join(workspace, '.agent-done-a1'))).rejects.toMatchObject({ code: 'ENOENT' });
});
});
diff --git a/server/services/agentSentinelSweep.js b/server/services/agentSentinelSweep.js
index ddab5690c2..ed842fb943 100644
--- a/server/services/agentSentinelSweep.js
+++ b/server/services/agentSentinelSweep.js
@@ -35,6 +35,7 @@ import { readdir, stat } from 'fs/promises';
import { join } from 'path';
import { doneSentinelAgentId } from '../lib/agentSentinel.js';
+import { execGit } from '../lib/execGit.js';
import { HOUR, rmGuarded } from '../lib/fileUtils.js';
/** How long a sentinel must have sat untouched before it counts as abandoned. */
@@ -79,3 +80,24 @@ export async function sweepStaleDoneSentinels(dirs, protectedAgentIds) {
return removed;
}
+
+/**
+ * Every checkout of the given repos: the repo itself plus its linked worktrees.
+ * A sentinel stranded in a preserved agent worktree reads as uncommitted work, so
+ * it blocks that worktree's merged-tree cleanup until something deletes it; the
+ * sweep above only helps if it is pointed at those directories.
+ *
+ * @param {string[]} repoPaths
+ * @returns {Promise<string[]>} unique checkout paths; an unreadable repo contributes itself only
+ */
+export async function checkoutDirsOf(repoPaths) {
+ const dirs = new Set();
+ for (const repo of new Set((repoPaths || []).filter(p => typeof p === 'string' && p))) {
+ dirs.add(repo);
+ const listed = await execGit(['worktree', 'list', '--porcelain'], repo).catch(() => null);
+ for (const line of (listed?.stdout || '').split(/\r?\n/)) {
+ if (line.startsWith('worktree ')) dirs.add(line.slice(9));
+ }
+ }
+ return [...dirs];
+}
diff --git a/server/services/agentSentinelSweep.test.js b/server/services/agentSentinelSweep.test.js
index dfda606d31..7a5d21046f 100644
--- a/server/services/agentSentinelSweep.test.js
+++ b/server/services/agentSentinelSweep.test.js
@@ -1,11 +1,13 @@
import { mkdtemp, readdir, utimes, writeFile } from 'fs/promises';
-import { rmSync } from 'fs';
+import { realpathSync, rmSync } from 'fs';
import { tmpdir } from 'os';
import { join } from 'path';
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
-import { sweepStaleDoneSentinels, STALE_SENTINEL_MIN_AGE_MS } from './agentSentinelSweep.js';
+import { sweepStaleDoneSentinels, checkoutDirsOf, STALE_SENTINEL_MIN_AGE_MS } from './agentSentinelSweep.js';
+import { execGit } from '../lib/execGit.js';
+import { materializeGitRepo, SKIP_HEAVY_INTEGRATION } from '../lib/gitTestRepo.js';
const tempRoots = [];
let dir;
@@ -78,3 +80,24 @@ describe('sweepStaleDoneSentinels', () => {
expect(await sweepStaleDoneSentinels([join(dir, 'nope'), dir, null, ''], new Set())).toBe(1);
});
});
+
+describe.skipIf(SKIP_HEAVY_INTEGRATION)('sweeping a repo\'s linked worktrees', () => {
+ it('removes a stale sentinel stranded in a linked worktree and leaves a live run\'s alone', async () => {
+ const repo = join(dir, 'repo');
+ await materializeGitRepo(repo, { identity: { email: '[email protected]', name: 'Test' } });
+ const tree = join(dir, 'linked');
+ await execGit(['worktree', 'add', '-b', 'feature/done', tree, 'main'], repo);
+ for (const name of ['.agent-done-agent-dead', '.agent-done-agent-live']) {
+ await writeFile(join(tree, name), 'summary');
+ const when = new Date(Date.now() - OLD);
+ await utimes(join(tree, name), when, when);
+ }
+
+ const dirs = await checkoutDirsOf([repo]);
+ expect(dirs).toContain(realpathSync(tree));
+ expect(await sweepStaleDoneSentinels(dirs, new Set(['agent-live']))).toBe(1);
+ expect(await readdir(tree)).toContain('.agent-done-agent-live');
+ expect(await readdir(tree)).not.toContain('.agent-done-agent-dead');
+ expect((await execGit(['status', '--porcelain'], tree)).stdout).not.toContain('agent-dead');
+ });
+});
diff --git a/server/services/autonomousJobs/scriptHandlers.js b/server/services/autonomousJobs/scriptHandlers.js
index 0f45027479..f1744d1e95 100644
--- a/server/services/autonomousJobs/scriptHandlers.js
+++ b/server/services/autonomousJobs/scriptHandlers.js
@@ -15,7 +15,7 @@ import { checkAndPrompt as autobiographyCheckAndPrompt } from '../autobiography.
import { runGoalCheckIn } from '../goalCheckIn.js'
import { cleanupOrphanedWorktrees, reapMergedWorktrees } from '../worktreeManager.js'
import { getActiveAgentIds } from '../agentState.js'
-import { sweepStaleDoneSentinels } from '../agentSentinelSweep.js'
+import { sweepStaleDoneSentinels, checkoutDirsOf } from '../agentSentinelSweep.js'
import { runSelfDiagnostics } from './selfDiagnostics.js'
import { runBrainParitySweep } from './brainParitySweep.js'
@@ -124,7 +124,9 @@ async function agentDataCleanup() {
// Worktree-less agents run in a REAL checkout — this repo, or a managed app's
// own repo — so a sentinel their cleanup never reached sits there as untracked
// dirt in the user's git status forever. Sweep every workspace they can land
- // in; a worktree-backed sentinel is already carried off with its worktree.
+ // in, and every linked worktree of those repos: a worktree-backed sentinel is
+ // carried off with its worktree only once that worktree is removed, and the
+ // sentinel itself is what keeps a preserved one from being removed.
// Lazy, like `eidoverseProjection` above: `apps.js` drags the PM2/status
// subtree into this registry's static graph, and this job only needs the list
// of repo paths (see the import-budget guard in lib/importScoping.test.js).
@@ -136,7 +138,7 @@ async function agentDataCleanup() {
// the in-memory map, which is EMPTY after a restart — sweeping on that would
// delete the resume signal of every paused and interrupted run.
const sentinelsRemoved = await sweepStaleDoneSentinels(
- [PATHS.root, ...apps.map(app => app?.repoPath)],
+ await checkoutDirsOf([PATHS.root, ...apps.map(app => app?.repoPath)]),
storedStatuses ? unfinishedIds : null
).catch((err) => {
console.warn(`⚠️ Stale sentinel sweep failed: ${err.message}`)