Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: sweep leaked autofixer worktrees, atomic history index, cooldown before persist (#11757)

[._.]/ Adam Eivy6719c45ce61c
diff --git a/autofixer/sandbox.git.test.js b/autofixer/sandbox.git.test.js
index 3235c9d455..d7132737a8 100644
--- a/autofixer/sandbox.git.test.js
+++ b/autofixer/sandbox.git.test.js
@@ -12,6 +12,7 @@ import {
createDisposableWorktree,
collectWorktreeDiff,
removeWorktree,
+ sweepStaleWorktrees,
applyDiffToLive,
execGit,
} from './sandbox.js';
@@ -50,6 +51,18 @@ describe('git worktree isolation (integration)', () => {
await rm(scratch, { recursive: true, force: true }).catch(() => {});
});
+ it('sweeps leftover worktrees: removes the directory and unregisters it from the repo', async () => {
+ const parent = join(scratch, 'worktrees');
+ const wt = await createDisposableWorktree(repo, parent, 'autofixer_leaked_1_abc');
+ expect(await execGit(['worktree', 'list', '--porcelain'], repo).then((r) => r.stdout)).toContain('autofixer_leaked_1_abc');
+
+ expect(await sweepStaleWorktrees(parent)).toEqual(['autofixer_leaked_1_abc']);
+
+ await expect(readFile(join(wt.path, 'app.js'), 'utf8')).rejects.toThrow();
+ expect(await execGit(['worktree', 'list', '--porcelain'], repo).then((r) => r.stdout)).not.toContain('autofixer_leaked_1_abc');
+ expect(await sweepStaleWorktrees(join(scratch, 'missing'))).toEqual([]);
+ });
+
it('detects a git checkout', async () => {
expect(await isGitRepo(repo)).toBe(true);
const nonRepo = join(scratch, 'plain');
diff --git a/autofixer/sandbox.js b/autofixer/sandbox.js
index a425294741..625ddef3a8 100644
--- a/autofixer/sandbox.js
+++ b/autofixer/sandbox.js
@@ -25,8 +25,8 @@
import { spawn } from 'child_process';
import { waitForChild, COMMAND_TIMEOUT_MS } from './childProcess.js';
-import { writeFile, rm, mkdir } from 'fs/promises';
-import { join, isAbsolute, normalize } from 'path';
+import { writeFile, rm, mkdir, readdir } from 'fs/promises';
+import { join, isAbsolute, normalize, resolve, dirname } from 'path';
import { randomBytes } from 'crypto';
// ---------------------------------------------------------------------------
@@ -414,6 +414,29 @@ export async function removeWorktree(repoPath, worktreePath) {
await rm(worktreePath, { recursive: true, force: true }).catch(() => {});
}
+/**
+ * Remove every worktree left in `parentDir` by a repair a restart interrupted.
+ * The daemon runs one repair at a time, so at startup nothing there is in use.
+ * Each entry is unregistered from the repository that owns it (found via its
+ * git common dir) before the directory is deleted. Returns the swept ids.
+ */
+export async function sweepStaleWorktrees(parentDir) {
+ const entries = await readdir(parentDir).catch(() => []);
+ const swept = [];
+ for (const id of entries) {
+ const entry = join(parentDir, id);
+ const { code, stdout } = await execGit(['rev-parse', '--git-common-dir'], entry);
+ if (code === 0 && stdout.trim()) {
+ const repoPath = dirname(resolve(entry, stdout.trim()));
+ await execGit(['worktree', 'remove', '--force', entry], repoPath);
+ await execGit(['worktree', 'prune'], repoPath);
+ }
+ await rm(entry, { recursive: true, force: true }).catch(() => {});
+ swept.push(id);
+ }
+ return swept;
+}
+
/**
* Run the user-configured verification command (e.g. `npm test`) against the
* isolated worktree with the sanitized env, before any change reaches the live
diff --git a/autofixer/server.js b/autofixer/server.js
index 53460737b3..4988568ecb 100644
--- a/autofixer/server.js
+++ b/autofixer/server.js
@@ -1,10 +1,11 @@
-import { readFile, writeFile, mkdir, access } from 'fs/promises';
+import { readFile, writeFile, mkdir, access, rename } from 'fs/promises';
import { join } from 'path';
// Dependency-light shared module (node builtins + pure arg builder only), so
// importing it from this standalone process doesn't pull in the AI toolkit.
// Lets the autofixer honor the user's configured CLI provider/model instead
// of hardcoding `claude -p`.
import { pickCliProvider, runCliProviderPrompt } from '../server/lib/cliProviderRun.js';
+import { signalDetachedGroups } from '../server/lib/credentialBootstrap.js';
import { agentGuardEnv } from '../server/lib/agentGuard/index.js';
import {
sanitizeChildEnv,
@@ -16,6 +17,7 @@ import {
createDisposableWorktree,
collectWorktreeDiff,
removeWorktree,
+ sweepStaleWorktrees,
applyDiffToLive,
revertDiffFromLive,
runVerifyCommand,
@@ -45,6 +47,8 @@ const FIX_COOLDOWN = 30 * 60 * 1000; // 30 minutes
const CHECK_INTERVAL = 15 * 60 * 1000; // 15 minutes
let checkTimer = null;
let shuttingDown = false;
+// The repair in flight, so a shutdown can record it as interrupted.
+let activeRepair = null;
// Parse JSON, returning `fallback` on read OR parse failure. A corrupt config
// file (partial write, hand-edit) must not throw inside fixProcess — this runs
@@ -97,6 +101,33 @@ async function findAppByProcess(processName) {
}
// History management
+// Record a repair that never reached finalize (daemon restarted mid-run).
+async function recordInterrupted({ sessionId, processName, app, startTime, prompt = '', output = '' }) {
+ const endTime = new Date().toISOString();
+ const metadata = {
+ sessionId, startTime, endTime,
+ duration: new Date(endTime).getTime() - new Date(startTime).getTime(),
+ exitCode: -1, success: false, processName,
+ appName: app?.name, appId: app?.id, repoPath: app?.repoPath,
+ type: 'autofixer', isolated: true, interrupted: true,
+ error: 'interrupted: autofixer restarted during repair',
+ };
+ await saveSession(sessionId, prompt, output, metadata);
+}
+
+// Sweep worktrees leaked by repairs a restart interrupted, and record each.
+async function sweepInterruptedRepairs() {
+ const swept = await sweepStaleWorktrees(WORKTREES_DIR);
+ for (const sessionId of swept) {
+ const processName = sessionId.match(/^autofixer_(.+)_\d+_[a-z0-9]+$/)?.[1] || 'unknown';
+ const now = new Date().toISOString();
+ await recordInterrupted({ sessionId, processName, startTime: now }).catch((err) => {
+ console.error(`❌ [Autofixer] Could not record interrupted session ${sessionId}: ${err.message}`);
+ });
+ }
+ if (swept.length) console.warn(`🧹 [Autofixer] Swept ${swept.length} worktree(s) left by interrupted repairs`);
+}
+
async function ensureHistoryDir() {
await mkdir(SESSIONS_DIR, { recursive: true });
await access(INDEX_FILE).catch(async () => {
@@ -116,7 +147,10 @@ async function loadIndex() {
}
async function saveIndex(index) {
- await writeFile(INDEX_FILE, JSON.stringify(index, null, 2));
+ // Temp + rename: a signal mid-write must not truncate the live index.
+ const tmp = `${INDEX_FILE}.tmp`;
+ await writeFile(tmp, JSON.stringify(index, null, 2));
+ await rename(tmp, INDEX_FILE);
}
async function saveSession(sessionId, prompt, output, metadata, patch) {
@@ -243,13 +277,14 @@ async function fixProcess(processName, app, errorLogs, outputLogs) {
...(error ? { error } : {}),
};
- await saveSession(sessionId, prompt, output, metadata, patch);
- console.log(`💾 [Autofixer] Saved session: ${sessionId}`);
-
// Cool down whenever a usable fix was produced (promoted, staged, OR
// applied-but-restart-failed) — not just on full success — so a crash the
// agent already addressed doesn't re-trigger the LLM every 15-min cycle.
+ // Set before persisting: a failed history write must not disable the cooldown.
if (cooldown ?? success) markAsFixed(processName);
+ activeRepair = null;
+ await saveSession(sessionId, prompt, output, metadata, patch);
+ console.log(`💾 [Autofixer] Saved session: ${sessionId}`);
return { success, sessionId, output, ...(error ? { error } : {}) };
};
@@ -273,6 +308,7 @@ async function fixProcess(processName, app, errorLogs, outputLogs) {
return finalize({ success: false, exitCode: -1, error: `worktree isolation failed: ${wt.error}`, extra: { isolated: false } });
}
const worktreePath = wt.path;
+ activeRepair = { sessionId, processName, app, startTime, prompt, outputBuffer, worktreePath };
// Strip host credentials the agent has no business seeing — it keeps only
// system + AI-provider auth. provider.envVars still overlays inside the runner.
const childEnv = sanitizeChildEnv(process.env);
@@ -469,13 +505,17 @@ async function main() {
console.log(`⏳ [Autofixer] Fix cooldown: ${FIX_COOLDOWN / 60000} minutes per process`);
await ensureHistoryDir();
+ await sweepInterruptedRepairs().catch((err) => {
+ console.error(`❌ [Autofixer] Worktree sweep failed: ${err.message}`);
+ });
// Run immediately, then wait a full interval after each pass completes.
await runCheckCycle();
}
// Graceful shutdown
-function shutdown() {
+async function shutdown() {
+ if (shuttingDown) return;
shuttingDown = true;
if (checkTimer) {
clearTimeout(checkTimer);
@@ -483,6 +523,15 @@ function shutdown() {
}
console.log(`\n🛑 [Autofixer] Shutting down...`);
+ // Stop the in-flight provider (its detached group) and record the attempt.
+ // Worktree removal is left to the next startup's sweep, the guarantee.
+ signalDetachedGroups('SIGTERM');
+ if (activeRepair) {
+ const { outputBuffer, ...repair } = activeRepair;
+ await recordInterrupted({ ...repair, output: outputBuffer.join('') }).catch((err) => {
+ console.error(`❌ [Autofixer] Could not record interrupted session: ${err.message}`);
+ });
+ }
process.exit(0);
}
diff --git a/autofixer/server.test.js b/autofixer/server.test.js
index 796ac3c016..bf0913224f 100644
--- a/autofixer/server.test.js
+++ b/autofixer/server.test.js
@@ -1,7 +1,8 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { writeFile, rename } from 'fs/promises';
const harness = vi.hoisted(() => ({
- apps: vi.fn(), pm2: vi.fn(), list: vi.fn(), provider: vi.fn(),
+ apps: vi.fn(), sweep: vi.fn(), validate: vi.fn(), pm2: vi.fn(), list: vi.fn(), provider: vi.fn(),
}));
vi.mock('./shared.js', () => ({
DATA_DIR: '/example/data', AUTOFIXER_DIR: '/example/autofixer', INDEX_FILE: '/example/index.json',
@@ -10,6 +11,7 @@ vi.mock('./shared.js', () => ({
vi.mock('fs/promises', () => ({
readFile: vi.fn(async file => file.endsWith('index.json') ? '[]' : '{}'),
writeFile: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), access: vi.fn(async () => {}),
+ rename: vi.fn(async () => {}),
}));
vi.mock('../server/lib/agentGuard/index.js', () => ({ agentGuardEnv: () => ({}) }));
vi.mock('../server/lib/cliProviderRun.js', () => ({
@@ -22,7 +24,8 @@ vi.mock('./sandbox.js', () => ({
restrictedToolArgs: () => [], isGitRepo: async () => true,
createDisposableWorktree: async () => ({ path: '/example/worktree' }),
collectWorktreeDiff: async () => '', removeWorktree: vi.fn(),
- validateProposedDiff: () => ({ ok: false, reason: 'No changes', files: [] }),
+ sweepStaleWorktrees: harness.sweep,
+ validateProposedDiff: harness.validate,
applyDiffToLive: vi.fn(), revertDiffFromLive: vi.fn(), runVerifyCommand: vi.fn(),
}));
@@ -36,6 +39,8 @@ beforeEach(() => {
progressLog = vi.spyOn(console, 'log').mockImplementation(() => {});
harness.apps.mockReset().mockResolvedValue([{ id: 'example', name: 'Example app', repoPath: '/example/repo', pm2ProcessNames: ['example-process'] }]);
harness.list.mockReset().mockResolvedValue([{ name: 'example-process', pm2_env: { status: 'errored' } }]);
+ harness.validate.mockReset().mockReturnValue({ ok: false, reason: 'No changes', files: [] });
+ harness.sweep.mockReset().mockResolvedValue([]);
harness.pm2.mockReset().mockResolvedValue({ stdout: '' });
harness.provider.mockReset().mockResolvedValue({ error: 'Example agent stopped', exitCode: 1 });
});
@@ -86,4 +91,28 @@ describe('Autofixer daemon repair input boundary', () => {
expect(harness.list).not.toHaveBeenCalled();
expect(harness.provider).not.toHaveBeenCalled();
});
+
+ it('records an interrupted session for each worktree swept at startup', async () => {
+ harness.apps.mockResolvedValue([]);
+ harness.sweep.mockResolvedValue(['autofixer_example-process_1700000000000_abc123']);
+ await start();
+ const meta = writeFile.mock.calls.find(([f]) => String(f).endsWith('metadata.json'));
+ expect(JSON.parse(meta[1])).toMatchObject({ processName: 'example-process', interrupted: true, success: false });
+ });
+
+ it('writes the history index via temp file + rename', async () => {
+ await start();
+ const tmpWrite = writeFile.mock.calls.findIndex(([f]) => f === '/example/index.json.tmp');
+ expect(tmpWrite).toBeGreaterThanOrEqual(0);
+ expect(rename).toHaveBeenCalledWith('/example/index.json.tmp', '/example/index.json');
+ });
+
+ it('still cools the process down when persisting the session fails', async () => {
+ harness.provider.mockResolvedValue({ exitCode: 0 });
+ harness.validate.mockReturnValue({ ok: true, files: ['a.js'] });
+ writeFile.mockImplementation(async (f) => { if (String(f).endsWith('prompt.txt')) throw new Error('disk full'); });
+ await start();
+ await nextCycle();
+ expect(harness.provider).toHaveBeenCalledOnce();
+ });
});
diff --git a/server/services/backup.js b/server/services/backup.js
index 7f1309ae41..ead8cb5af0 100644
--- a/server/services/backup.js
+++ b/server/services/backup.js
@@ -236,6 +236,7 @@ export const DEFAULT_EXCLUDES = [
{ path: '/python/laya-mlx/', reason: 'Rebuildable Laya-MLX experiment runtime and pinned model weights', overridable: false },
{ path: '/browser-profile/', reason: 'Browser CDP profile — cache/cookies, can be several GB', overridable: false },
{ path: '/cos/worktrees/', reason: 'Ephemeral agent git worktrees — recreated on demand', overridable: false },
+ { path: '/autofixer/worktrees/', reason: 'Disposable Autofixer repair worktrees — recreated for each repair', overridable: false },
{ path: '/cos/slashdo-resolved/', reason: 'Resolved slashdo command bodies staged for agent prompts — derived from the bundled submodule, regenerated on demand', overridable: false },
{ path: '/cos/feature-agents/*/worktree/', reason: 'Per-feature-agent git worktrees — recreated on demand', overridable: false },
// `**` (not `*`) so both engines' checkpoint dirs match: the torch trainer
diff --git a/server/services/backup.test.js b/server/services/backup.test.js
index 29280430dd..b745b684fb 100644
--- a/server/services/backup.test.js
+++ b/server/services/backup.test.js
@@ -218,6 +218,10 @@ describe('DEFAULT_EXCLUDES anchoring', () => {
// ANY depth, so `model.obj` would drop a file of that name anywhere under data/.
// That is a silent data-loss bug rather than a style nit, which is why it gets a
// test rather than a convention.
+ it('excludes disposable Autofixer repair worktrees', () => {
+ expect(DEFAULT_EXCLUDES.map((e) => e.path)).toContain('/autofixer/worktrees/');
+ });
+
it('anchors every default exclude at the data root', () => {
const unanchored = DEFAULT_EXCLUDES.filter((e) => !e.path.startsWith('/'));
expect(unanchored.map((e) => e.path)).toEqual([]);