Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: sweep leaked autofixer worktrees, atomic history index, cooldown before persist (#11757)
[._.]/ Adam Eivy6719c45ce61c
diff --git a/autofixer/sandbox.git.test.js b/autofixer/sandbox.git.test.jsindex 3235c9d455..d7132737a8 100644--- a/autofixer/sandbox.git.test.js+++ b/autofixer/sandbox.git.test.js@@ -12,6 +12,7 @@ import {createDisposableWorktree,collectWorktreeDiff,removeWorktree,+ sweepStaleWorktrees,applyDiffToLive,execGit,} from './sandbox.js';@@ -50,6 +51,18 @@ describe('git worktree isolation (integration)', () => {await rm(scratch, { recursive: true, force: true }).catch(() => {});});+ it('sweeps leftover worktrees: removes the directory and unregisters it from the repo', async () => {+ const parent = join(scratch, 'worktrees');+ const wt = await createDisposableWorktree(repo, parent, 'autofixer_leaked_1_abc');+ expect(await execGit(['worktree', 'list', '--porcelain'], repo).then((r) => r.stdout)).toContain('autofixer_leaked_1_abc');++ expect(await sweepStaleWorktrees(parent)).toEqual(['autofixer_leaked_1_abc']);++ await expect(readFile(join(wt.path, 'app.js'), 'utf8')).rejects.toThrow();+ expect(await execGit(['worktree', 'list', '--porcelain'], repo).then((r) => r.stdout)).not.toContain('autofixer_leaked_1_abc');+ expect(await sweepStaleWorktrees(join(scratch, 'missing'))).toEqual([]);+ });+it('detects a git checkout', async () => {expect(await isGitRepo(repo)).toBe(true);const nonRepo = join(scratch, 'plain');diff --git a/autofixer/sandbox.js b/autofixer/sandbox.jsindex a425294741..625ddef3a8 100644--- a/autofixer/sandbox.js+++ b/autofixer/sandbox.js@@ -25,8 +25,8 @@import { spawn } from 'child_process';import { waitForChild, COMMAND_TIMEOUT_MS } from './childProcess.js';-import { writeFile, rm, mkdir } from 'fs/promises';-import { join, isAbsolute, normalize } from 'path';+import { writeFile, rm, mkdir, readdir } from 'fs/promises';+import { join, isAbsolute, normalize, resolve, dirname } from 'path';import { randomBytes } from 'crypto';// ---------------------------------------------------------------------------@@ -414,6 +414,29 @@ export async function removeWorktree(repoPath, worktreePath) {await rm(worktreePath, { recursive: true, force: true }).catch(() => {});}+/**+ * Remove every worktree left in `parentDir` by a repair a restart interrupted.+ * The daemon runs one repair at a time, so at startup nothing there is in use.+ * Each entry is unregistered from the repository that owns it (found via its+ * git common dir) before the directory is deleted. Returns the swept ids.+ */+export async function sweepStaleWorktrees(parentDir) {+ const entries = await readdir(parentDir).catch(() => []);+ const swept = [];+ for (const id of entries) {+ const entry = join(parentDir, id);+ const { code, stdout } = await execGit(['rev-parse', '--git-common-dir'], entry);+ if (code === 0 && stdout.trim()) {+ const repoPath = dirname(resolve(entry, stdout.trim()));+ await execGit(['worktree', 'remove', '--force', entry], repoPath);+ await execGit(['worktree', 'prune'], repoPath);+ }+ await rm(entry, { recursive: true, force: true }).catch(() => {});+ swept.push(id);+ }+ return swept;+}+/*** Run the user-configured verification command (e.g. `npm test`) against the* isolated worktree with the sanitized env, before any change reaches the livediff --git a/autofixer/server.js b/autofixer/server.jsindex 53460737b3..4988568ecb 100644--- a/autofixer/server.js+++ b/autofixer/server.js@@ -1,10 +1,11 @@-import { readFile, writeFile, mkdir, access } from 'fs/promises';+import { readFile, writeFile, mkdir, access, rename } from 'fs/promises';import { join } from 'path';// Dependency-light shared module (node builtins + pure arg builder only), so// importing it from this standalone process doesn't pull in the AI toolkit.// Lets the autofixer honor the user's configured CLI provider/model instead// of hardcoding `claude -p`.import { pickCliProvider, runCliProviderPrompt } from '../server/lib/cliProviderRun.js';+import { signalDetachedGroups } from '../server/lib/credentialBootstrap.js';import { agentGuardEnv } from '../server/lib/agentGuard/index.js';import {sanitizeChildEnv,@@ -16,6 +17,7 @@ import {createDisposableWorktree,collectWorktreeDiff,removeWorktree,+ sweepStaleWorktrees,applyDiffToLive,revertDiffFromLive,runVerifyCommand,@@ -45,6 +47,8 @@ const FIX_COOLDOWN = 30 * 60 * 1000; // 30 minutesconst CHECK_INTERVAL = 15 * 60 * 1000; // 15 minuteslet checkTimer = null;let shuttingDown = false;+// The repair in flight, so a shutdown can record it as interrupted.+let activeRepair = null;// Parse JSON, returning `fallback` on read OR parse failure. A corrupt config// file (partial write, hand-edit) must not throw inside fixProcess — this runs@@ -97,6 +101,33 @@ async function findAppByProcess(processName) {}// History management+// Record a repair that never reached finalize (daemon restarted mid-run).+async function recordInterrupted({ sessionId, processName, app, startTime, prompt = '', output = '' }) {+ const endTime = new Date().toISOString();+ const metadata = {+ sessionId, startTime, endTime,+ duration: new Date(endTime).getTime() - new Date(startTime).getTime(),+ exitCode: -1, success: false, processName,+ appName: app?.name, appId: app?.id, repoPath: app?.repoPath,+ type: 'autofixer', isolated: true, interrupted: true,+ error: 'interrupted: autofixer restarted during repair',+ };+ await saveSession(sessionId, prompt, output, metadata);+}++// Sweep worktrees leaked by repairs a restart interrupted, and record each.+async function sweepInterruptedRepairs() {+ const swept = await sweepStaleWorktrees(WORKTREES_DIR);+ for (const sessionId of swept) {+ const processName = sessionId.match(/^autofixer_(.+)_\d+_[a-z0-9]+$/)?.[1] || 'unknown';+ const now = new Date().toISOString();+ await recordInterrupted({ sessionId, processName, startTime: now }).catch((err) => {+ console.error(`❌ [Autofixer] Could not record interrupted session ${sessionId}: ${err.message}`);+ });+ }+ if (swept.length) console.warn(`🧹 [Autofixer] Swept ${swept.length} worktree(s) left by interrupted repairs`);+}+async function ensureHistoryDir() {await mkdir(SESSIONS_DIR, { recursive: true });await access(INDEX_FILE).catch(async () => {@@ -116,7 +147,10 @@ async function loadIndex() {}async function saveIndex(index) {- await writeFile(INDEX_FILE, JSON.stringify(index, null, 2));+ // Temp + rename: a signal mid-write must not truncate the live index.+ const tmp = `${INDEX_FILE}.tmp`;+ await writeFile(tmp, JSON.stringify(index, null, 2));+ await rename(tmp, INDEX_FILE);}async function saveSession(sessionId, prompt, output, metadata, patch) {@@ -243,13 +277,14 @@ async function fixProcess(processName, app, errorLogs, outputLogs) {...(error ? { error } : {}),};- await saveSession(sessionId, prompt, output, metadata, patch);- console.log(`💾 [Autofixer] Saved session: ${sessionId}`);-// Cool down whenever a usable fix was produced (promoted, staged, OR// applied-but-restart-failed) — not just on full success — so a crash the// agent already addressed doesn't re-trigger the LLM every 15-min cycle.+ // Set before persisting: a failed history write must not disable the cooldown.if (cooldown ?? success) markAsFixed(processName);+ activeRepair = null;+ await saveSession(sessionId, prompt, output, metadata, patch);+ console.log(`💾 [Autofixer] Saved session: ${sessionId}`);return { success, sessionId, output, ...(error ? { error } : {}) };};@@ -273,6 +308,7 @@ async function fixProcess(processName, app, errorLogs, outputLogs) {return finalize({ success: false, exitCode: -1, error: `worktree isolation failed: ${wt.error}`, extra: { isolated: false } });}const worktreePath = wt.path;+ activeRepair = { sessionId, processName, app, startTime, prompt, outputBuffer, worktreePath };// Strip host credentials the agent has no business seeing — it keeps only// system + AI-provider auth. provider.envVars still overlays inside the runner.const childEnv = sanitizeChildEnv(process.env);@@ -469,13 +505,17 @@ async function main() {console.log(`⏳ [Autofixer] Fix cooldown: ${FIX_COOLDOWN / 60000} minutes per process`);await ensureHistoryDir();+ await sweepInterruptedRepairs().catch((err) => {+ console.error(`❌ [Autofixer] Worktree sweep failed: ${err.message}`);+ });// Run immediately, then wait a full interval after each pass completes.await runCheckCycle();}// Graceful shutdown-function shutdown() {+async function shutdown() {+ if (shuttingDown) return;shuttingDown = true;if (checkTimer) {clearTimeout(checkTimer);@@ -483,6 +523,15 @@ function shutdown() {}console.log(`\n🛑 [Autofixer] Shutting down...`);+ // Stop the in-flight provider (its detached group) and record the attempt.+ // Worktree removal is left to the next startup's sweep, the guarantee.+ signalDetachedGroups('SIGTERM');+ if (activeRepair) {+ const { outputBuffer, ...repair } = activeRepair;+ await recordInterrupted({ ...repair, output: outputBuffer.join('') }).catch((err) => {+ console.error(`❌ [Autofixer] Could not record interrupted session: ${err.message}`);+ });+ }process.exit(0);}diff --git a/autofixer/server.test.js b/autofixer/server.test.jsindex 796ac3c016..bf0913224f 100644--- a/autofixer/server.test.js+++ b/autofixer/server.test.js@@ -1,7 +1,8 @@import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { writeFile, rename } from 'fs/promises';const harness = vi.hoisted(() => ({- apps: vi.fn(), pm2: vi.fn(), list: vi.fn(), provider: vi.fn(),+ apps: vi.fn(), sweep: vi.fn(), validate: vi.fn(), pm2: vi.fn(), list: vi.fn(), provider: vi.fn(),}));vi.mock('./shared.js', () => ({DATA_DIR: '/example/data', AUTOFIXER_DIR: '/example/autofixer', INDEX_FILE: '/example/index.json',@@ -10,6 +11,7 @@ vi.mock('./shared.js', () => ({vi.mock('fs/promises', () => ({readFile: vi.fn(async file => file.endsWith('index.json') ? '[]' : '{}'),writeFile: vi.fn(async () => {}), mkdir: vi.fn(async () => {}), access: vi.fn(async () => {}),+ rename: vi.fn(async () => {}),}));vi.mock('../server/lib/agentGuard/index.js', () => ({ agentGuardEnv: () => ({}) }));vi.mock('../server/lib/cliProviderRun.js', () => ({@@ -22,7 +24,8 @@ vi.mock('./sandbox.js', () => ({restrictedToolArgs: () => [], isGitRepo: async () => true,createDisposableWorktree: async () => ({ path: '/example/worktree' }),collectWorktreeDiff: async () => '', removeWorktree: vi.fn(),- validateProposedDiff: () => ({ ok: false, reason: 'No changes', files: [] }),+ sweepStaleWorktrees: harness.sweep,+ validateProposedDiff: harness.validate,applyDiffToLive: vi.fn(), revertDiffFromLive: vi.fn(), runVerifyCommand: vi.fn(),}));@@ -36,6 +39,8 @@ beforeEach(() => {progressLog = vi.spyOn(console, 'log').mockImplementation(() => {});harness.apps.mockReset().mockResolvedValue([{ id: 'example', name: 'Example app', repoPath: '/example/repo', pm2ProcessNames: ['example-process'] }]);harness.list.mockReset().mockResolvedValue([{ name: 'example-process', pm2_env: { status: 'errored' } }]);+ harness.validate.mockReset().mockReturnValue({ ok: false, reason: 'No changes', files: [] });+ harness.sweep.mockReset().mockResolvedValue([]);harness.pm2.mockReset().mockResolvedValue({ stdout: '' });harness.provider.mockReset().mockResolvedValue({ error: 'Example agent stopped', exitCode: 1 });});@@ -86,4 +91,28 @@ describe('Autofixer daemon repair input boundary', () => {expect(harness.list).not.toHaveBeenCalled();expect(harness.provider).not.toHaveBeenCalled();});++ it('records an interrupted session for each worktree swept at startup', async () => {+ harness.apps.mockResolvedValue([]);+ harness.sweep.mockResolvedValue(['autofixer_example-process_1700000000000_abc123']);+ await start();+ const meta = writeFile.mock.calls.find(([f]) => String(f).endsWith('metadata.json'));+ expect(JSON.parse(meta[1])).toMatchObject({ processName: 'example-process', interrupted: true, success: false });+ });++ it('writes the history index via temp file + rename', async () => {+ await start();+ const tmpWrite = writeFile.mock.calls.findIndex(([f]) => f === '/example/index.json.tmp');+ expect(tmpWrite).toBeGreaterThanOrEqual(0);+ expect(rename).toHaveBeenCalledWith('/example/index.json.tmp', '/example/index.json');+ });++ it('still cools the process down when persisting the session fails', async () => {+ harness.provider.mockResolvedValue({ exitCode: 0 });+ harness.validate.mockReturnValue({ ok: true, files: ['a.js'] });+ writeFile.mockImplementation(async (f) => { if (String(f).endsWith('prompt.txt')) throw new Error('disk full'); });+ await start();+ await nextCycle();+ expect(harness.provider).toHaveBeenCalledOnce();+ });});diff --git a/server/services/backup.js b/server/services/backup.jsindex 7f1309ae41..ead8cb5af0 100644--- a/server/services/backup.js+++ b/server/services/backup.js@@ -236,6 +236,7 @@ export const DEFAULT_EXCLUDES = [{ path: '/python/laya-mlx/', reason: 'Rebuildable Laya-MLX experiment runtime and pinned model weights', overridable: false },{ path: '/browser-profile/', reason: 'Browser CDP profile — cache/cookies, can be several GB', overridable: false },{ path: '/cos/worktrees/', reason: 'Ephemeral agent git worktrees — recreated on demand', overridable: false },+ { path: '/autofixer/worktrees/', reason: 'Disposable Autofixer repair worktrees — recreated for each repair', overridable: false },{ path: '/cos/slashdo-resolved/', reason: 'Resolved slashdo command bodies staged for agent prompts — derived from the bundled submodule, regenerated on demand', overridable: false },{ path: '/cos/feature-agents/*/worktree/', reason: 'Per-feature-agent git worktrees — recreated on demand', overridable: false },// `**` (not `*`) so both engines' checkpoint dirs match: the torch trainerdiff --git a/server/services/backup.test.js b/server/services/backup.test.jsindex 29280430dd..b745b684fb 100644--- a/server/services/backup.test.js+++ b/server/services/backup.test.js@@ -218,6 +218,10 @@ describe('DEFAULT_EXCLUDES anchoring', () => {// ANY depth, so `model.obj` would drop a file of that name anywhere under data/.// That is a silent data-loss bug rather than a style nit, which is why it gets a// test rather than a convention.+ it('excludes disposable Autofixer repair worktrees', () => {+ expect(DEFAULT_EXCLUDES.map((e) => e.path)).toContain('/autofixer/worktrees/');+ });+it('anchors every default exclude at the data root', () => {const unanchored = DEFAULT_EXCLUDES.filter((e) => !e.path.startsWith('/'));expect(unanchored.map((e) => e.path)).toEqual([]);