Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: cut server memory churn and release reservations of restart-interrupted agents
Adam Eivy779d32012643
diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.jsindex 1c212f87cd..1b3f0363bd 100644--- a/server/lib/maintenanceAdmission.js+++ b/server/lib/maintenanceAdmission.js@@ -59,9 +59,26 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, assereturn schema.parse(JSON.parse(io.readFileSync(file, 'utf8')));} catch { throw unavailable(); }};+ // status()/held() are polled several times a second (media drain loop, spawn+ // gates); re-parsing a journal that carries every settled operation allocated+ // ~400 KB per call. Writers publish by renaming a new file into place, so any+ // change gives it a new identity; the stat is taken before the read, so a+ // racing publish can only cost a re-parse. Read-only: transactions mutate the+ // state they read and must keep calling read() for a private copy.+ let statusCache = null;+ const readForStatus = () => {+ if (uncertain) throw unavailable();+ let stats;+ try { stats = io.lstatSync(file); }+ catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(); }+ if (!stats.isFile()) throw unavailable();+ const key = `${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;+ if (statusCache?.key !== key) statusCache = { key, state: read() };+ return statusCache.state;+ };const project = state => ({state: state.hold ? (state.operations.length || state.exclusive ? 'draining' : 'ready') : 'normal',- revision: state.revision, hold: state.hold,+ revision: state.revision, hold: state.hold && { ...state.hold },blockers: [...state.operations.map(({ kind, resource, startedAt, pid, unsettled }) => ({ kind, resource, startedAt, pid, unsettled })),...(state.exclusive ? [{ kind: 'exclusive-maintenance', resource: state.exclusive.operation.operationId,startedAt: state.exclusive.claimedAt, unsettled: state.exclusive.phase === 'uncertain' }] : [])],@@ -70,7 +87,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asseconst status = () => {try {if (exists(lock)) throw unavailable();- return project(read());+ return project(readForStatus());} catch {return { state: 'unavailable', revision: null, hold: null, blockers: [],error: 'Maintenance state needs recovery. New work is held; readiness is unknown.' };diff --git a/server/lib/maintenanceAdmission.test.js b/server/lib/maintenanceAdmission.test.jsindex db382835dd..a3ae761ae3 100644--- a/server/lib/maintenanceAdmission.test.js+++ b/server/lib/maintenanceAdmission.test.js@@ -163,6 +163,30 @@ describe('durable graceful maintenance', () => {expect(fs.readFileSync(file, 'utf8')).toBe(before);});+ it('serves status from one parse until the journal file changes', () => {+ const { root } = gate();+ let reads = 0;+ const io = { ...fs, readFileSync: (...args) => { reads++; return fs.readFileSync(...args); } };+ const admission = createMaintenanceAdmission(root, { io });+ admission.admit('media', 'render-a');+ const before = reads;+ for (let i = 0; i < 5; i++) expect(admission.held()).toBe(false);+ expect(reads - before).toBeLessThanOrEqual(1);+ const settled = reads;+ admission.status();+ expect(reads).toBe(settled);++ // A second process publishing (here: a second admission over the same root) is seen at once.+ const other = createMaintenanceAdmission(root);+ const hold = other.begin({ reason: 'Drain', owner: 'Operator' }).hold;+ expect(admission.held()).toBe(true);+ // Callers cannot corrupt the cached journal through the projection they receive.+ admission.status().hold.reason = 'tampered';+ expect(admission.status().hold.reason).toBe('Drain');+ other.resume({ id: hold.id, revision: hold.revision });+ expect(admission.held()).toBe(false);+ });+it('keeps journal write failure unavailable without claiming recovered completion', () => {const { root } = gate();let fail = false;diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex 36073b7254..504ebae0e8 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -1180,6 +1180,10 @@ async function runCleanupOrphanedAgents() {const agents = await getAgents();let cleanedCount = 0;const orphanedTaskIds = [];+ // Agents retired by this sweep, with or without a task: each still holds the+ // maintenance reservation its original spawn took, and retireDeadAgent does+ // not release it.+ const retiredAgentIds = [];// Agents the PREVIOUS process owned when it was signalled down (#3202). Their// processes died because PortOS restarted, not because they failed — so they@@ -1356,6 +1360,7 @@ async function runCleanupOrphanedAgents() {agentResultExtra: { interruptedByRestart: interrupted },});cleanedCount++;+ retiredAgentIds.push(agent.id);if (agent.taskId) {// Carry the agent's metadata forward rather than re-reading the record@@ -1407,6 +1412,16 @@ async function runCleanupOrphanedAgents() {});}+ // Same release order as a normal completion (agentLifecycle): only after the+ // run is retired, its worktree cleaned and its task settled. Without it the+ // reservation outlives the process and every later status read parses it.+ // finishResource reads the journal synchronously and throws while it needs+ // recovery; that must not abort the marker clear and stranded-agent sweep below.+ for (const agentId of retiredAgentIds) {+ try { await maintenance.finishResource('agent', agentId); }+ catch (err) { emitLog('warn', `Maintenance reservation for ${agentId} was not released: ${err.message}`, { agentId }); }+ }+// Consume the marker. Doing it AFTER the retry pass (not before) means a crash// mid-sweep leaves it in place and the next boot still classifies these agents// correctly; re-reading a stale marker is harmless because the sweep only everdiff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.jsindex aab4d284f7..bbc649c922 100644--- a/server/services/agentManagement.test.js+++ b/server/services/agentManagement.test.js@@ -120,6 +120,13 @@ vi.mock('../lib/hostShutdown.js', async (importOriginal) => ({readHostShutdownMarker: vi.fn().mockResolvedValue(null),clearHostShutdownMarker: vi.fn().mockResolvedValue(undefined),}));+// Real admission with only the settlement release observable: the default instance+// resolves the live data root, so a spy keeps this suite off real journal state.+vi.mock('../lib/maintenanceAdmission.js', async (importOriginal) => {+ const actual = await importOriginal();+ const finishResource = vi.fn().mockResolvedValue([]);+ return { ...actual, maintenance: new Proxy(actual.maintenance, { get: (target, key) => key === 'finishResource' ? finishResource : target[key] }) };+});vi.mock('./agentRunnerSync.js', () => ({ syncRunnerAgents: vi.fn().mockResolvedValue(0) }));vi.mock('./agentRunnerOutputBatchers.js', () => ({ flushRunnerOutputBatcher: vi.fn().mockResolvedValue(undefined) }));vi.mock('./worktreeManager.js', () => ({ cleanupOrphanedWorktrees: vi.fn() }));@@ -132,6 +139,7 @@ vi.mock('./creativeDirector/completionHook.js', () => ({ advanceAfterSceneSettleimport { handleOrphanedTask, pauseAgent, resumeAgent, relaunchAgent, settleOrphanedCreativeDirectorRun, cleanupOrphanedAgents, terminateAgent, killAgent, getAgentProcessStats } from './agentManagement.js';import { retireDeadAgent, stampLiExecutionVerdict } from './agentFinalization.js';+import { maintenance } from '../lib/maintenanceAdmission.js';import { cleanupAgentWorktree, resolveTaskResumePatch } from './agentWorktreeCleanup.js';import { getAgents, updateAgent, getAgentRecord, readAgentRecordOrUnreadable, AGENT_RECORD_UNREADABLE, completeAgent as markAgentComplete } from './cosAgentLifecycle.js';import { updateRun, getProject } from './creativeDirector/local.js';@@ -256,6 +264,36 @@ describe('cleanupOrphanedAgents — startup recovery coordination', () => {expect(retireDeadAgent).toHaveBeenCalledWith(expect.objectContaining({ agent: expect.objectContaining({ id: 'agent-old' }) }));});+ it('releases the maintenance reservation of every agent the sweep retires, task or not', async () => {+ getActiveAgentsFromRunner.mockResolvedValue([]);+ getTaskById.mockResolvedValue({ id: 'task-1', taskType: 'user', status: 'in_progress', metadata: {} });+ const old = new Date(Date.now() - 120000).toISOString();+ getAgents.mockResolvedValueOnce([+ { id: 'agent-with-task', status: 'running', taskId: 'task-1', startedAt: old, metadata: {} },+ { id: 'agent-no-task', status: 'running', startedAt: old, metadata: {} },+ ]);+ maintenance.finishResource.mockClear();+ await cleanupOrphanedAgents();+ expect(retireDeadAgent).toHaveBeenCalledTimes(2);+ expect(maintenance.finishResource.mock.calls).toEqual([['agent', 'agent-with-task'], ['agent', 'agent-no-task']]);+ // Release follows the task settlement, as in a normal completion.+ expect(cleanupAgentWorktree.mock.invocationCallOrder[0]).toBeLessThan(maintenance.finishResource.mock.invocationCallOrder[0]);+ });++ it('finishes the sweep when a maintenance release fails', async () => {+ getActiveAgentsFromRunner.mockResolvedValue([]);+ const old = new Date(Date.now() - 120000).toISOString();+ getAgents.mockResolvedValueOnce([+ { id: 'agent-first', status: 'running', startedAt: old, metadata: {} },+ { id: 'agent-second', status: 'running', startedAt: old, metadata: {} },+ ]);+ maintenance.finishResource.mockClear();+ maintenance.finishResource.mockImplementationOnce(() => { throw new Error('Maintenance state needs recovery'); });+ await expect(cleanupOrphanedAgents()).resolves.toBe(2);+ // The failed release neither stops the next one nor the rest of the sweep.+ expect(maintenance.finishResource.mock.calls).toEqual([['agent', 'agent-first'], ['agent', 'agent-second']]);+ });+it('reaps a durable running record whose runner listing is stale', async () => {getAgents.mockResolvedValueOnce([{id: 'agent-stale',diff --git a/server/services/cosDequeue.js b/server/services/cosDequeue.jsindex 6276626e14..f3b557ccb7 100644--- a/server/services/cosDequeue.js+++ b/server/services/cosDequeue.js@@ -277,8 +277,8 @@ export async function admitIdleReviewTask({// Deferred: the generator imports this owner, and ordinary capacity users// need neither the task store nor the prompt-generation dependency graph.- const { getCosTasks } = await import('./cosTaskStore.js');- if ((await getCosTasks()).autoApproved?.length) return;+ const { getCosApprovalView } = await import('./cosTaskStore.js');+ if ((await getCosApprovalView()).autoApprovedCount) return;const { generateIdleReviewTask, recordDeferredPerpetualDispatch } = await import('./cosTaskGenerator.js');const { task, pendingPerpetualDispatch, preflightCardId } = await generateIdleReviewTask(state, {ignoreTaskId,diff --git a/server/services/cosIdleReviewParity.test.js b/server/services/cosIdleReviewParity.test.jsindex 5577812d93..1d4bc8f29f 100644--- a/server/services/cosIdleReviewParity.test.js+++ b/server/services/cosIdleReviewParity.test.js@@ -31,6 +31,7 @@ vi.mock('./cosTaskStore.js', async (importActual) => ({}),getUserTasks: async () => ({ exists: true, grouped: { pending: mocks.userTasks } }),getCosTasks: async () => mocks.cosTaskData,+ getCosApprovalView: async () => ({ autoApprovedCount: mocks.cosTaskData.autoApproved?.length ?? 0 }),reconcileLegacyForgeMaintenanceTasks: async () => ({ recovered: 0, retired: 0 }),addTask: async (task) => { mocks.persisted.push(task); return task; },}));diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.jsindex eb1aec90d6..d441a6d91f 100644--- a/server/services/cosTaskStore.js+++ b/server/services/cosTaskStore.js@@ -452,6 +452,24 @@ export async function getTaskDiagnostics() {return structuredClone(sources.flat());}+/**+ * Approval-state view of the internal queue for pollers that need the awaiting+ * list or the auto-approved count, not the whole queue. `getCosTasks()` clones+ * every archived prompt body to answer the same question. Derived once per+ * snapshot; only the first `limit` awaiting tasks are cloned.+ */+export async function getCosApprovalView({ limit = Infinity } = {}) {+ const filePath = await resolveSourceFile('internal');+ if (!filePath) return { awaitingApproval: [], autoApprovedCount: 0 };+ const snapshot = await readTaskSnapshot(filePath);+ snapshot.approval ??= {+ awaiting: getAwaitingApprovalTasks(snapshot.tasks),+ autoApprovedCount: getAutoApprovedTasks(snapshot.tasks).length,+ };+ const { awaiting, autoApprovedCount } = snapshot.approval;+ return { awaitingApproval: structuredClone(awaiting.slice(0, limit)), autoApprovedCount };+}+/*** Alias for backward compatibility*/diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.jsindex 7c2a354ce1..ec04205b86 100644--- a/server/services/cosTaskStore.test.js+++ b/server/services/cosTaskStore.test.js@@ -131,6 +131,7 @@ import {getAllTasks,getTaskView,getPendingTaskIds,+ getCosApprovalView,getTaskDiagnostics,getTasks,getTaskById,@@ -388,6 +389,29 @@ describe('cosTaskStore parsed-task cache (#3497)', () => {expect(await getPendingTaskIds()).toEqual([]);});+ it('projects approval state without cloning the queue and refreshes after writes', async () => {+ expect(await getCosApprovalView()).toEqual({ awaitingApproval: [], autoApprovedCount: 0 });+ await addTask({ description: 'auto sys', approvalRequired: false }, 'internal');+ await addTask({ description: 'needs approval', approvalRequired: true }, 'internal');+ await addTask({ description: 'also needs approval', approvalRequired: true }, 'internal');++ const expectedFirst = (await getCosTasks()).awaitingApproval[0].id;+ const clone = vi.spyOn(globalThis, 'structuredClone');+ let view;+ let clonedPayload;+ try { view = await getCosApprovalView({ limit: 1 }); clonedPayload = clone.mock.calls[0]?.[0]; } finally { clone.mockRestore(); }+ expect(view.autoApprovedCount).toBe(1);+ expect(view.awaitingApproval.map(t => t.id)).toEqual([expectedFirst]);+ // Only the returned page is cloned, never the whole queue.+ expect(clonedPayload).toHaveLength(1);++ view.awaitingApproval[0].description = 'caller edit';+ expect((await getCosApprovalView()).awaitingApproval[0].description).toBe('needs approval');++ await addTask({ description: 'third approval', approvalRequired: true }, 'internal');+ expect((await getCosApprovalView({ limit: 10 })).awaitingApproval).toHaveLength(3);+ });+it('serves a cached parse while the file is unchanged', async () => {await addTask({ description: 'cached thing' }, 'user');mock.parseCalls = 0;diff --git a/server/services/reviewQueue.js b/server/services/reviewQueue.jsindex 66977e91ac..e2d26e87cd 100644--- a/server/services/reviewQueue.js+++ b/server/services/reviewQueue.js@@ -413,8 +413,8 @@ const PRODUCERS = [},async gather(limit = REVIEW_QUEUE_SOURCE_READ_LIMIT) {// Internal CoS tasks parked awaiting the user's approval before they run.- const { awaitingApproval = [] } = await cosTaskStore.getCosTasks();- return (Array.isArray(awaitingApproval) ? awaitingApproval : []).slice(0, limit);+ const { awaitingApproval } = await cosTaskStore.getCosApprovalView({ limit });+ return awaitingApproval;},map(task) {// Surface the task priority as a triage badge. Only HIGH/MEDIUM/LOW arediff --git a/server/services/reviewQueue.test.js b/server/services/reviewQueue.test.jsindex 99f9ccc7c8..e626cf695d 100644--- a/server/services/reviewQueue.test.js+++ b/server/services/reviewQueue.test.js@@ -7,7 +7,7 @@ vi.mock('./apps.js', () => apps);const brain = { getInboxLog: vi.fn(), markInboxDone: vi.fn() };const brainStorage = { getThreads: vi.fn(), getThreadById: vi.fn(), updateWith: vi.fn() };const askConversations = { listConversations: vi.fn() };-const cosTaskStore = { getCosTasks: vi.fn(), approveTask: vi.fn(), getTaskById: vi.fn() };+const cosTaskStore = { getCosApprovalView: vi.fn(), approveTask: vi.fn(), getTaskById: vi.fn() };const cosAgentFeedback = { getPendingAgentFeedback: vi.fn(), submitAgentFeedback: vi.fn() };const messageDrafts = { listDrafts: vi.fn(), approveDraft: vi.fn() };const proactiveAlerts = { generateNonProductAlerts: vi.fn(), resolveHealthAlert: vi.fn() };@@ -86,7 +86,7 @@ function resetEmpty() {brainStorage.getThreadById.mockResolvedValue(null);brainStorage.updateWith.mockResolvedValue(null);askConversations.listConversations.mockResolvedValue([]);- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [] });+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [] });cosAgentFeedback.getPendingAgentFeedback.mockResolvedValue({ agents: [], unavailable: [], count: 0 });cosAgentFeedback.submitAgentFeedback.mockResolvedValue({ success: true });messageDrafts.listDrafts.mockResolvedValue([]);@@ -517,7 +517,7 @@ describe('reviewQueue.buildQueue', () => {});it('surfaces drafts and CoS approvals from their producers', async () => {- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });// The producer pushes a multi-status filter down to listDrafts, so the mock// honors it the way the real implementation does (drops the 'sent' draft).const allDrafts = [@@ -964,7 +964,7 @@ describe('reviewQueue.buildQueue', () => {it('tags resolvable rows with an inline action verb, leaves no-clean-resolve sources without one', async () => {brain.getInboxLog.mockResolvedValue([{ id: 'b1', capturedText: 'classify', capturedAt: '2026-06-03T10:00:00.000Z' }]);- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });askConversations.listConversations.mockResolvedValue([{ id: 'a1', title: 'promote me', promoted: false, turnCount: 1, assistantTurnCount: 1 }]);proactiveAlerts.generateNonProductAlerts.mockResolvedValue([{ id: 'disk:primary', type: 'disk', severity: 'critical', title: 'crit', detail: 'full', link: '/apps' }]);backup.getState.mockResolvedValue({ status: 'error', error: 'disk full' });@@ -982,7 +982,7 @@ describe('reviewQueue.buildQueue', () => {it('attaches source-appropriate meta chips when the raw field is present', async () => {brain.getInboxLog.mockResolvedValue([{ id: 'b1', capturedText: 'classify', source: 'voice', capturedAt: '2026-06-03T10:00:00.000Z' }]);askConversations.listConversations.mockResolvedValue([{ id: 'a1', title: 'promote me', promoted: false, turnCount: 4, assistantTurnCount: 2 }]);- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'MEDIUM', createdAt: '2026-06-03T09:00:00.000Z' }] });+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'MEDIUM', createdAt: '2026-06-03T09:00:00.000Z' }] });messageDrafts.listDrafts.mockResolvedValue([{ id: 'd1', status: 'draft', subject: 's', to: ['[email protected]'], sendVia: 'gmail', updatedAt: '2026-06-03T08:00:00.000Z' }]);proactiveAlerts.generateNonProductAlerts.mockResolvedValue([{ id: 'system_resource:memory', type: 'system_resource', severity: 'critical', title: 'mem', detail: 'high', link: '/apps' }]);const queue = await buildQueue();@@ -1004,7 +1004,7 @@ describe('reviewQueue.buildQueue', () => {});it('drops an out-of-range cos priority rather than badge-ing it', async () => {- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'x', priority: 'URGENT', createdAt: '2026-06-03T09:00:00.000Z' }] });+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'x', priority: 'URGENT', createdAt: '2026-06-03T09:00:00.000Z' }] });const queue = await buildQueue();expect(queue.items.find(i => i.source === 'cos').meta).toBeUndefined();});