Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: cut server memory churn and release reservations of restart-interrupted agents

Adam Eivy779d32012643
diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.js
index 1c212f87cd..1b3f0363bd 100644
--- a/server/lib/maintenanceAdmission.js
+++ b/server/lib/maintenanceAdmission.js
@@ -59,9 +59,26 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
return schema.parse(JSON.parse(io.readFileSync(file, 'utf8')));
} catch { throw unavailable(); }
};
+ // status()/held() are polled several times a second (media drain loop, spawn
+ // gates); re-parsing a journal that carries every settled operation allocated
+ // ~400 KB per call. Writers publish by renaming a new file into place, so any
+ // change gives it a new identity; the stat is taken before the read, so a
+ // racing publish can only cost a re-parse. Read-only: transactions mutate the
+ // state they read and must keep calling read() for a private copy.
+ let statusCache = null;
+ const readForStatus = () => {
+ if (uncertain) throw unavailable();
+ let stats;
+ try { stats = io.lstatSync(file); }
+ catch (err) { if (err.code === 'ENOENT') return empty(); throw unavailable(); }
+ if (!stats.isFile()) throw unavailable();
+ const key = `${stats.ino}:${stats.size}:${stats.mtimeMs}:${stats.ctimeMs}`;
+ if (statusCache?.key !== key) statusCache = { key, state: read() };
+ return statusCache.state;
+ };
const project = state => ({
state: state.hold ? (state.operations.length || state.exclusive ? 'draining' : 'ready') : 'normal',
- revision: state.revision, hold: state.hold,
+ revision: state.revision, hold: state.hold && { ...state.hold },
blockers: [...state.operations.map(({ kind, resource, startedAt, pid, unsettled }) => ({ kind, resource, startedAt, pid, unsettled })),
...(state.exclusive ? [{ kind: 'exclusive-maintenance', resource: state.exclusive.operation.operationId,
startedAt: state.exclusive.claimedAt, unsettled: state.exclusive.phase === 'uncertain' }] : [])],
@@ -70,7 +87,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
const status = () => {
try {
if (exists(lock)) throw unavailable();
- return project(read());
+ return project(readForStatus());
} catch {
return { state: 'unavailable', revision: null, hold: null, blockers: [],
error: 'Maintenance state needs recovery. New work is held; readiness is unknown.' };
diff --git a/server/lib/maintenanceAdmission.test.js b/server/lib/maintenanceAdmission.test.js
index db382835dd..a3ae761ae3 100644
--- a/server/lib/maintenanceAdmission.test.js
+++ b/server/lib/maintenanceAdmission.test.js
@@ -163,6 +163,30 @@ describe('durable graceful maintenance', () => {
expect(fs.readFileSync(file, 'utf8')).toBe(before);
});
+ it('serves status from one parse until the journal file changes', () => {
+ const { root } = gate();
+ let reads = 0;
+ const io = { ...fs, readFileSync: (...args) => { reads++; return fs.readFileSync(...args); } };
+ const admission = createMaintenanceAdmission(root, { io });
+ admission.admit('media', 'render-a');
+ const before = reads;
+ for (let i = 0; i < 5; i++) expect(admission.held()).toBe(false);
+ expect(reads - before).toBeLessThanOrEqual(1);
+ const settled = reads;
+ admission.status();
+ expect(reads).toBe(settled);
+
+ // A second process publishing (here: a second admission over the same root) is seen at once.
+ const other = createMaintenanceAdmission(root);
+ const hold = other.begin({ reason: 'Drain', owner: 'Operator' }).hold;
+ expect(admission.held()).toBe(true);
+ // Callers cannot corrupt the cached journal through the projection they receive.
+ admission.status().hold.reason = 'tampered';
+ expect(admission.status().hold.reason).toBe('Drain');
+ other.resume({ id: hold.id, revision: hold.revision });
+ expect(admission.held()).toBe(false);
+ });
+
it('keeps journal write failure unavailable without claiming recovered completion', () => {
const { root } = gate();
let fail = false;
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index 36073b7254..504ebae0e8 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -1180,6 +1180,10 @@ async function runCleanupOrphanedAgents() {
const agents = await getAgents();
let cleanedCount = 0;
const orphanedTaskIds = [];
+ // Agents retired by this sweep, with or without a task: each still holds the
+ // maintenance reservation its original spawn took, and retireDeadAgent does
+ // not release it.
+ const retiredAgentIds = [];
// Agents the PREVIOUS process owned when it was signalled down (#3202). Their
// processes died because PortOS restarted, not because they failed — so they
@@ -1356,6 +1360,7 @@ async function runCleanupOrphanedAgents() {
agentResultExtra: { interruptedByRestart: interrupted },
});
cleanedCount++;
+ retiredAgentIds.push(agent.id);
if (agent.taskId) {
// Carry the agent's metadata forward rather than re-reading the record
@@ -1407,6 +1412,16 @@ async function runCleanupOrphanedAgents() {
});
}
+ // Same release order as a normal completion (agentLifecycle): only after the
+ // run is retired, its worktree cleaned and its task settled. Without it the
+ // reservation outlives the process and every later status read parses it.
+ // finishResource reads the journal synchronously and throws while it needs
+ // recovery; that must not abort the marker clear and stranded-agent sweep below.
+ for (const agentId of retiredAgentIds) {
+ try { await maintenance.finishResource('agent', agentId); }
+ catch (err) { emitLog('warn', `Maintenance reservation for ${agentId} was not released: ${err.message}`, { agentId }); }
+ }
+
// Consume the marker. Doing it AFTER the retry pass (not before) means a crash
// mid-sweep leaves it in place and the next boot still classifies these agents
// correctly; re-reading a stale marker is harmless because the sweep only ever
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index aab4d284f7..bbc649c922 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -120,6 +120,13 @@ vi.mock('../lib/hostShutdown.js', async (importOriginal) => ({
readHostShutdownMarker: vi.fn().mockResolvedValue(null),
clearHostShutdownMarker: vi.fn().mockResolvedValue(undefined),
}));
+// Real admission with only the settlement release observable: the default instance
+// resolves the live data root, so a spy keeps this suite off real journal state.
+vi.mock('../lib/maintenanceAdmission.js', async (importOriginal) => {
+ const actual = await importOriginal();
+ const finishResource = vi.fn().mockResolvedValue([]);
+ return { ...actual, maintenance: new Proxy(actual.maintenance, { get: (target, key) => key === 'finishResource' ? finishResource : target[key] }) };
+});
vi.mock('./agentRunnerSync.js', () => ({ syncRunnerAgents: vi.fn().mockResolvedValue(0) }));
vi.mock('./agentRunnerOutputBatchers.js', () => ({ flushRunnerOutputBatcher: vi.fn().mockResolvedValue(undefined) }));
vi.mock('./worktreeManager.js', () => ({ cleanupOrphanedWorktrees: vi.fn() }));
@@ -132,6 +139,7 @@ vi.mock('./creativeDirector/completionHook.js', () => ({ advanceAfterSceneSettle
import { handleOrphanedTask, pauseAgent, resumeAgent, relaunchAgent, settleOrphanedCreativeDirectorRun, cleanupOrphanedAgents, terminateAgent, killAgent, getAgentProcessStats } from './agentManagement.js';
import { retireDeadAgent, stampLiExecutionVerdict } from './agentFinalization.js';
+import { maintenance } from '../lib/maintenanceAdmission.js';
import { cleanupAgentWorktree, resolveTaskResumePatch } from './agentWorktreeCleanup.js';
import { getAgents, updateAgent, getAgentRecord, readAgentRecordOrUnreadable, AGENT_RECORD_UNREADABLE, completeAgent as markAgentComplete } from './cosAgentLifecycle.js';
import { updateRun, getProject } from './creativeDirector/local.js';
@@ -256,6 +264,36 @@ describe('cleanupOrphanedAgents — startup recovery coordination', () => {
expect(retireDeadAgent).toHaveBeenCalledWith(expect.objectContaining({ agent: expect.objectContaining({ id: 'agent-old' }) }));
});
+ it('releases the maintenance reservation of every agent the sweep retires, task or not', async () => {
+ getActiveAgentsFromRunner.mockResolvedValue([]);
+ getTaskById.mockResolvedValue({ id: 'task-1', taskType: 'user', status: 'in_progress', metadata: {} });
+ const old = new Date(Date.now() - 120000).toISOString();
+ getAgents.mockResolvedValueOnce([
+ { id: 'agent-with-task', status: 'running', taskId: 'task-1', startedAt: old, metadata: {} },
+ { id: 'agent-no-task', status: 'running', startedAt: old, metadata: {} },
+ ]);
+ maintenance.finishResource.mockClear();
+ await cleanupOrphanedAgents();
+ expect(retireDeadAgent).toHaveBeenCalledTimes(2);
+ expect(maintenance.finishResource.mock.calls).toEqual([['agent', 'agent-with-task'], ['agent', 'agent-no-task']]);
+ // Release follows the task settlement, as in a normal completion.
+ expect(cleanupAgentWorktree.mock.invocationCallOrder[0]).toBeLessThan(maintenance.finishResource.mock.invocationCallOrder[0]);
+ });
+
+ it('finishes the sweep when a maintenance release fails', async () => {
+ getActiveAgentsFromRunner.mockResolvedValue([]);
+ const old = new Date(Date.now() - 120000).toISOString();
+ getAgents.mockResolvedValueOnce([
+ { id: 'agent-first', status: 'running', startedAt: old, metadata: {} },
+ { id: 'agent-second', status: 'running', startedAt: old, metadata: {} },
+ ]);
+ maintenance.finishResource.mockClear();
+ maintenance.finishResource.mockImplementationOnce(() => { throw new Error('Maintenance state needs recovery'); });
+ await expect(cleanupOrphanedAgents()).resolves.toBe(2);
+ // The failed release neither stops the next one nor the rest of the sweep.
+ expect(maintenance.finishResource.mock.calls).toEqual([['agent', 'agent-first'], ['agent', 'agent-second']]);
+ });
+
it('reaps a durable running record whose runner listing is stale', async () => {
getAgents.mockResolvedValueOnce([{
id: 'agent-stale',
diff --git a/server/services/cosDequeue.js b/server/services/cosDequeue.js
index 6276626e14..f3b557ccb7 100644
--- a/server/services/cosDequeue.js
+++ b/server/services/cosDequeue.js
@@ -277,8 +277,8 @@ export async function admitIdleReviewTask({
// Deferred: the generator imports this owner, and ordinary capacity users
// need neither the task store nor the prompt-generation dependency graph.
- const { getCosTasks } = await import('./cosTaskStore.js');
- if ((await getCosTasks()).autoApproved?.length) return;
+ const { getCosApprovalView } = await import('./cosTaskStore.js');
+ if ((await getCosApprovalView()).autoApprovedCount) return;
const { generateIdleReviewTask, recordDeferredPerpetualDispatch } = await import('./cosTaskGenerator.js');
const { task, pendingPerpetualDispatch, preflightCardId } = await generateIdleReviewTask(state, {
ignoreTaskId,
diff --git a/server/services/cosIdleReviewParity.test.js b/server/services/cosIdleReviewParity.test.js
index 5577812d93..1d4bc8f29f 100644
--- a/server/services/cosIdleReviewParity.test.js
+++ b/server/services/cosIdleReviewParity.test.js
@@ -31,6 +31,7 @@ vi.mock('./cosTaskStore.js', async (importActual) => ({
}),
getUserTasks: async () => ({ exists: true, grouped: { pending: mocks.userTasks } }),
getCosTasks: async () => mocks.cosTaskData,
+ getCosApprovalView: async () => ({ autoApprovedCount: mocks.cosTaskData.autoApproved?.length ?? 0 }),
reconcileLegacyForgeMaintenanceTasks: async () => ({ recovered: 0, retired: 0 }),
addTask: async (task) => { mocks.persisted.push(task); return task; },
}));
diff --git a/server/services/cosTaskStore.js b/server/services/cosTaskStore.js
index eb1aec90d6..d441a6d91f 100644
--- a/server/services/cosTaskStore.js
+++ b/server/services/cosTaskStore.js
@@ -452,6 +452,24 @@ export async function getTaskDiagnostics() {
return structuredClone(sources.flat());
}
+/**
+ * Approval-state view of the internal queue for pollers that need the awaiting
+ * list or the auto-approved count, not the whole queue. `getCosTasks()` clones
+ * every archived prompt body to answer the same question. Derived once per
+ * snapshot; only the first `limit` awaiting tasks are cloned.
+ */
+export async function getCosApprovalView({ limit = Infinity } = {}) {
+ const filePath = await resolveSourceFile('internal');
+ if (!filePath) return { awaitingApproval: [], autoApprovedCount: 0 };
+ const snapshot = await readTaskSnapshot(filePath);
+ snapshot.approval ??= {
+ awaiting: getAwaitingApprovalTasks(snapshot.tasks),
+ autoApprovedCount: getAutoApprovedTasks(snapshot.tasks).length,
+ };
+ const { awaiting, autoApprovedCount } = snapshot.approval;
+ return { awaitingApproval: structuredClone(awaiting.slice(0, limit)), autoApprovedCount };
+}
+
/**
* Alias for backward compatibility
*/
diff --git a/server/services/cosTaskStore.test.js b/server/services/cosTaskStore.test.js
index 7c2a354ce1..ec04205b86 100644
--- a/server/services/cosTaskStore.test.js
+++ b/server/services/cosTaskStore.test.js
@@ -131,6 +131,7 @@ import {
getAllTasks,
getTaskView,
getPendingTaskIds,
+ getCosApprovalView,
getTaskDiagnostics,
getTasks,
getTaskById,
@@ -388,6 +389,29 @@ describe('cosTaskStore parsed-task cache (#3497)', () => {
expect(await getPendingTaskIds()).toEqual([]);
});
+ it('projects approval state without cloning the queue and refreshes after writes', async () => {
+ expect(await getCosApprovalView()).toEqual({ awaitingApproval: [], autoApprovedCount: 0 });
+ await addTask({ description: 'auto sys', approvalRequired: false }, 'internal');
+ await addTask({ description: 'needs approval', approvalRequired: true }, 'internal');
+ await addTask({ description: 'also needs approval', approvalRequired: true }, 'internal');
+
+ const expectedFirst = (await getCosTasks()).awaitingApproval[0].id;
+ const clone = vi.spyOn(globalThis, 'structuredClone');
+ let view;
+ let clonedPayload;
+ try { view = await getCosApprovalView({ limit: 1 }); clonedPayload = clone.mock.calls[0]?.[0]; } finally { clone.mockRestore(); }
+ expect(view.autoApprovedCount).toBe(1);
+ expect(view.awaitingApproval.map(t => t.id)).toEqual([expectedFirst]);
+ // Only the returned page is cloned, never the whole queue.
+ expect(clonedPayload).toHaveLength(1);
+
+ view.awaitingApproval[0].description = 'caller edit';
+ expect((await getCosApprovalView()).awaitingApproval[0].description).toBe('needs approval');
+
+ await addTask({ description: 'third approval', approvalRequired: true }, 'internal');
+ expect((await getCosApprovalView({ limit: 10 })).awaitingApproval).toHaveLength(3);
+ });
+
it('serves a cached parse while the file is unchanged', async () => {
await addTask({ description: 'cached thing' }, 'user');
mock.parseCalls = 0;
diff --git a/server/services/reviewQueue.js b/server/services/reviewQueue.js
index 66977e91ac..e2d26e87cd 100644
--- a/server/services/reviewQueue.js
+++ b/server/services/reviewQueue.js
@@ -413,8 +413,8 @@ const PRODUCERS = [
},
async gather(limit = REVIEW_QUEUE_SOURCE_READ_LIMIT) {
// Internal CoS tasks parked awaiting the user's approval before they run.
- const { awaitingApproval = [] } = await cosTaskStore.getCosTasks();
- return (Array.isArray(awaitingApproval) ? awaitingApproval : []).slice(0, limit);
+ const { awaitingApproval } = await cosTaskStore.getCosApprovalView({ limit });
+ return awaitingApproval;
},
map(task) {
// Surface the task priority as a triage badge. Only HIGH/MEDIUM/LOW are
diff --git a/server/services/reviewQueue.test.js b/server/services/reviewQueue.test.js
index 99f9ccc7c8..e626cf695d 100644
--- a/server/services/reviewQueue.test.js
+++ b/server/services/reviewQueue.test.js
@@ -7,7 +7,7 @@ vi.mock('./apps.js', () => apps);
const brain = { getInboxLog: vi.fn(), markInboxDone: vi.fn() };
const brainStorage = { getThreads: vi.fn(), getThreadById: vi.fn(), updateWith: vi.fn() };
const askConversations = { listConversations: vi.fn() };
-const cosTaskStore = { getCosTasks: vi.fn(), approveTask: vi.fn(), getTaskById: vi.fn() };
+const cosTaskStore = { getCosApprovalView: vi.fn(), approveTask: vi.fn(), getTaskById: vi.fn() };
const cosAgentFeedback = { getPendingAgentFeedback: vi.fn(), submitAgentFeedback: vi.fn() };
const messageDrafts = { listDrafts: vi.fn(), approveDraft: vi.fn() };
const proactiveAlerts = { generateNonProductAlerts: vi.fn(), resolveHealthAlert: vi.fn() };
@@ -86,7 +86,7 @@ function resetEmpty() {
brainStorage.getThreadById.mockResolvedValue(null);
brainStorage.updateWith.mockResolvedValue(null);
askConversations.listConversations.mockResolvedValue([]);
- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [] });
+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [] });
cosAgentFeedback.getPendingAgentFeedback.mockResolvedValue({ agents: [], unavailable: [], count: 0 });
cosAgentFeedback.submitAgentFeedback.mockResolvedValue({ success: true });
messageDrafts.listDrafts.mockResolvedValue([]);
@@ -517,7 +517,7 @@ describe('reviewQueue.buildQueue', () => {
});
it('surfaces drafts and CoS approvals from their producers', async () => {
- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });
+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });
// The producer pushes a multi-status filter down to listDrafts, so the mock
// honors it the way the real implementation does (drops the 'sent' draft).
const allDrafts = [
@@ -964,7 +964,7 @@ describe('reviewQueue.buildQueue', () => {
it('tags resolvable rows with an inline action verb, leaves no-clean-resolve sources without one', async () => {
brain.getInboxLog.mockResolvedValue([{ id: 'b1', capturedText: 'classify', capturedAt: '2026-06-03T10:00:00.000Z' }]);
- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });
+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'HIGH', createdAt: '2026-06-03T09:00:00.000Z' }] });
askConversations.listConversations.mockResolvedValue([{ id: 'a1', title: 'promote me', promoted: false, turnCount: 1, assistantTurnCount: 1 }]);
proactiveAlerts.generateNonProductAlerts.mockResolvedValue([{ id: 'disk:primary', type: 'disk', severity: 'critical', title: 'crit', detail: 'full', link: '/apps' }]);
backup.getState.mockResolvedValue({ status: 'error', error: 'disk full' });
@@ -982,7 +982,7 @@ describe('reviewQueue.buildQueue', () => {
it('attaches source-appropriate meta chips when the raw field is present', async () => {
brain.getInboxLog.mockResolvedValue([{ id: 'b1', capturedText: 'classify', source: 'voice', capturedAt: '2026-06-03T10:00:00.000Z' }]);
askConversations.listConversations.mockResolvedValue([{ id: 'a1', title: 'promote me', promoted: false, turnCount: 4, assistantTurnCount: 2 }]);
- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'MEDIUM', createdAt: '2026-06-03T09:00:00.000Z' }] });
+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'approve me', priority: 'MEDIUM', createdAt: '2026-06-03T09:00:00.000Z' }] });
messageDrafts.listDrafts.mockResolvedValue([{ id: 'd1', status: 'draft', subject: 's', to: ['[email protected]'], sendVia: 'gmail', updatedAt: '2026-06-03T08:00:00.000Z' }]);
proactiveAlerts.generateNonProductAlerts.mockResolvedValue([{ id: 'system_resource:memory', type: 'system_resource', severity: 'critical', title: 'mem', detail: 'high', link: '/apps' }]);
const queue = await buildQueue();
@@ -1004,7 +1004,7 @@ describe('reviewQueue.buildQueue', () => {
});
it('drops an out-of-range cos priority rather than badge-ing it', async () => {
- cosTaskStore.getCosTasks.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'x', priority: 'URGENT', createdAt: '2026-06-03T09:00:00.000Z' }] });
+ cosTaskStore.getCosApprovalView.mockResolvedValue({ awaitingApproval: [{ id: 'sys-1', description: 'x', priority: 'URGENT', createdAt: '2026-06-03T09:00:00.000Z' }] });
const queue = await buildQueue();
expect(queue.items.find(i => i.source === 'cos').meta).toBeUndefined();
});