Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix([issue-12737]): release agent maintenance reservations on every agent exit and reconcile stale ones

Adam Eivyae38b2f7c6eb
diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.js
index 1b3f0363bd..296c2bb8b0 100644
--- a/server/lib/maintenanceAdmission.js
+++ b/server/lib/maintenanceAdmission.js
@@ -299,6 +299,17 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
});
return status();
};
+ // Snapshot for trusted lifecycle reconcilers; callers re-verify under releaseOperations.
+ const operations = () => read().operations.map(op => ({ ...op }));
+ // Removes only settled records still identical to what the caller inspected;
+ // anything that changed (or settled elsewhere) since then is left alone.
+ const releaseOperations = expected => transaction(state => {
+ const before = state.operations.length;
+ state.operations = state.operations.filter(op => op.unsettled
+ || !expected.some(seen => seen.id === op.id && seen.kind === op.kind
+ && seen.resource === op.resource && seen.pid === op.pid && seen.startedAt === op.startedAt));
+ return before - state.operations.length;
+ });
// Owning-service recovery only. Validation and durable receipt publication
// run under the same journal lock as the exact reservation removal.
const reconcileAbandonedAgent = ({ hold, agentId, validate, publish, replay }) => transaction(state => {
@@ -315,7 +326,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse
return receipt;
});
const { observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability } = exclusive;
- return { directory, status, held, assertOpen, admit, tryAdmit, recoverOwned, reconcilePublicationRefusal, run, currentId, finish, finishResource, withResource, markResourceUnsettled, markCurrentUnsettled, continueSettlement, begin, resume, events,
+ return { directory, status, held, assertOpen, admit, tryAdmit, recoverOwned, reconcilePublicationRefusal, operations, releaseOperations, run, currentId, finish, finishResource, withResource, markResourceUnsettled, markCurrentUnsettled, continueSettlement, begin, resume, events,
observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability, reconcileAbandonedAgent };
}
diff --git a/server/services/agentMaintenanceReservations.js b/server/services/agentMaintenanceReservations.js
new file mode 100644
index 0000000000..309239daab
--- /dev/null
+++ b/server/services/agentMaintenanceReservations.js
@@ -0,0 +1,40 @@
+/**
+ * Release path for `agent` maintenance reservations that no lifecycle owner
+ * will ever settle: the agent ended, was purged, or its owning process died.
+ *
+ * Leaf module (imports only the admission journal and pid probe) so both the
+ * state layer and the process layer can call it without an import cycle.
+ */
+import { maintenance } from '../lib/maintenanceAdmission.js';
+import { isPidAlive } from '../lib/heavyJobClaim.js';
+
+/** Idempotent, never throws: a journal needing recovery must not abort the caller's terminal transition. */
+export async function releaseAgentReservation(agentId) {
+ try { await maintenance.finishResource('agent', agentId); }
+ catch (err) { console.warn(`⚠️ Maintenance reservation for ${agentId} was not released: ${err.message}`); }
+}
+
+/**
+ * Drop `agent` reservations whose agent is completed or no longer in state AND
+ * whose admitting process is dead. A running/paused agent, a live owner pid and
+ * an `unsettled` (uncertain-cleanup) record are never touched; operator
+ * abandonment receipts stay the path for those.
+ *
+ * @param {{ agents: Array<{id: string, status?: string}>, admission?: object, pidAlive?: (pid:number)=>boolean }} input
+ * @returns {Promise<number>} reservations released
+ */
+export async function reconcileStaleAgentReservations({ agents, admission = maintenance, pidAlive = isPidAlive }) {
+ try {
+ const statusById = new Map(agents.map(agent => [agent.id, agent.status]));
+ const ended = id => !statusById.has(id) || statusById.get(id) === 'completed';
+ const stale = admission.operations().filter(op => op.kind === 'agent' && !op.unsettled
+ && ended(op.resource) && !pidAlive(op.pid));
+ if (stale.length === 0) return 0;
+ const released = admission.releaseOperations(stale);
+ if (released > 0) console.log(`🧹 Released ${released} stale agent maintenance reservation(s)`);
+ return released;
+ } catch (err) {
+ console.warn(`⚠️ Stale agent reservation reconcile skipped: ${err.message}`);
+ return 0;
+ }
+}
diff --git a/server/services/agentMaintenanceReservations.test.js b/server/services/agentMaintenanceReservations.test.js
new file mode 100644
index 0000000000..10d2d91093
--- /dev/null
+++ b/server/services/agentMaintenanceReservations.test.js
@@ -0,0 +1,69 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import * as fs from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+
+const gateRef = vi.hoisted(() => ({ gate: null }));
+vi.mock('../lib/maintenanceAdmission.js', async importOriginal => ({
+ ...(await importOriginal()),
+ maintenance: new Proxy({}, { get: (_t, key) => gateRef.gate[key] }),
+}));
+import { createMaintenanceAdmission } from '../lib/maintenanceAdmission.js';
+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
+
+let root;
+const resources = () => gateRef.gate.status().blockers.map(b => b.resource).sort();
+beforeEach(() => {
+ root = fs.mkdtempSync(join(tmpdir(), 'agent-reservations-'));
+ gateRef.gate = createMaintenanceAdmission(root);
+});
+afterEach(() => fs.rmSync(root, { recursive: true, force: true }));
+
+describe('releaseAgentReservation', () => {
+ it('drops only that agent reservation and is idempotent', async () => {
+ gateRef.gate.admit('agent', 'agent-a');
+ gateRef.gate.admit('agent', 'agent-b');
+ await releaseAgentReservation('agent-a');
+ await releaseAgentReservation('agent-a');
+ expect(resources()).toEqual(['agent-b']);
+ });
+
+ it('does not throw when the journal needs recovery', async () => {
+ gateRef.gate.admit('agent', 'agent-a');
+ fs.writeFileSync(join(gateRef.gate.directory, 'state.json'), '{not json');
+ await expect(releaseAgentReservation('agent-a')).resolves.toBeUndefined();
+ });
+});
+
+describe('reconcileStaleAgentReservations', () => {
+ const agents = [
+ { id: 'agent-done', status: 'completed' },
+ { id: 'agent-live', status: 'running' },
+ { id: 'agent-paused', status: 'paused' },
+ { id: 'agent-hurt', status: 'completed' },
+ ];
+ const seed = () => {
+ for (const id of ['agent-done', 'agent-gone', 'agent-live', 'agent-paused']) gateRef.gate.admit('agent', id);
+ gateRef.gate.admit('agent', 'agent-hurt').markUnsettled();
+ gateRef.gate.admit('provider', 'agent-done');
+ };
+
+ it('releases completed or absent agents with a dead owner and nothing else', async () => {
+ seed();
+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => false })).toBe(2);
+ expect(resources()).toEqual(['agent-done', 'agent-hurt', 'agent-live', 'agent-paused']);
+ expect(gateRef.gate.status().blockers.filter(b => b.resource === 'agent-done').map(b => b.kind)).toEqual(['provider']);
+ });
+
+ it('leaves every reservation whose owning process is alive', async () => {
+ seed();
+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => true })).toBe(0);
+ expect(resources()).toHaveLength(6);
+ });
+
+ it('skips quietly when the journal needs recovery', async () => {
+ seed();
+ fs.writeFileSync(join(gateRef.gate.directory, 'state.json'), '{not json');
+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => false })).toBe(0);
+ });
+});
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index 504ebae0e8..16c5255921 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -2,6 +2,7 @@ import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';
import { hasClaimFlowContract } from '../lib/claimFlowTaskTypes.js';
import { isFalsyMeta, isTruthyMeta } from '../lib/metadataFlags.js';
import { maintenance } from '../lib/maintenanceAdmission.js';
+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
/**
* Agent Management
*
@@ -98,6 +99,7 @@ async function terminateRunnerAgent(agentId, runnerFn, errorMessage, blockedReas
// flushRunnerOutputBatcher is a no-op if no batcher exists.
await flushRunnerOutputBatcher(agentId);
await completeAgent(agentId, { success: false, error: errorMessage });
+ await releaseAgentReservation(agentId);
const task = agentInfo?.task;
if (task) {
await updateTask(task.id, {
@@ -1417,10 +1419,12 @@ async function runCleanupOrphanedAgents() {
// reservation outlives the process and every later status read parses it.
// finishResource reads the journal synchronously and throws while it needs
// recovery; that must not abort the marker clear and stranded-agent sweep below.
- for (const agentId of retiredAgentIds) {
- try { await maintenance.finishResource('agent', agentId); }
- catch (err) { emitLog('warn', `Maintenance reservation for ${agentId} was not released: ${err.message}`, { agentId }); }
- }
+ for (const agentId of retiredAgentIds) await releaseAgentReservation(agentId);
+ // Backstop for reservations a previous process leaked on any other exit path.
+ // Sweep-start snapshot plus what this sweep retired: `completed` is terminal and a
+ // dead owner pid cannot belong to an agent spawned meanwhile, so no re-read is needed.
+ const retired = new Set(retiredAgentIds);
+ await reconcileStaleAgentReservations({ agents: agents.map(a => retired.has(a.id) ? { ...a, status: 'completed' } : a) });
// Consume the marker. Doing it AFTER the retry pass (not before) means a crash
// mid-sweep leaves it in place and the next boot still classifies these agents
diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.js
index bf8ac6c8fb..b827c079af 100644
--- a/server/services/cosAgentLifecycle.js
+++ b/server/services/cosAgentLifecycle.js
@@ -23,6 +23,7 @@ import { recordDomainUsage } from './domainUsage.js';
import { repairCodexTaskSummary } from './codexSummaryRepair.js';
import { loadAgentIndex, saveAgentIndex, getAgentDir, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';
import { isAgentHandoff } from '../lib/agentOutcome.js';
+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';
export async function registerAgent(agentId, taskId, metadata = {}) {
return withStateLock(async () => {
@@ -924,6 +925,8 @@ export async function cleanupZombieAgents() {
for (const agent of finalized) {
cosEvents.emit('agent:completed', agent);
}
+ for (const agentId of summary.cleaned) await releaseAgentReservation(agentId);
+ await reconcileStaleAgentReservations({ agents: await getAgents() });
return summary;
}
@@ -940,6 +943,7 @@ export async function deleteAgent(agentId) {
throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });
}
+ const wasRunning = state.agents[agentId]?.status === 'running';
delete state.agents[agentId];
await saveState(state);
@@ -955,6 +959,8 @@ export async function deleteAgent(agentId) {
}
cosEvents.emit('agents:changed', { action: 'deleted', agentId });
+ // A purged record can never settle its own reservation; a live run keeps its hold.
+ if (!wasRunning) await releaseAgentReservation(agentId);
return { success: true, agentId };
}));
}
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index ff7958f8fa..0943d03b57 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -98,6 +98,7 @@ import { saveState, loadState, readAgentsStateForSafetyCheck } from './cosState.
import { recordDomainUsage } from './domainUsage.js';
import { cosEvents } from './cosEvents.js';
import { loadAgentIndex } from './cosAgentIndex.js';
+import { maintenance } from '../lib/maintenanceAdmission.js';
import { getActiveAgentsFromRunner, requestSecurityScanViaRunner } from './cosRunnerClient.js';
import { persistAgentSecurityScan } from './agentSecurityScan.js';
import { captureCompletedCheckouts } from './completedCheckoutEvidence.js';
@@ -1093,3 +1094,42 @@ describe('deleteAgent directory cleanup (#11446)', () => {
}
});
});
+
+describe('agent maintenance reservations on terminal transitions (#12737)', () => {
+ const reserved = (agentId) => maintenance.status().blockers.some(b => b.kind === 'agent' && b.resource === agentId);
+ beforeEach(async () => {
+ await rm(mockCosState.agentsDir, { recursive: true, force: true });
+ await mkdir(mockCosState.agentsDir, { recursive: true });
+ await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');
+ mockCosState.state = { agents: {} };
+ vi.mocked(getActiveAgentsFromRunner).mockResolvedValue([]);
+ activeAgents.clear();
+ runnerAgents.clear();
+ spawningTasks.clear();
+ });
+ afterEach(async () => {
+ for (const id of ['agent-zombie', 'agent-purged', 'agent-live']) await maintenance.finishResource('agent', id);
+ await rm(mockCosState.agentsDir, { recursive: true, force: true });
+ });
+
+ it('releases the reservation of a reaped zombie', async () => {
+ maintenance.admit('agent', 'agent-zombie');
+ mockCosState.state.agents['agent-zombie'] = {
+ id: 'agent-zombie', status: 'running', pid: 2147483646,
+ startedAt: new Date(Date.now() - 60_000).toISOString(),
+ };
+ expect((await cleanupZombieAgents()).cleaned).toEqual(['agent-zombie']);
+ expect(reserved('agent-zombie')).toBe(false);
+ });
+
+ it('releases the reservation of a purged completed agent but keeps a running one', async () => {
+ maintenance.admit('agent', 'agent-purged');
+ maintenance.admit('agent', 'agent-live');
+ mockCosState.state.agents['agent-purged'] = { id: 'agent-purged', status: 'completed' };
+ mockCosState.state.agents['agent-live'] = { id: 'agent-live', status: 'running' };
+ await deleteAgent('agent-purged');
+ await deleteAgent('agent-live');
+ expect(reserved('agent-purged')).toBe(false);
+ expect(reserved('agent-live')).toBe(true);
+ });
+});
diff --git a/server/services/subAgentSpawner.js b/server/services/subAgentSpawner.js
index a32b7b0bc3..662d0f1442 100644
--- a/server/services/subAgentSpawner.js
+++ b/server/services/subAgentSpawner.js
@@ -52,6 +52,7 @@ import { loadState } from './cosState.js';
import { acquireLocalEndpointSpawnSlot } from './cosLocalEndpointSlots.js';
import { forgeSpawnHoldReason } from './cosForgeSpawnGate.js';
import { acquireCosGlobalSlot } from './cosAdmissionReservations.js';
+import { releaseAgentReservation } from './agentMaintenanceReservations.js';
// This module's own event wiring drives three LIFECYCLE TRANSITIONS, so it takes
// them from the facade rather than from the three separate leaves that happen to
// implement them (#3450). It can: nothing the facade imports imports this module
@@ -443,6 +444,7 @@ async function runInitSpawner() {
// criterion was evaluated, so validation is the null sentinel (issue
// #2344), never a false that would look like a declared-and-failed run.
await completeAgent(agentId, { success: false, validationPassed: null, error });
+ await releaseAgentReservation(agentId);
await completeAgentRun(agent.runId, '', 1, 0, { message: error, category: 'runner-error' });
runnerAgents.delete(agentId);
}