Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix([issue-12737]): release agent maintenance reservations on every agent exit and reconcile stale ones
Adam Eivyae38b2f7c6eb
diff --git a/server/lib/maintenanceAdmission.js b/server/lib/maintenanceAdmission.jsindex 1b3f0363bd..296c2bb8b0 100644--- a/server/lib/maintenanceAdmission.js+++ b/server/lib/maintenanceAdmission.js@@ -299,6 +299,17 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, asse});return status();};+ // Snapshot for trusted lifecycle reconcilers; callers re-verify under releaseOperations.+ const operations = () => read().operations.map(op => ({ ...op }));+ // Removes only settled records still identical to what the caller inspected;+ // anything that changed (or settled elsewhere) since then is left alone.+ const releaseOperations = expected => transaction(state => {+ const before = state.operations.length;+ state.operations = state.operations.filter(op => op.unsettled+ || !expected.some(seen => seen.id === op.id && seen.kind === op.kind+ && seen.resource === op.resource && seen.pid === op.pid && seen.startedAt === op.startedAt));+ return before - state.operations.length;+ });// Owning-service recovery only. Validation and durable receipt publication// run under the same journal lock as the exact reservation removal.const reconcileAbandonedAgent = ({ hold, agentId, validate, publish, replay }) => transaction(state => {@@ -315,7 +326,7 @@ export function createMaintenanceAdmission(dataDir = PATHS.data, { io = fs, assereturn receipt;});const { observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability } = exclusive;- return { directory, status, held, assertOpen, admit, tryAdmit, recoverOwned, reconcilePublicationRefusal, run, currentId, finish, finishResource, withResource, markResourceUnsettled, markCurrentUnsettled, continueSettlement, begin, resume, events,+ return { directory, status, held, assertOpen, admit, tryAdmit, recoverOwned, reconcilePublicationRefusal, operations, releaseOperations, run, currentId, finish, finishResource, withResource, markResourceUnsettled, markCurrentUnsettled, continueSettlement, begin, resume, events,observeIdle, claimReady, getExclusive, transitionExclusive, settleExclusive, issueExecutionCapability, reconcileAbandonedAgent };}diff --git a/server/services/agentMaintenanceReservations.js b/server/services/agentMaintenanceReservations.jsnew file mode 100644index 0000000000..309239daab--- /dev/null+++ b/server/services/agentMaintenanceReservations.js@@ -0,0 +1,40 @@+/**+ * Release path for `agent` maintenance reservations that no lifecycle owner+ * will ever settle: the agent ended, was purged, or its owning process died.+ *+ * Leaf module (imports only the admission journal and pid probe) so both the+ * state layer and the process layer can call it without an import cycle.+ */+import { maintenance } from '../lib/maintenanceAdmission.js';+import { isPidAlive } from '../lib/heavyJobClaim.js';++/** Idempotent, never throws: a journal needing recovery must not abort the caller's terminal transition. */+export async function releaseAgentReservation(agentId) {+ try { await maintenance.finishResource('agent', agentId); }+ catch (err) { console.warn(`⚠️ Maintenance reservation for ${agentId} was not released: ${err.message}`); }+}++/**+ * Drop `agent` reservations whose agent is completed or no longer in state AND+ * whose admitting process is dead. A running/paused agent, a live owner pid and+ * an `unsettled` (uncertain-cleanup) record are never touched; operator+ * abandonment receipts stay the path for those.+ *+ * @param {{ agents: Array<{id: string, status?: string}>, admission?: object, pidAlive?: (pid:number)=>boolean }} input+ * @returns {Promise<number>} reservations released+ */+export async function reconcileStaleAgentReservations({ agents, admission = maintenance, pidAlive = isPidAlive }) {+ try {+ const statusById = new Map(agents.map(agent => [agent.id, agent.status]));+ const ended = id => !statusById.has(id) || statusById.get(id) === 'completed';+ const stale = admission.operations().filter(op => op.kind === 'agent' && !op.unsettled+ && ended(op.resource) && !pidAlive(op.pid));+ if (stale.length === 0) return 0;+ const released = admission.releaseOperations(stale);+ if (released > 0) console.log(`🧹 Released ${released} stale agent maintenance reservation(s)`);+ return released;+ } catch (err) {+ console.warn(`⚠️ Stale agent reservation reconcile skipped: ${err.message}`);+ return 0;+ }+}diff --git a/server/services/agentMaintenanceReservations.test.js b/server/services/agentMaintenanceReservations.test.jsnew file mode 100644index 0000000000..10d2d91093--- /dev/null+++ b/server/services/agentMaintenanceReservations.test.js@@ -0,0 +1,69 @@+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import * as fs from 'node:fs';+import { join } from 'node:path';+import { tmpdir } from 'node:os';++const gateRef = vi.hoisted(() => ({ gate: null }));+vi.mock('../lib/maintenanceAdmission.js', async importOriginal => ({+ ...(await importOriginal()),+ maintenance: new Proxy({}, { get: (_t, key) => gateRef.gate[key] }),+}));+import { createMaintenanceAdmission } from '../lib/maintenanceAdmission.js';+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';++let root;+const resources = () => gateRef.gate.status().blockers.map(b => b.resource).sort();+beforeEach(() => {+ root = fs.mkdtempSync(join(tmpdir(), 'agent-reservations-'));+ gateRef.gate = createMaintenanceAdmission(root);+});+afterEach(() => fs.rmSync(root, { recursive: true, force: true }));++describe('releaseAgentReservation', () => {+ it('drops only that agent reservation and is idempotent', async () => {+ gateRef.gate.admit('agent', 'agent-a');+ gateRef.gate.admit('agent', 'agent-b');+ await releaseAgentReservation('agent-a');+ await releaseAgentReservation('agent-a');+ expect(resources()).toEqual(['agent-b']);+ });++ it('does not throw when the journal needs recovery', async () => {+ gateRef.gate.admit('agent', 'agent-a');+ fs.writeFileSync(join(gateRef.gate.directory, 'state.json'), '{not json');+ await expect(releaseAgentReservation('agent-a')).resolves.toBeUndefined();+ });+});++describe('reconcileStaleAgentReservations', () => {+ const agents = [+ { id: 'agent-done', status: 'completed' },+ { id: 'agent-live', status: 'running' },+ { id: 'agent-paused', status: 'paused' },+ { id: 'agent-hurt', status: 'completed' },+ ];+ const seed = () => {+ for (const id of ['agent-done', 'agent-gone', 'agent-live', 'agent-paused']) gateRef.gate.admit('agent', id);+ gateRef.gate.admit('agent', 'agent-hurt').markUnsettled();+ gateRef.gate.admit('provider', 'agent-done');+ };++ it('releases completed or absent agents with a dead owner and nothing else', async () => {+ seed();+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => false })).toBe(2);+ expect(resources()).toEqual(['agent-done', 'agent-hurt', 'agent-live', 'agent-paused']);+ expect(gateRef.gate.status().blockers.filter(b => b.resource === 'agent-done').map(b => b.kind)).toEqual(['provider']);+ });++ it('leaves every reservation whose owning process is alive', async () => {+ seed();+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => true })).toBe(0);+ expect(resources()).toHaveLength(6);+ });++ it('skips quietly when the journal needs recovery', async () => {+ seed();+ fs.writeFileSync(join(gateRef.gate.directory, 'state.json'), '{not json');+ expect(await reconcileStaleAgentReservations({ agents, pidAlive: () => false })).toBe(0);+ });+});diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex 504ebae0e8..16c5255921 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -2,6 +2,7 @@ import { isLegacyDeepAudit } from '../lib/auditWorkflow.js';import { hasClaimFlowContract } from '../lib/claimFlowTaskTypes.js';import { isFalsyMeta, isTruthyMeta } from '../lib/metadataFlags.js';import { maintenance } from '../lib/maintenanceAdmission.js';+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';/*** Agent Management*@@ -98,6 +99,7 @@ async function terminateRunnerAgent(agentId, runnerFn, errorMessage, blockedReas// flushRunnerOutputBatcher is a no-op if no batcher exists.await flushRunnerOutputBatcher(agentId);await completeAgent(agentId, { success: false, error: errorMessage });+ await releaseAgentReservation(agentId);const task = agentInfo?.task;if (task) {await updateTask(task.id, {@@ -1417,10 +1419,12 @@ async function runCleanupOrphanedAgents() {// reservation outlives the process and every later status read parses it.// finishResource reads the journal synchronously and throws while it needs// recovery; that must not abort the marker clear and stranded-agent sweep below.- for (const agentId of retiredAgentIds) {- try { await maintenance.finishResource('agent', agentId); }- catch (err) { emitLog('warn', `Maintenance reservation for ${agentId} was not released: ${err.message}`, { agentId }); }- }+ for (const agentId of retiredAgentIds) await releaseAgentReservation(agentId);+ // Backstop for reservations a previous process leaked on any other exit path.+ // Sweep-start snapshot plus what this sweep retired: `completed` is terminal and a+ // dead owner pid cannot belong to an agent spawned meanwhile, so no re-read is needed.+ const retired = new Set(retiredAgentIds);+ await reconcileStaleAgentReservations({ agents: agents.map(a => retired.has(a.id) ? { ...a, status: 'completed' } : a) });// Consume the marker. Doing it AFTER the retry pass (not before) means a crash// mid-sweep leaves it in place and the next boot still classifies these agentsdiff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.jsindex bf8ac6c8fb..b827c079af 100644--- a/server/services/cosAgentLifecycle.js+++ b/server/services/cosAgentLifecycle.js@@ -23,6 +23,7 @@ import { recordDomainUsage } from './domainUsage.js';import { repairCodexTaskSummary } from './codexSummaryRepair.js';import { loadAgentIndex, saveAgentIndex, getAgentDir, recordArchivedAgentOrder, moveAgentDir } from './cosAgentIndex.js';import { isAgentHandoff } from '../lib/agentOutcome.js';+import { reconcileStaleAgentReservations, releaseAgentReservation } from './agentMaintenanceReservations.js';export async function registerAgent(agentId, taskId, metadata = {}) {return withStateLock(async () => {@@ -924,6 +925,8 @@ export async function cleanupZombieAgents() {for (const agent of finalized) {cosEvents.emit('agent:completed', agent);}+ for (const agentId of summary.cleaned) await releaseAgentReservation(agentId);+ await reconcileStaleAgentReservations({ agents: await getAgents() });return summary;}@@ -940,6 +943,7 @@ export async function deleteAgent(agentId) {throw new ServerError('Agent not found', { status: 404, code: 'NOT_FOUND' });}+ const wasRunning = state.agents[agentId]?.status === 'running';delete state.agents[agentId];await saveState(state);@@ -955,6 +959,8 @@ export async function deleteAgent(agentId) {}cosEvents.emit('agents:changed', { action: 'deleted', agentId });+ // A purged record can never settle its own reservation; a live run keeps its hold.+ if (!wasRunning) await releaseAgentReservation(agentId);return { success: true, agentId };}));}diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.jsindex ff7958f8fa..0943d03b57 100644--- a/server/services/cosAgentLifecycle.test.js+++ b/server/services/cosAgentLifecycle.test.js@@ -98,6 +98,7 @@ import { saveState, loadState, readAgentsStateForSafetyCheck } from './cosState.import { recordDomainUsage } from './domainUsage.js';import { cosEvents } from './cosEvents.js';import { loadAgentIndex } from './cosAgentIndex.js';+import { maintenance } from '../lib/maintenanceAdmission.js';import { getActiveAgentsFromRunner, requestSecurityScanViaRunner } from './cosRunnerClient.js';import { persistAgentSecurityScan } from './agentSecurityScan.js';import { captureCompletedCheckouts } from './completedCheckoutEvidence.js';@@ -1093,3 +1094,42 @@ describe('deleteAgent directory cleanup (#11446)', () => {}});});++describe('agent maintenance reservations on terminal transitions (#12737)', () => {+ const reserved = (agentId) => maintenance.status().blockers.some(b => b.kind === 'agent' && b.resource === agentId);+ beforeEach(async () => {+ await rm(mockCosState.agentsDir, { recursive: true, force: true });+ await mkdir(mockCosState.agentsDir, { recursive: true });+ await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');+ mockCosState.state = { agents: {} };+ vi.mocked(getActiveAgentsFromRunner).mockResolvedValue([]);+ activeAgents.clear();+ runnerAgents.clear();+ spawningTasks.clear();+ });+ afterEach(async () => {+ for (const id of ['agent-zombie', 'agent-purged', 'agent-live']) await maintenance.finishResource('agent', id);+ await rm(mockCosState.agentsDir, { recursive: true, force: true });+ });++ it('releases the reservation of a reaped zombie', async () => {+ maintenance.admit('agent', 'agent-zombie');+ mockCosState.state.agents['agent-zombie'] = {+ id: 'agent-zombie', status: 'running', pid: 2147483646,+ startedAt: new Date(Date.now() - 60_000).toISOString(),+ };+ expect((await cleanupZombieAgents()).cleaned).toEqual(['agent-zombie']);+ expect(reserved('agent-zombie')).toBe(false);+ });++ it('releases the reservation of a purged completed agent but keeps a running one', async () => {+ maintenance.admit('agent', 'agent-purged');+ maintenance.admit('agent', 'agent-live');+ mockCosState.state.agents['agent-purged'] = { id: 'agent-purged', status: 'completed' };+ mockCosState.state.agents['agent-live'] = { id: 'agent-live', status: 'running' };+ await deleteAgent('agent-purged');+ await deleteAgent('agent-live');+ expect(reserved('agent-purged')).toBe(false);+ expect(reserved('agent-live')).toBe(true);+ });+});diff --git a/server/services/subAgentSpawner.js b/server/services/subAgentSpawner.jsindex a32b7b0bc3..662d0f1442 100644--- a/server/services/subAgentSpawner.js+++ b/server/services/subAgentSpawner.js@@ -52,6 +52,7 @@ import { loadState } from './cosState.js';import { acquireLocalEndpointSpawnSlot } from './cosLocalEndpointSlots.js';import { forgeSpawnHoldReason } from './cosForgeSpawnGate.js';import { acquireCosGlobalSlot } from './cosAdmissionReservations.js';+import { releaseAgentReservation } from './agentMaintenanceReservations.js';// This module's own event wiring drives three LIFECYCLE TRANSITIONS, so it takes// them from the facade rather than from the three separate leaves that happen to// implement them (#3450). It can: nothing the facade imports imports this module@@ -443,6 +444,7 @@ async function runInitSpawner() {// criterion was evaluated, so validation is the null sentinel (issue// #2344), never a false that would look like a declared-and-failed run.await completeAgent(agentId, { success: false, validationPassed: null, error });+ await releaseAgentReservation(agentId);await completeAgentRun(agent.runId, '', 1, 0, { message: error, category: 'runner-error' });runnerAgents.delete(agentId);}