Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: protect runner completion from premature zombie cleanup
Adam Eivyf704d05badd8
diff --git a/server/services/agentCompletionRouting.test.js b/server/services/agentCompletionRouting.test.jsindex 18d88aa232..71967538bc 100644--- a/server/services/agentCompletionRouting.test.js+++ b/server/services/agentCompletionRouting.test.js@@ -64,7 +64,7 @@ vi.mock('./agentCompletionCleanup.js', () => ({}));import { handleAgentCompletion } from './agentLifecycle.js';-import { runnerAgents, pausedAgents } from './agentState.js';+import { runnerAgents, completingAgents, pausedAgents } from './agentState.js';import { completeAgent } from './cosAgentLifecycle.js';import { getAgent, getAgentRecord, getTaskById, updateTask } from './cos.js';import { retireDeadAgent } from './agentFinalization.js';@@ -202,3 +202,26 @@ describe('handleAgentCompletion — post-restart recovery hand-off', () => {expect(completeAgent).not.toHaveBeenCalled();});});++// A recovered TUI's process has exited, but output hooks/validation still own+// its verdict. Watchdogs must see that ownership through success AND failure.+describe('runner completion ownership', () => {+ it.each([false, true])('holds ownership until retirement settles (reject=%s)', async (reject) => {+ let settle;+ const pending = new Promise((resolve, rejectPromise) => { settle = reject ? rejectPromise : resolve; });+ getAgent.mockResolvedValue({ id: 'agent-finishing', status: 'running', metadata: {} });+ retireDeadAgent.mockImplementationOnce(async () => {+ expect(completingAgents.has('agent-finishing')).toBe(true);+ await pending;+ return { success: true };+ });+ const done = handleAgentCompletion('agent-finishing', 0, true, 1000);+ const assertion = reject ? expect(done).rejects.toThrow('write failed') : expect(done).resolves.toBeUndefined();+ expect(completingAgents.has('agent-finishing')).toBe(true);+ await handleAgentCompletion('agent-finishing', 0, true, 1000);+ settle(reject ? new Error('write failed') : undefined);+ await assertion;+ expect(retireDeadAgent).toHaveBeenCalledTimes(1);+ expect(completingAgents.has('agent-finishing')).toBe(false);+ });+});diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.jsindex 73257c1755..83fc1ba74e 100644--- a/server/services/agentLifecycle.js+++ b/server/services/agentLifecycle.js@@ -48,7 +48,7 @@ import { cliProviderAuthDescriptor } from '../lib/processEnv.js';import { isClaudeCliProvider, getClaudeSettingsEnv } from './agentCliSpawning.js';import { resolveForgeTokenEnv } from './forgeAuth.js';import { resolveAgentApiEnv } from './agentApiAuth.js';-import { runnerAgents, pausedAgents, consumePausedAgentExit, spawningTasks, isTruthyMeta } from './agentState.js';+import { runnerAgents, completingAgents, pausedAgents, consumePausedAgentExit, spawningTasks, isTruthyMeta } from './agentState.js';import { withSpawnDedupGuard, withMapEntryCleanup, withUpdateInProgressGuard, SPAWN_DEDUP_SKIP, SPAWN_UPDATE_SKIP } from './agentGuards.js';import { isUpdateInProgress } from './updateChecker.js';import { createAgentSpawnContext, prepareAgentSpawn } from './agentSpawnPreparation.js';@@ -499,10 +499,17 @@ async function completeUntrackedAgentFromCosState(agentId, exitCode, success, du* finalization runs against a live agent.*/export async function handleAgentCompletion(agentId, exitCode, success, duration) {- return maintenance.withResource('agent', agentId, async () => {- try { return await handleAdmittedAgentCompletion(agentId, exitCode, success, duration); }- catch (err) { maintenance.markResourceUnsettled('agent', agentId); throw err; }- });+ if (completingAgents.has(agentId)) return;+ // Claim before admission's first await: the process may already be gone.+ completingAgents.add(agentId);+ try {+ return await maintenance.withResource('agent', agentId, async () => {+ try { return await handleAdmittedAgentCompletion(agentId, exitCode, success, duration); }+ catch (err) { maintenance.markResourceUnsettled('agent', agentId); throw err; }+ });+ } finally {+ completingAgents.delete(agentId);+ }}async function handleAdmittedAgentCompletion(agentId, exitCode, success, duration) {diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex 27dbb47df6..fabb57ca9c 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -26,7 +26,7 @@ import { terminateAgentViaRunner, killAgentViaRunner, pauseAgentViaRunner, getAgimport { runnerEntryShieldsRunningRecord } from '../lib/runnerAgentLiveness.js';import { MAX_TOTAL_SPAWNS } from '../lib/validation.js';import { isInternalTaskId } from '../lib/taskParser.js';-import { activeAgents, runnerAgents, spawningTasks, userTerminatedAgents, pausedAgents, whenPausedAgentExits, useRunner, unregisterSpawnedAgent } from './agentState.js';+import { activeAgents, runnerAgents, completingAgents, spawningTasks, userTerminatedAgents, pausedAgents, whenPausedAgentExits, useRunner, unregisterSpawnedAgent } from './agentState.js';// Both were extracted out of agentLifecycle.js (issue #2837) so this module no// longer depends on the lifecycle orchestrator — which depends on THIS module// for handleOrphanedTask. Importing them from their own leaf modules is what@@ -1228,6 +1228,7 @@ async function runCleanupOrphanedAgents() {for (const agent of agents) {if (agent.status === 'running') {+ if (completingAgents.has(agent.id)) continue;const inRemoteRunner = await runnerEntryShieldsRunningRecord(runnerById.get(agent.id),isPidAlive,@@ -1279,6 +1280,9 @@ async function runCleanupOrphanedAgents() {}}+ // Recheck after the awaited pid probes: completion owns its terminal write.+ if (completingAgents.has(agent.id)) continue;+const interrupted = interruptedByRestart.has(agent.id);const errorMessage = interrupted? 'Agent was interrupted by a PortOS server restart'diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.jsindex f60ea36eb4..ad5735922e 100644--- a/server/services/agentManagement.test.js+++ b/server/services/agentManagement.test.js@@ -142,7 +142,7 @@ import { pauseAgentViaRunner, terminateAgentViaRunner, getActiveAgentsFromRunnerimport * as shellService from './shell.js';import { readHostShutdownMarker, clearHostShutdownMarker } from '../lib/hostShutdown.js';import { committedDuringRun } from '../lib/gitCommitProbe.js';-import { activeAgents, runnerAgents, spawningTasks, pausedAgents, consumePausedAgentExit } from './agentState.js';+import { activeAgents, runnerAgents, completingAgents, spawningTasks, pausedAgents, consumePausedAgentExit } from './agentState.js';/*** A direct-mode agent's spawned handle. Its prototype is ChildProcess so it@@ -1715,6 +1715,18 @@ describe('orphan retries resume what the dead run left behind', () => {// The sweep is the only place that knows an agent DIED rather than exited, so// it is the only place that can put that distinction in the ledger (#4540).+ it('does not retire or retry an exited runner while completion owns its outcome', async () => {+ completingAgents.add(deadAgent.id);+ try {+ await cleanupOrphanedAgents();+ expect(retireDeadAgent).not.toHaveBeenCalled();+ expect(resolveTaskResumePatch).not.toHaveBeenCalled();+ expect(updateTask).not.toHaveBeenCalled();+ } finally {+ completingAgents.delete(deadAgent.id);+ }+ });+it('records the reap in the lifecycle ledger, keyed by agent when no run id survived', async () => {await cleanupOrphanedAgents();diff --git a/server/services/agentState.js b/server/services/agentState.jsindex 31928b112d..4f7411d3c4 100644--- a/server/services/agentState.js+++ b/server/services/agentState.js@@ -13,6 +13,10 @@ export const activeAgents = new Map();// runnerAgents: runner-spawned agents (Map<agentId, { taskId, task, runId, model, ... }>)export const runnerAgents = new Map();+// Completion events can outlive the runner process (hooks, forge checks and+// persistence all await). Sweeps must not race them to the first terminal write.+export const completingAgents = new Set();+// userTerminatedAgents: agents the user explicitly killed (Set<agentId>)export const userTerminatedAgents = new Set();diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.jsindex dece6f5a85..d99e2db0c7 100644--- a/server/services/cosAgentLifecycle.js+++ b/server/services/cosAgentLifecycle.js@@ -666,7 +666,7 @@ export async function cleanupZombieAgents() {// Direct-spawn handles in this process (not leftover runnerAgents adopts —// those are only a handle, same as GET /agents). Read from the side-effect-free// state module, not subAgentSpawner.- const { activeAgents, spawningTasks } = await import('./agentState.js');+ const { activeAgents, completingAgents, spawningTasks } = await import('./agentState.js');// Also check with the CoS runner for agents it's actively trackingconst { getActiveAgentsFromRunner } = await import('./cosRunnerClient.js');@@ -728,6 +728,10 @@ export async function cleanupZombieAgents() {if (ageMs < 30000) continue;}+ // Completion may have started while the liveness probes awaited. Its+ // hook/validation/persistence chain owns the outcome even after PTY exit.+ if (completingAgents.has(agent.id)) continue;+// Agent is not tracked anywhere and process is dead — it's a zombie.// A record that never recorded a pid never spawned at all (the pid is// written only after a successful spawn), so "terminated unexpectedly"diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.jsindex 2fea48cfbd..5bdf335c4d 100644--- a/server/services/cosAgentLifecycle.test.js+++ b/server/services/cosAgentLifecycle.test.js@@ -95,7 +95,7 @@ import { cosEvents } from './cosEvents.js';import { loadAgentIndex } from './cosAgentIndex.js';import { getActiveAgentsFromRunner, requestSecurityScanViaRunner } from './cosRunnerClient.js';import { persistAgentSecurityScan } from './agentSecurityScan.js';-import { activeAgents, runnerAgents, spawningTasks } from './agentState.js';+import { activeAgents, runnerAgents, completingAgents, spawningTasks } from './agentState.js';afterAll(cleanupTempDataRoots);@@ -769,6 +769,22 @@ describe('cleanupZombieAgents — runner listing is not proof of life', () => {await rm(mockCosState.agentsDir, { recursive: true, force: true });});+ it('leaves an exited runner to its in-flight completion, then reaps if completion failed', async () => {+ await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');+ mockCosState.state.agents['agent-finishing'] = {+ id: 'agent-finishing', status: 'running', pid: 2147483646,+ startedAt: new Date(Date.now() - 60_000).toISOString(),+ };+ completingAgents.add('agent-finishing');+ try {+ expect((await cleanupZombieAgents()).cleaned).toEqual([]);+ expect(mockCosState.state.agents['agent-finishing'].status).toBe('running');+ } finally {+ completingAgents.delete('agent-finishing');+ }+ expect((await cleanupZombieAgents()).cleaned).toEqual(['agent-finishing']);+ });+it('does not reap a pid-less record while its task is still being dispatched', async () => {await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');mockCosState.state.agents['agent-slow'] = {