Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: protect runner completion from premature zombie cleanup

Adam Eivyf704d05badd8
diff --git a/server/services/agentCompletionRouting.test.js b/server/services/agentCompletionRouting.test.js
index 18d88aa232..71967538bc 100644
--- a/server/services/agentCompletionRouting.test.js
+++ b/server/services/agentCompletionRouting.test.js
@@ -64,7 +64,7 @@ vi.mock('./agentCompletionCleanup.js', () => ({
}));
import { handleAgentCompletion } from './agentLifecycle.js';
-import { runnerAgents, pausedAgents } from './agentState.js';
+import { runnerAgents, completingAgents, pausedAgents } from './agentState.js';
import { completeAgent } from './cosAgentLifecycle.js';
import { getAgent, getAgentRecord, getTaskById, updateTask } from './cos.js';
import { retireDeadAgent } from './agentFinalization.js';
@@ -202,3 +202,26 @@ describe('handleAgentCompletion — post-restart recovery hand-off', () => {
expect(completeAgent).not.toHaveBeenCalled();
});
});
+
+// A recovered TUI's process has exited, but output hooks/validation still own
+// its verdict. Watchdogs must see that ownership through success AND failure.
+describe('runner completion ownership', () => {
+ it.each([false, true])('holds ownership until retirement settles (reject=%s)', async (reject) => {
+ let settle;
+ const pending = new Promise((resolve, rejectPromise) => { settle = reject ? rejectPromise : resolve; });
+ getAgent.mockResolvedValue({ id: 'agent-finishing', status: 'running', metadata: {} });
+ retireDeadAgent.mockImplementationOnce(async () => {
+ expect(completingAgents.has('agent-finishing')).toBe(true);
+ await pending;
+ return { success: true };
+ });
+ const done = handleAgentCompletion('agent-finishing', 0, true, 1000);
+ const assertion = reject ? expect(done).rejects.toThrow('write failed') : expect(done).resolves.toBeUndefined();
+ expect(completingAgents.has('agent-finishing')).toBe(true);
+ await handleAgentCompletion('agent-finishing', 0, true, 1000);
+ settle(reject ? new Error('write failed') : undefined);
+ await assertion;
+ expect(retireDeadAgent).toHaveBeenCalledTimes(1);
+ expect(completingAgents.has('agent-finishing')).toBe(false);
+ });
+});
diff --git a/server/services/agentLifecycle.js b/server/services/agentLifecycle.js
index 73257c1755..83fc1ba74e 100644
--- a/server/services/agentLifecycle.js
+++ b/server/services/agentLifecycle.js
@@ -48,7 +48,7 @@ import { cliProviderAuthDescriptor } from '../lib/processEnv.js';
import { isClaudeCliProvider, getClaudeSettingsEnv } from './agentCliSpawning.js';
import { resolveForgeTokenEnv } from './forgeAuth.js';
import { resolveAgentApiEnv } from './agentApiAuth.js';
-import { runnerAgents, pausedAgents, consumePausedAgentExit, spawningTasks, isTruthyMeta } from './agentState.js';
+import { runnerAgents, completingAgents, pausedAgents, consumePausedAgentExit, spawningTasks, isTruthyMeta } from './agentState.js';
import { withSpawnDedupGuard, withMapEntryCleanup, withUpdateInProgressGuard, SPAWN_DEDUP_SKIP, SPAWN_UPDATE_SKIP } from './agentGuards.js';
import { isUpdateInProgress } from './updateChecker.js';
import { createAgentSpawnContext, prepareAgentSpawn } from './agentSpawnPreparation.js';
@@ -499,10 +499,17 @@ async function completeUntrackedAgentFromCosState(agentId, exitCode, success, du
* finalization runs against a live agent.
*/
export async function handleAgentCompletion(agentId, exitCode, success, duration) {
- return maintenance.withResource('agent', agentId, async () => {
- try { return await handleAdmittedAgentCompletion(agentId, exitCode, success, duration); }
- catch (err) { maintenance.markResourceUnsettled('agent', agentId); throw err; }
- });
+ if (completingAgents.has(agentId)) return;
+ // Claim before admission's first await: the process may already be gone.
+ completingAgents.add(agentId);
+ try {
+ return await maintenance.withResource('agent', agentId, async () => {
+ try { return await handleAdmittedAgentCompletion(agentId, exitCode, success, duration); }
+ catch (err) { maintenance.markResourceUnsettled('agent', agentId); throw err; }
+ });
+ } finally {
+ completingAgents.delete(agentId);
+ }
}
async function handleAdmittedAgentCompletion(agentId, exitCode, success, duration) {
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index 27dbb47df6..fabb57ca9c 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -26,7 +26,7 @@ import { terminateAgentViaRunner, killAgentViaRunner, pauseAgentViaRunner, getAg
import { runnerEntryShieldsRunningRecord } from '../lib/runnerAgentLiveness.js';
import { MAX_TOTAL_SPAWNS } from '../lib/validation.js';
import { isInternalTaskId } from '../lib/taskParser.js';
-import { activeAgents, runnerAgents, spawningTasks, userTerminatedAgents, pausedAgents, whenPausedAgentExits, useRunner, unregisterSpawnedAgent } from './agentState.js';
+import { activeAgents, runnerAgents, completingAgents, spawningTasks, userTerminatedAgents, pausedAgents, whenPausedAgentExits, useRunner, unregisterSpawnedAgent } from './agentState.js';
// Both were extracted out of agentLifecycle.js (issue #2837) so this module no
// longer depends on the lifecycle orchestrator — which depends on THIS module
// for handleOrphanedTask. Importing them from their own leaf modules is what
@@ -1228,6 +1228,7 @@ async function runCleanupOrphanedAgents() {
for (const agent of agents) {
if (agent.status === 'running') {
+ if (completingAgents.has(agent.id)) continue;
const inRemoteRunner = await runnerEntryShieldsRunningRecord(
runnerById.get(agent.id),
isPidAlive,
@@ -1279,6 +1280,9 @@ async function runCleanupOrphanedAgents() {
}
}
+ // Recheck after the awaited pid probes: completion owns its terminal write.
+ if (completingAgents.has(agent.id)) continue;
+
const interrupted = interruptedByRestart.has(agent.id);
const errorMessage = interrupted
? 'Agent was interrupted by a PortOS server restart'
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index f60ea36eb4..ad5735922e 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -142,7 +142,7 @@ import { pauseAgentViaRunner, terminateAgentViaRunner, getActiveAgentsFromRunner
import * as shellService from './shell.js';
import { readHostShutdownMarker, clearHostShutdownMarker } from '../lib/hostShutdown.js';
import { committedDuringRun } from '../lib/gitCommitProbe.js';
-import { activeAgents, runnerAgents, spawningTasks, pausedAgents, consumePausedAgentExit } from './agentState.js';
+import { activeAgents, runnerAgents, completingAgents, spawningTasks, pausedAgents, consumePausedAgentExit } from './agentState.js';
/**
* A direct-mode agent's spawned handle. Its prototype is ChildProcess so it
@@ -1715,6 +1715,18 @@ describe('orphan retries resume what the dead run left behind', () => {
// The sweep is the only place that knows an agent DIED rather than exited, so
// it is the only place that can put that distinction in the ledger (#4540).
+ it('does not retire or retry an exited runner while completion owns its outcome', async () => {
+ completingAgents.add(deadAgent.id);
+ try {
+ await cleanupOrphanedAgents();
+ expect(retireDeadAgent).not.toHaveBeenCalled();
+ expect(resolveTaskResumePatch).not.toHaveBeenCalled();
+ expect(updateTask).not.toHaveBeenCalled();
+ } finally {
+ completingAgents.delete(deadAgent.id);
+ }
+ });
+
it('records the reap in the lifecycle ledger, keyed by agent when no run id survived', async () => {
await cleanupOrphanedAgents();
diff --git a/server/services/agentState.js b/server/services/agentState.js
index 31928b112d..4f7411d3c4 100644
--- a/server/services/agentState.js
+++ b/server/services/agentState.js
@@ -13,6 +13,10 @@ export const activeAgents = new Map();
// runnerAgents: runner-spawned agents (Map<agentId, { taskId, task, runId, model, ... }>)
export const runnerAgents = new Map();
+// Completion events can outlive the runner process (hooks, forge checks and
+// persistence all await). Sweeps must not race them to the first terminal write.
+export const completingAgents = new Set();
+
// userTerminatedAgents: agents the user explicitly killed (Set<agentId>)
export const userTerminatedAgents = new Set();
diff --git a/server/services/cosAgentLifecycle.js b/server/services/cosAgentLifecycle.js
index dece6f5a85..d99e2db0c7 100644
--- a/server/services/cosAgentLifecycle.js
+++ b/server/services/cosAgentLifecycle.js
@@ -666,7 +666,7 @@ export async function cleanupZombieAgents() {
// Direct-spawn handles in this process (not leftover runnerAgents adopts —
// those are only a handle, same as GET /agents). Read from the side-effect-free
// state module, not subAgentSpawner.
- const { activeAgents, spawningTasks } = await import('./agentState.js');
+ const { activeAgents, completingAgents, spawningTasks } = await import('./agentState.js');
// Also check with the CoS runner for agents it's actively tracking
const { getActiveAgentsFromRunner } = await import('./cosRunnerClient.js');
@@ -728,6 +728,10 @@ export async function cleanupZombieAgents() {
if (ageMs < 30000) continue;
}
+ // Completion may have started while the liveness probes awaited. Its
+ // hook/validation/persistence chain owns the outcome even after PTY exit.
+ if (completingAgents.has(agent.id)) continue;
+
// Agent is not tracked anywhere and process is dead — it's a zombie.
// A record that never recorded a pid never spawned at all (the pid is
// written only after a successful spawn), so "terminated unexpectedly"
diff --git a/server/services/cosAgentLifecycle.test.js b/server/services/cosAgentLifecycle.test.js
index 2fea48cfbd..5bdf335c4d 100644
--- a/server/services/cosAgentLifecycle.test.js
+++ b/server/services/cosAgentLifecycle.test.js
@@ -95,7 +95,7 @@ import { cosEvents } from './cosEvents.js';
import { loadAgentIndex } from './cosAgentIndex.js';
import { getActiveAgentsFromRunner, requestSecurityScanViaRunner } from './cosRunnerClient.js';
import { persistAgentSecurityScan } from './agentSecurityScan.js';
-import { activeAgents, runnerAgents, spawningTasks } from './agentState.js';
+import { activeAgents, runnerAgents, completingAgents, spawningTasks } from './agentState.js';
afterAll(cleanupTempDataRoots);
@@ -769,6 +769,22 @@ describe('cleanupZombieAgents — runner listing is not proof of life', () => {
await rm(mockCosState.agentsDir, { recursive: true, force: true });
});
+ it('leaves an exited runner to its in-flight completion, then reaps if completion failed', async () => {
+ await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');
+ mockCosState.state.agents['agent-finishing'] = {
+ id: 'agent-finishing', status: 'running', pid: 2147483646,
+ startedAt: new Date(Date.now() - 60_000).toISOString(),
+ };
+ completingAgents.add('agent-finishing');
+ try {
+ expect((await cleanupZombieAgents()).cleaned).toEqual([]);
+ expect(mockCosState.state.agents['agent-finishing'].status).toBe('running');
+ } finally {
+ completingAgents.delete('agent-finishing');
+ }
+ expect((await cleanupZombieAgents()).cleaned).toEqual(['agent-finishing']);
+ });
+
it('does not reap a pid-less record while its task is still being dispatched', async () => {
await writeFile(join(mockCosState.agentsDir, 'index.json'), '{}');
mockCosState.state.agents['agent-slow'] = {