Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve agent dispatches interrupted by server restarts
Adam Eivy75a61e16c06f
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.jsindex fabb57ca9c..00f0c96050 100644--- a/server/services/agentManagement.js+++ b/server/services/agentManagement.js@@ -1260,7 +1260,10 @@ async function runCleanupOrphanedAgents() {// completes a run that goes on to launch — its claim ownership bind then// fails with owner-not-running. Mirrors the zombie sweep's guards.if (agent.taskId && spawningTasks.has(agent.taskId)) continue;- if (!agent.pid) {+ // A shutdown-marked dispatch cannot still be initializing in this process.+ // Retire it now; otherwise the grace skips it and we consume its marker,+ // so the next sweep charges a genuine host interruption as an orphan.+ if (!agent.pid && !interruptedByRestart.has(agent.id)) {const startedAtMs = agent.startedAt ? Date.parse(agent.startedAt) : 0;if (Date.now() - startedAtMs < SPAWN_GRACE_MS) continue;}diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.jsindex ad5735922e..1aaba23f0c 100644--- a/server/services/agentManagement.test.js+++ b/server/services/agentManagement.test.js@@ -2025,6 +2025,26 @@ describe('host-restart interruptions are not charged orphan-retry budget (#3202)const requeuedMetadata = () => updateTask.mock.calls.at(-1)[1].metadata;+ it('recovers a shutdown-marked dispatch during startup grace without charging either retry budget', async () => {+ getAgents.mockResolvedValue([{+ ...deadAgent,+ startedAt: new Date().toISOString(),+ metadata: { ...deadMetadata, phase: 'initializing', useRunner: true },+ }]);+ getTaskById.mockResolvedValue({+ id: 'task-1', taskType: 'user', status: 'in_progress',+ metadata: { orphanRetryCount: 1, totalSpawnCount: 2 },+ });+ readHostShutdownMarker.mockResolvedValue({ at: new Date().toISOString(), signal: 'SIGTERM', agentIds: ['agent-dead'] });++ await cleanupOrphanedAgents();++ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user');+ expect(requeuedMetadata()).toMatchObject({ orphanRetryCount: 1, totalSpawnCount: 1, interruptedByRestart: true });+ expect(retireDeadAgent).toHaveBeenCalledWith(expect.objectContaining({ category: 'interrupted', exitCode: 143 }));+ expect(clearHostShutdownMarker).toHaveBeenCalled();+ });+it('requeues an interrupted run without incrementing orphanRetryCount or stamping lastOrphanedAt', async () => {getTaskById.mockResolvedValue({id: 'task-1', taskType: 'user', status: 'in_progress',diff --git a/server/services/agentState.js b/server/services/agentState.jsindex 4f7411d3c4..e9f6042782 100644--- a/server/services/agentState.js+++ b/server/services/agentState.js@@ -123,7 +123,7 @@ export const setUseRunner = (val) => { useRunner = val; };// at the default branch's HEAD, so a reaper reads it as clean and merged. Listing// the id in `getActiveAgentIds()` hands every reaper that already consults it the// protection, without reordering branch-reconcile's MERGED-before-owner check.-const dispatchingAgentIds = new Set();+export const dispatchingAgentIds = new Set();export const markAgentDispatching = (agentId) => { dispatchingAgentIds.add(agentId); };export const clearAgentDispatching = (agentId) => { dispatchingAgentIds.delete(agentId); };diff --git a/server/services/bootstrap.js b/server/services/bootstrap.jsindex b74c53889d..f4ad28a7df 100644--- a/server/services/bootstrap.js+++ b/server/services/bootstrap.js@@ -79,7 +79,7 @@ import * as cos from './cos.js';// Side-effect-free shared state module — importing it here does NOT pull in the// agent-lifecycle orchestrator graph (that's why activeAgents lives in its own// leaf module), so the shutdown handler can read the live agent set cheaply.-import { activeAgents } from './agentState.js';+import { activeAgents, dispatchingAgentIds } from './agentState.js';import * as automationScheduler from './automationScheduler.js';import * as agentActionExecutor from './agentActionExecutor.js';import * as telegram from './telegram.js';@@ -1152,10 +1152,10 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// Record which agents this process owned, so the NEXT boot's orphan sweep can// tell "PortOS was restarted out from under a healthy agent" apart from "the// agent's own process died" and skip the retry-budget/cooldown penalty for a- // fault the agent didn't cause (#3202). Only `activeAgents` — the ones whose- // child processes live in THIS process's tree — belong here; agents the CoS- // runner owns survive this restart untouched and are re-adopted by- // `syncRunnerAgents` on the next boot.+ // fault the agent didn't cause (#3202). Include dispatches whose durable+ // registration exists but whose process handoff has not finished: shutdown+ // destroys that setup too. Exclude `runnerAgents`; those handed-off runs+ // survive this restart and are re-adopted on the next boot.//// Note `activeAgents` is NOT "direct-spawn only": runner-launched TUI agents// register there too (`agentTuiSpawning.js`, both launch modes). The@@ -1169,7 +1169,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>// `.catch` is the belt-and-suspenders for an out-of-lifecycle rejection, and a// marker we fail to write only degrades recovery to the pre-existing orphan path.const markerWritten = withGrace('Host-shutdown marker', 1500, ({ finish, finishWithError }) =>- writeHostShutdownMarker({ agentIds: [...activeAgents.keys()], signal })+ writeHostShutdownMarker({ agentIds: [...activeAgents.keys(), ...dispatchingAgentIds], signal }).then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));// Reach every child spawned into its own process group — agent runs, CLIdiff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.jsindex 93f6f3b1ea..4dcf9c1845 100644--- a/server/services/bootstrap.shutdown.test.js+++ b/server/services/bootstrap.shutdown.test.js@@ -10,7 +10,7 @@* TreeKill signals the whole tree at once, so an agent PTY can exit during* the very first await — and if the flag isn't latched yet, that exit is* recorded as a completed run, which is the bug this issue is about.- * 2. The marker is built from `activeAgents` (the agents THIS process owns),+ * 2. The marker includes active agents and unfinished dispatches,* not from `getActiveAgentIds()` — runner-mode agents live in portos-cos,* survive the restart untouched, and must never be named as interrupted.*/@@ -51,8 +51,16 @@ describe('shutdown handler — host-restart bookkeeping (#3202)', () => {if (firstAwaitAt > -1) expect(latchAt).toBeLessThan(firstAwaitAt);});- it('snapshots the agent set from activeAgents only — runner agents survive the restart', () => {- expect(code).toMatch(/writeHostShutdownMarker\(\{\s*agentIds:\s*\[\.\.\.activeAgents\.keys\(\)\]/);+ it('snapshots active agents and unfinished dispatches without adopted runner agents', () => {+ const markerArgs = code.match(/writeHostShutdownMarker\((\{[^}]+\})\)/)?.[1];+ expect(markerArgs).toBeTruthy();+ const snapshot = runInNewContext(`(${markerArgs})`, {+ activeAgents: new Map([['agent-active', {}]]),+ dispatchingAgentIds: new Set(['agent-initializing']),+ runnerAgents: new Map([['agent-survivor', {}]]),+ signal: 'SIGTERM',+ });+ expect([...snapshot.agentIds]).toEqual(['agent-active', 'agent-initializing']);// getActiveAgentIds() folds in runnerAgents — naming those would tell the// next boot that portos-cos-owned agents were interrupted when they weren't.expect(code).not.toContain('getActiveAgentIds');