Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve agent dispatches interrupted by server restarts

Adam Eivy75a61e16c06f
diff --git a/server/services/agentManagement.js b/server/services/agentManagement.js
index fabb57ca9c..00f0c96050 100644
--- a/server/services/agentManagement.js
+++ b/server/services/agentManagement.js
@@ -1260,7 +1260,10 @@ async function runCleanupOrphanedAgents() {
// completes a run that goes on to launch — its claim ownership bind then
// fails with owner-not-running. Mirrors the zombie sweep's guards.
if (agent.taskId && spawningTasks.has(agent.taskId)) continue;
- if (!agent.pid) {
+ // A shutdown-marked dispatch cannot still be initializing in this process.
+ // Retire it now; otherwise the grace skips it and we consume its marker,
+ // so the next sweep charges a genuine host interruption as an orphan.
+ if (!agent.pid && !interruptedByRestart.has(agent.id)) {
const startedAtMs = agent.startedAt ? Date.parse(agent.startedAt) : 0;
if (Date.now() - startedAtMs < SPAWN_GRACE_MS) continue;
}
diff --git a/server/services/agentManagement.test.js b/server/services/agentManagement.test.js
index ad5735922e..1aaba23f0c 100644
--- a/server/services/agentManagement.test.js
+++ b/server/services/agentManagement.test.js
@@ -2025,6 +2025,26 @@ describe('host-restart interruptions are not charged orphan-retry budget (#3202)
const requeuedMetadata = () => updateTask.mock.calls.at(-1)[1].metadata;
+ it('recovers a shutdown-marked dispatch during startup grace without charging either retry budget', async () => {
+ getAgents.mockResolvedValue([{
+ ...deadAgent,
+ startedAt: new Date().toISOString(),
+ metadata: { ...deadMetadata, phase: 'initializing', useRunner: true },
+ }]);
+ getTaskById.mockResolvedValue({
+ id: 'task-1', taskType: 'user', status: 'in_progress',
+ metadata: { orphanRetryCount: 1, totalSpawnCount: 2 },
+ });
+ readHostShutdownMarker.mockResolvedValue({ at: new Date().toISOString(), signal: 'SIGTERM', agentIds: ['agent-dead'] });
+
+ await cleanupOrphanedAgents();
+
+ expect(updateTask).toHaveBeenCalledWith('task-1', expect.objectContaining({ status: 'pending' }), 'user');
+ expect(requeuedMetadata()).toMatchObject({ orphanRetryCount: 1, totalSpawnCount: 1, interruptedByRestart: true });
+ expect(retireDeadAgent).toHaveBeenCalledWith(expect.objectContaining({ category: 'interrupted', exitCode: 143 }));
+ expect(clearHostShutdownMarker).toHaveBeenCalled();
+ });
+
it('requeues an interrupted run without incrementing orphanRetryCount or stamping lastOrphanedAt', async () => {
getTaskById.mockResolvedValue({
id: 'task-1', taskType: 'user', status: 'in_progress',
diff --git a/server/services/agentState.js b/server/services/agentState.js
index 4f7411d3c4..e9f6042782 100644
--- a/server/services/agentState.js
+++ b/server/services/agentState.js
@@ -123,7 +123,7 @@ export const setUseRunner = (val) => { useRunner = val; };
// at the default branch's HEAD, so a reaper reads it as clean and merged. Listing
// the id in `getActiveAgentIds()` hands every reaper that already consults it the
// protection, without reordering branch-reconcile's MERGED-before-owner check.
-const dispatchingAgentIds = new Set();
+export const dispatchingAgentIds = new Set();
export const markAgentDispatching = (agentId) => { dispatchingAgentIds.add(agentId); };
export const clearAgentDispatching = (agentId) => { dispatchingAgentIds.delete(agentId); };
diff --git a/server/services/bootstrap.js b/server/services/bootstrap.js
index b74c53889d..f4ad28a7df 100644
--- a/server/services/bootstrap.js
+++ b/server/services/bootstrap.js
@@ -79,7 +79,7 @@ import * as cos from './cos.js';
// Side-effect-free shared state module — importing it here does NOT pull in the
// agent-lifecycle orchestrator graph (that's why activeAgents lives in its own
// leaf module), so the shutdown handler can read the live agent set cheaply.
-import { activeAgents } from './agentState.js';
+import { activeAgents, dispatchingAgentIds } from './agentState.js';
import * as automationScheduler from './automationScheduler.js';
import * as agentActionExecutor from './agentActionExecutor.js';
import * as telegram from './telegram.js';
@@ -1152,10 +1152,10 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// Record which agents this process owned, so the NEXT boot's orphan sweep can
// tell "PortOS was restarted out from under a healthy agent" apart from "the
// agent's own process died" and skip the retry-budget/cooldown penalty for a
- // fault the agent didn't cause (#3202). Only `activeAgents` — the ones whose
- // child processes live in THIS process's tree — belong here; agents the CoS
- // runner owns survive this restart untouched and are re-adopted by
- // `syncRunnerAgents` on the next boot.
+ // fault the agent didn't cause (#3202). Include dispatches whose durable
+ // registration exists but whose process handoff has not finished: shutdown
+ // destroys that setup too. Exclude `runnerAgents`; those handed-off runs
+ // survive this restart and are re-adopted on the next boot.
//
// Note `activeAgents` is NOT "direct-spawn only": runner-launched TUI agents
// register there too (`agentTuiSpawning.js`, both launch modes). The
@@ -1169,7 +1169,7 @@ export const registerShutdownHandlers = ({ io, httpServer, localHttpServer }) =>
// `.catch` is the belt-and-suspenders for an out-of-lifecycle rejection, and a
// marker we fail to write only degrades recovery to the pre-existing orphan path.
const markerWritten = withGrace('Host-shutdown marker', 1500, ({ finish, finishWithError }) =>
- writeHostShutdownMarker({ agentIds: [...activeAgents.keys()], signal })
+ writeHostShutdownMarker({ agentIds: [...activeAgents.keys(), ...dispatchingAgentIds], signal })
.then(() => finish(), (err) => finishWithError('⚠️ Host-shutdown marker failed', err)));
// Reach every child spawned into its own process group — agent runs, CLI
diff --git a/server/services/bootstrap.shutdown.test.js b/server/services/bootstrap.shutdown.test.js
index 93f6f3b1ea..4dcf9c1845 100644
--- a/server/services/bootstrap.shutdown.test.js
+++ b/server/services/bootstrap.shutdown.test.js
@@ -10,7 +10,7 @@
* TreeKill signals the whole tree at once, so an agent PTY can exit during
* the very first await — and if the flag isn't latched yet, that exit is
* recorded as a completed run, which is the bug this issue is about.
- * 2. The marker is built from `activeAgents` (the agents THIS process owns),
+ * 2. The marker includes active agents and unfinished dispatches,
* not from `getActiveAgentIds()` — runner-mode agents live in portos-cos,
* survive the restart untouched, and must never be named as interrupted.
*/
@@ -51,8 +51,16 @@ describe('shutdown handler — host-restart bookkeeping (#3202)', () => {
if (firstAwaitAt > -1) expect(latchAt).toBeLessThan(firstAwaitAt);
});
- it('snapshots the agent set from activeAgents only — runner agents survive the restart', () => {
- expect(code).toMatch(/writeHostShutdownMarker\(\{\s*agentIds:\s*\[\.\.\.activeAgents\.keys\(\)\]/);
+ it('snapshots active agents and unfinished dispatches without adopted runner agents', () => {
+ const markerArgs = code.match(/writeHostShutdownMarker\((\{[^}]+\})\)/)?.[1];
+ expect(markerArgs).toBeTruthy();
+ const snapshot = runInNewContext(`(${markerArgs})`, {
+ activeAgents: new Map([['agent-active', {}]]),
+ dispatchingAgentIds: new Set(['agent-initializing']),
+ runnerAgents: new Map([['agent-survivor', {}]]),
+ signal: 'SIGTERM',
+ });
+ expect([...snapshot.agentIds]).toEqual(['agent-active', 'agent-initializing']);
// getActiveAgentIds() folds in runnerAgents — naming those would tell the
// next boot that portos-cos-owned agents were interrupted when they weren't.
expect(code).not.toContain('getActiveAgentIds');