Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { maintenance } from '../lib/maintenanceAdmission.js';
/**
* Runner agent recovery.
*
* Rehydrates the in-memory `runnerAgents` map from the CoS Runner after a
* server restart, so completion events for agents spawned before the restart
* still land. Extracted from `agentLifecycle.js` (issue #2837): both
* `agentManagement.js` and `subAgentSpawner.js` need it, and importing it from
* the lifecycle orchestrator dragged that whole module graph into a cycle.
*
* Leaf with respect to the agent cluster — must not import `agentLifecycle.js`
* or `agentManagement.js` (enforced by `agentImportCycles.test.js`).
*/
import { connectTuiSessionViaRunner, getActiveAgentsFromRunner, terminateAgentViaRunner } from './cosRunnerClient.js';
import { runnerEntryShieldsRunningRecord } from '../lib/runnerAgentLiveness.js';
import { isInternalTaskId } from '../lib/taskParser.js';
import { isAgentOwnedLocally, runnerAgents, spawningTasks } from './agentState.js';
import { AGENT_RECORD_UNREADABLE, isLiveAgentRecord, readAgentRecordOrUnreadable } from './cosAgentLifecycle.js';
import { appendRunEvent } from './agentRunEventLog.js';
import * as shellService from './shell.js';
// Recovery cannot replay a paste safely: it may already be sitting unsent in
// the composer. Stop only a PTY whose runner positively knows no task submit
// reached it; normal completion owns task retry and workspace cleanup.
const startupErrorAnalysis = {
category: 'startup-failure',
origin: 'runner',
actionable: false,
message: 'Server restarted before the TUI task prompt was submitted',
suggestedFix: 'Retry the task in a fresh terminal; the recovered startup terminal was stopped without replaying its prompt.',
};
async function stopInterruptedStartup(agentId) {
try {
const result = await terminateAgentViaRunner(agentId, { onlyIfPromptUnsubmitted: true });
// Submission can land between GET /agents and the guarded stop. The runner
// decides atomically; preserve the now-working run and its normal verdict.
if (result.terminated === false) {
const owned = runnerAgents.get(agentId);
if (owned) delete owned.startupErrorAnalysis;
}
} catch (err) {
// Keep ownership and retry on the next recovery sweep. An RPC failure is
// not proof of exit and must not finalize a still-live PTY.
console.error(`❌ Failed to stop interrupted TUI startup ${agentId}: ${err.message}`);
}
}
/**
* Sync running agents from the runner (recovery after server restart).
* This allows us to receive completion events for agents spawned before restart.
*
* Recovery adopts only agents this process does NOT already own — see
* `isAgentOwnedLocally` for why that question needs both maps, and why asking
* `runnerAgents` alone silently hoisted live TUI runs. `cleanupOrphanedAgents`
* calls this every 15 minutes, so the bug reached any TUI run that outlived a
* single health-check tick: once hoisted, the `agent:completed` the runner emits
* when the TUI spawner's own `finish()` kills the session passed subAgentSpawner's
* `if (!agent) return` guard and ran a SECOND `finalizeAgent`, overwriting a
* sentinel-signalled success with `success: false, exitCode: 143` and a bogus
* `startup-failure` analysis — which flipped the agent card to Failed and requeued
* the finished task. Observed on a 55-min release-check run that had already
* published its release and merged all of its PRs.
*/
export async function syncRunnerAgents() {
const agents = await getActiveAgentsFromRunner().catch(err => {
console.error(`❌ Failed to get active agents from runner: ${err.message}`);
return [];
});
if (agents.length === 0) return 0;
console.log(`🔄 Syncing ${agents.length} running agents from CoS Runner`);
// Get all tasks to find task data for each agent
const { getAllTasks } = await import('./cosTaskStore.js');
const allTasksData = await getAllTasks().catch(() => ({ user: {}, cos: {} }));
// Build a task lookup map from all task sources, tagging each with its taskType
const taskMap = new Map();
const addTasks = (groupedTasks, taskType) => {
if (!groupedTasks) return;
for (const tasks of Object.values(groupedTasks)) {
if (Array.isArray(tasks)) {
for (const task of tasks) {
taskMap.set(task.id, { ...task, taskType });
}
}
}
};
addTasks(allTasksData.user?.grouped, 'user');
addTasks(allTasksData.cos?.grouped, 'internal');
let syncedCount = 0;
for (const agent of agents) {
if (agent.securityScan) {
const { persistAgentSecurityScan } = await import('./agentSecurityScan.js');
await persistAgentSecurityScan(agent.id, agent.securityScan);
}
// A stale runner handle (listed, process gone) is not a survivor to
// re-adopt — adopting it would put it in runnerAgents and the orphan
// sweep would treat that local map as ownership. Live rows still sync.
if (!(await runnerEntryShieldsRunningRecord(agent))) continue;
// A reconnect can list a PTY while this process still awaits its spawn
// acknowledgement/ledger write. Its controller owns startup already, even
// though it has not registered in activeAgents yet.
if (spawningTasks.has(agent.taskId)) continue;
maintenance.recoverOwned('agent', agent.id);
if (runnerAgents.get(agent.id)?.startupErrorAnalysis) {
await stopInterruptedStartup(agent.id);
continue;
}
// Only sync if this process isn't already driving it
if (!isAgentOwnedLocally(agent.id)) {
const task = taskMap.get(agent.taskId);
const inferredType = isInternalTaskId(agent.taskId) ? 'internal' : 'user';
// Recover the run id from the PERSISTED agent record (#3244). The runner's
// /agents response describes the live process and carries no `metadata`,
// but `spawnViaRunner` wrote `runId` into the agent record before handing
// off, so it is still on disk — `agentManagement.js` reads it the same way
// on its orphan path. Dropping it here left the survivor's run permanently
// open (`completeAgentRun` returns early on a null id), so the Runs list
// showed it running forever and `recordCompletedRunUsage` never fired —
// silently exempting the longest-lived runs in the system from all cost
// accounting. Survivors are the normal case since #3202 made TUI agents
// durable, so this cannot stay a best-effort null.
// Adopt only what PortOS still believes is running. The runner keeps
// advertising a PTY it never managed to kill, and adopting one of those
// mints a phantom `runnerAgents` entry that nothing ever removes: it holds
// an open run, reads as locally owned forever, and counts against the
// "CoS agents running" gate that pauses the Update page — while the agent
// list, built from the durable records, shows nothing at all.
//
// A read that FAILED is not a record that is absent: the failure proves
// nothing, and the live process behind it still needs its completion event
// to land, so it gets its own sentinel and is adopted anyway.
const read = await readAgentRecordOrUnreadable(agent.id);
if (read !== AGENT_RECORD_UNREADABLE && !isLiveAgentRecord(read)) {
console.warn(`⚠️ Ignoring stale runner agent ${agent.id} — PortOS has no live record for it`);
continue;
}
const persisted = read === AGENT_RECORD_UNREADABLE ? null : read;
const recoveredRunId = persisted?.metadata?.runId || null;
runnerAgents.set(agent.id, {
taskId: agent.taskId,
task: task || { id: agent.taskId, taskType: inferredType, description: 'Recovered from runner' },
runId: recoveredRunId,
model: persisted?.metadata?.model || null,
// Completion hooks must read the registered worktree, not the task's
// primary checkout. The runner's cwd also survives an unreadable record.
workspacePath: persisted?.metadata?.workspacePath || agent.workspacePath || null,
recoveredFromRunner: true,
hasStartedWorking: true,
startedAt: agent.startedAt,
...(agent.kind === 'tui' && agent.promptSubmitted === false ? { startupErrorAnalysis } : {}),
});
if (!recoveredRunId) {
console.warn(`⚠️ Recovered agent ${agent.id} has no run id on record — its run stays open and unbilled`);
}
// Record the re-adoption in the lifecycle ledger (#4540). No explicit
// idempotency key here, unlike the orphan sweep: `runnerAgents.set` above
// has already made this agent locally owned, so the `isAgentOwnedLocally`
// guard stops this branch re-firing within the process. A SECOND
// re-adoption therefore means a second server lifetime — a genuinely
// distinct recovery, which `recoveryCount` is supposed to count — and that
// count is what turns "this run has been running for nine
// hours" into "this run has survived three restarts". `hasRunId:false`
// is the durable trace of the unbilled-run warning above — today that
// warning only exists in a console line nothing retains.
await appendRunEvent({
kind: 'run.runner-recovered',
runId: recoveredRunId,
agentId: agent.id,
taskId: agent.taskId,
data: {
kind: agent.kind ?? null,
hasRunId: Boolean(recoveredRunId),
startedAt: agent.startedAt ?? null,
},
});
if (agent.kind === 'tui' && agent.sessionId && !shellService.getSession(agent.sessionId)) {
const session = connectTuiSessionViaRunner(agent);
shellService.registerExternalSession(agent.sessionId, session.ptyProcess, {
cwd: agent.workspacePath,
kind: 'agent-tui',
agentId: agent.id,
label: `Recovered TUI ${agent.id}`,
command: agent.command,
});
// A live PTY was re-attached to a still-running agent (#4540). Distinct
// from the `run.runner-recovered` above: that says the RUN was
// re-adopted, this says its terminal stream was re-plumbed — and only
// one of the two can fail. No explicit idempotency key: the
// `!getSession` guard already means "not currently attached", so every
// event here is a genuinely distinct reconnect and `reconnectCount`
// reads as the number of times this run lost and regained its stream.
await appendRunEvent({
kind: 'run.reconnected',
runId: recoveredRunId,
agentId: agent.id,
taskId: agent.taskId,
data: { transport: 'runner-pty', sessionId: agent.sessionId ?? null },
});
session.ptyProcess.onExit(({ exitCode }) => {
shellService.unregisterExternalSession(agent.sessionId, { exitCode });
});
}
console.log(`🔄 Recovered agent ${agent.id} (task: ${agent.taskId})`);
syncedCount++;
if (runnerAgents.get(agent.id)?.startupErrorAnalysis) {
console.warn(`⚠️ Stopping recovered TUI ${agent.id}: ${startupErrorAnalysis.message}`);
await stopInterruptedStartup(agent.id);
}
}
}
return syncedCount;
}