Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(console): reap orphaned agent runtimes at boot (#309)

Louis Amaudruz8d5cbcd6f54f
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.test.ts b/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.test.ts
new file mode 100644
index 000000000..c281d718f
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.test.ts
@@ -0,0 +1,95 @@
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const sweep = vi.fn();
+const logInfo = vi.fn();
+const logWarn = vi.fn();
+
+vi.mock('@sandboxaq/switch-agent-runtime', () => ({
+ reapOrphanedRuntimes: (...args: unknown[]) => sweep(...args),
+}));
+
+vi.mock('@main/lib/logger', () => ({
+ log: { info: (...a: unknown[]) => logInfo(...a), warn: (...a: unknown[]) => logWarn(...a) },
+}));
+
+const { reapOrphanedAgentRuntimes } = await import('./reap-orphaned-runtimes');
+
+const NOTHING = { reaped: 0, removedSessionDirs: 0, failures: [] };
+
+beforeEach(() => {
+ sweep.mockReset();
+ logInfo.mockReset();
+ logWarn.mockReset();
+ sweep.mockResolvedValue(NOTHING);
+});
+
+afterEach(() => vi.restoreAllMocks());
+
+describe('reapOrphanedAgentRuntimes', () => {
+ it('sweeps the shared session root', async () => {
+ await reapOrphanedAgentRuntimes();
+ expect(sweep).toHaveBeenCalledWith({
+ sessionsRoot: path.join(os.homedir(), '.switch', 'sessions'),
+ keepSessionDir: null,
+ });
+ });
+
+ /**
+ * The app owns no session directory. Passing one of a runtime's — or a
+ * plausible-looking guess — would spare a directory that should be swept, and
+ * on a pid collision could spare a live one's.
+ */
+ it('spares no session directory, having none of its own', async () => {
+ await reapOrphanedAgentRuntimes();
+ expect(sweep.mock.calls[0][0].keepSessionDir).toBeNull();
+ });
+
+ it('says nothing when there was nothing to clear', async () => {
+ await reapOrphanedAgentRuntimes();
+ expect(logInfo).not.toHaveBeenCalled();
+ expect(logWarn).not.toHaveBeenCalled();
+ });
+
+ it('reports what it cleared, under one event', async () => {
+ sweep.mockResolvedValue({ reaped: 3, removedSessionDirs: 12, failures: [] });
+ await reapOrphanedAgentRuntimes();
+
+ expect(logInfo).toHaveBeenCalledTimes(1);
+ expect(logInfo.mock.calls[0][1]).toMatchObject({
+ event: 'agent_runtime_reap',
+ reaped: 3,
+ removedSessionDirs: 12,
+ });
+ });
+
+ it('reports a directory sweep even with nothing reaped', async () => {
+ sweep.mockResolvedValue({ reaped: 0, removedSessionDirs: 8016, failures: [] });
+ await reapOrphanedAgentRuntimes();
+ expect(logInfo).toHaveBeenCalledTimes(1);
+ });
+
+ it('warns per failed stage, with the stage as a code rather than prose', async () => {
+ sweep.mockResolvedValue({
+ reaped: 0,
+ removedSessionDirs: 0,
+ failures: [
+ { stage: 'scan', error: new Error('ps missing') },
+ { stage: 'sweep', error: new Error('permission denied') },
+ ],
+ });
+ await reapOrphanedAgentRuntimes();
+
+ expect(logWarn).toHaveBeenCalledTimes(2);
+ expect(logWarn.mock.calls.map((call) => call[1].stage)).toEqual(['scan', 'sweep']);
+ expect(logWarn.mock.calls[0][1]).toMatchObject({ event: 'agent_runtime_reap_failed' });
+ });
+
+ // Boot calls this without awaiting it. A rejection that escapes would be an
+ // unhandled rejection during startup rather than a warning in the log.
+ it('propagates a hard failure for the caller to catch, rather than throwing into boot', async () => {
+ sweep.mockRejectedValue(new Error('ps exploded'));
+ await expect(reapOrphanedAgentRuntimes()).rejects.toThrow('ps exploded');
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.ts b/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.ts
new file mode 100644
index 000000000..5440550a6
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/agent-runtime/reap-orphaned-runtimes.ts
@@ -0,0 +1,47 @@
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { reapOrphanedRuntimes as sweep } from '@sandboxaq/switch-agent-runtime';
+import { log } from '@main/lib/logger';
+
+/**
+ * Terminate `switch-agent-runtime` processes whose host is gone, at boot.
+ *
+ * The runtime does this for itself now, and also exits on its own when its host
+ * dies — but both fixes reach a machine only through the connector pin, which
+ * moves when a user updates a marketplace plugin. This path reaches them when
+ * they update the app instead. The two populations are not the same: someone
+ * who never revisits Settings → Agents is reached only from here, and someone
+ * running Codex standalone is reached only by the runtime. Neither alone is
+ * enough, which is why both exist.
+ *
+ * The predicate is imported rather than reimplemented. Two copies of "is this
+ * runtime abandoned?" is the shape of bug this repo already knows well — where
+ * they disagree, one of them kills a live session — so the one implementation
+ * lives in the runtime package and both callers share it.
+ *
+ * Everything found here is dead weight by construction: a runtime with no live
+ * host has nobody to serve, and the session it belonged to is already gone.
+ */
+export async function reapOrphanedAgentRuntimes(): Promise<void> {
+ const sessionsRoot = path.join(os.homedir(), '.switch', 'sessions');
+
+ // Nothing to spare: the app is not itself a runtime and owns no session
+ // directory. Its own pid is protected inside the sweep regardless.
+ const outcome = await sweep({ sessionsRoot, keepSessionDir: null });
+
+ if (outcome.reaped > 0 || outcome.removedSessionDirs > 0) {
+ log.info('reapOrphanedAgentRuntimes: cleared runtimes left by earlier sessions', {
+ event: 'agent_runtime_reap',
+ reaped: outcome.reaped,
+ removedSessionDirs: outcome.removedSessionDirs,
+ });
+ }
+
+ for (const failure of outcome.failures) {
+ log.warn('reapOrphanedAgentRuntimes: sweep step failed', {
+ event: 'agent_runtime_reap_failed',
+ stage: failure.stage,
+ error: String(failure.error),
+ });
+ }
+}
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.test.ts b/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.test.ts
new file mode 100644
index 000000000..d42e20de1
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.test.ts
@@ -0,0 +1,165 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+
+const checkForUpdates = vi.fn();
+const update = vi.fn();
+const listPlugins = vi.fn();
+const kvGet = vi.fn();
+const kvSet = vi.fn();
+
+vi.mock('@main/db/kv', () => ({
+ KV: class {
+ get = (...a: unknown[]) => kvGet(...a);
+ set = (...a: unknown[]) => kvSet(...a);
+ },
+}));
+
+vi.mock('@main/lib/logger', () => ({
+ log: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
+}));
+
+vi.mock('../providers/plugin-registry', () => ({ listPlugins: () => listPlugins() }));
+
+vi.mock('./switch-setup-service', () => ({
+ switchSetupService: {
+ checkForUpdates: (...a: unknown[]) => checkForUpdates(...a),
+ update: (...a: unknown[]) => update(...a),
+ },
+}));
+
+const { catchUpConnectorsToCurrentVersion } = await import('./catch-up-connectors');
+
+const GENERATION = 1;
+
+function plugin(id: string, kind = 'cli') {
+ return { metadata: { id }, capabilities: { switchSetup: { kind } } };
+}
+
+/** Installed, one version behind, catalog readable. */
+function behind(agentId: string) {
+ return {
+ agentId,
+ supported: true,
+ installed: true,
+ installedVersion: '0.9.9',
+ latestVersion: '0.9.10',
+ updateAvailable: true,
+ refreshError: null,
+ };
+}
+
+function upToDate(agentId: string) {
+ return { ...behind(agentId), installedVersion: '0.9.10', updateAvailable: false };
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ kvGet.mockResolvedValue(null);
+ listPlugins.mockReturnValue([plugin('claude'), plugin('codex')]);
+ checkForUpdates.mockImplementation((id: string) => Promise.resolve(upToDate(id)));
+ update.mockResolvedValue({ success: true });
+});
+
+describe('the connector catch-up', () => {
+ it('updates every installed connector that is behind', async () => {
+ checkForUpdates.mockImplementation((id: string) => Promise.resolve(behind(id)));
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(update.mock.calls.map((call) => call[0])).toEqual(['claude', 'codex']);
+ });
+
+ it('leaves an up-to-date connector alone', async () => {
+ await catchUpConnectorsToCurrentVersion();
+ expect(update).not.toHaveBeenCalled();
+ });
+
+ /**
+ * A version catch-up, not an installer. Which agents have a Switch connector
+ * is the user's decision, and taking it for them would put a plugin on a
+ * machine that never asked for one.
+ */
+ it('never installs a connector that is not installed', async () => {
+ checkForUpdates.mockImplementation((id: string) =>
+ Promise.resolve({ ...behind(id), installed: false })
+ );
+ await catchUpConnectorsToCurrentVersion();
+ expect(update).not.toHaveBeenCalled();
+ });
+
+ it('skips an agent type with no Switch setup at all', async () => {
+ listPlugins.mockReturnValue([plugin('claude'), plugin('cursor', 'none')]);
+ checkForUpdates.mockImplementation((id: string) => Promise.resolve(behind(id)));
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(checkForUpdates.mock.calls.map((call) => call[0])).toEqual(['claude']);
+ });
+});
+
+describe('latching — the difference between a one-shot and a standing job', () => {
+ it('latches the generation once every connector is settled', async () => {
+ await catchUpConnectorsToCurrentVersion();
+ expect(kvSet).toHaveBeenCalledWith('state', { generation: GENERATION, attempts: 0 });
+ });
+
+ it('does nothing at all once latched', async () => {
+ kvGet.mockResolvedValue({ generation: GENERATION, attempts: 0 });
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(checkForUpdates).not.toHaveBeenCalled();
+ expect(update).not.toHaveBeenCalled();
+ expect(kvSet).not.toHaveBeenCalled();
+ });
+
+ it('does not latch when an update failed, so the next launch retries', async () => {
+ checkForUpdates.mockImplementation((id: string) => Promise.resolve(behind(id)));
+ update.mockResolvedValue({ success: false, message: 'network down' });
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(kvSet).toHaveBeenCalledWith('state', { generation: 0, attempts: 1 });
+ });
+
+ /**
+ * `updateAvailable` is false both when there is no update and when the
+ * catalog could not be read. Latching on the second spends the single shot on
+ * an answer that was never received.
+ */
+ it('does not latch when the catalog could not be refreshed', async () => {
+ checkForUpdates.mockImplementation((id: string) =>
+ Promise.resolve({ ...upToDate(id), refreshError: 'could not reach the marketplace' })
+ );
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(update).not.toHaveBeenCalled();
+ expect(kvSet).toHaveBeenCalledWith('state', { generation: 0, attempts: 1 });
+ });
+
+ it('gives up after three attempts rather than retrying every launch forever', async () => {
+ kvGet.mockResolvedValue({ generation: 0, attempts: 3 });
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(checkForUpdates).not.toHaveBeenCalled();
+ expect(kvSet).not.toHaveBeenCalled();
+ });
+
+ it('still latches when one connector updated and the rest were current', async () => {
+ checkForUpdates.mockImplementation((id: string) =>
+ Promise.resolve(id === 'claude' ? behind(id) : upToDate(id))
+ );
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(update).toHaveBeenCalledTimes(1);
+ expect(kvSet).toHaveBeenCalledWith('state', { generation: GENERATION, attempts: 0 });
+ });
+
+ // One agent type's CLI being missing says nothing about the others, and must
+ // not abandon them halfway.
+ it('carries on past a connector that throws, and counts it as a failure', async () => {
+ checkForUpdates.mockImplementation((id: string) => {
+ if (id === 'claude') return Promise.reject(new Error('claude binary missing'));
+ return Promise.resolve(behind(id));
+ });
+ await catchUpConnectorsToCurrentVersion();
+
+ expect(update).toHaveBeenCalledWith('codex');
+ expect(kvSet).toHaveBeenCalledWith('state', { generation: 0, attempts: 1 });
+ });
+});
diff --git a/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.ts b/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.ts
new file mode 100644
index 000000000..23b700f1c
--- /dev/null
+++ b/console/apps/switch-console-desktop/src/main/core/switch-setup/catch-up-connectors.ts
@@ -0,0 +1,128 @@
+import { KV } from '@main/db/kv';
+import { log } from '@main/lib/logger';
+import { listPlugins } from '../providers/plugin-registry';
+import { switchSetupService } from './switch-setup-service';
+
+/**
+ * Bring every installed Switch connector up to the version this build ships,
+ * once.
+ *
+ * Deliberately not a standing auto-updater. The Update button in
+ * Settings → Agents stays the normal path; this exists because a connector's
+ * pinned `@sandboxaq/switch-agent-runtime` version only moves when a user goes
+ * looking for it, and a user who never does keeps whatever they first
+ * installed — indefinitely, across every release. One catch-up closes that gap
+ * without turning every launch into a plugin update.
+ *
+ * Generation rather than a boolean, following the agent-storage migration: a
+ * latched install short-circuits, and a deliberate bump here — not a release,
+ * not a version change elsewhere — is what makes it run again.
+ */
+const CATCH_UP_GENERATION = 1;
+
+/**
+ * A broken environment must not turn a one-shot into a per-launch job. Failures
+ * are usually transient (offline, a marketplace mid-push), so retry — but stop,
+ * because "runs every boot forever" is the thing this is not.
+ */
+const MAX_ATTEMPTS = 3;
+
+type CatchUpMarker = { generation: number; attempts: number };
+
+const store = new KV<{ state: CatchUpMarker }>('switchSetupCatchUp');
+
+/** What one connector's catch-up did, for deciding whether to latch. */
+type Step = 'nothing-to-do' | 'updated' | 'failed';
+
+async function catchUpOne(agentId: string): Promise<Step> {
+ // Refreshes the marketplace for a CLI connector; for a file-based one it is
+ // the same local read as getStatus, since that connector ships in the app.
+ const status = await switchSetupService.checkForUpdates(agentId);
+
+ // Not installed is not a gap to close. Installing a connector the user never
+ // asked for is a decision that is theirs, and this is a version catch-up.
+ if (!status.installed) return 'nothing-to-do';
+
+ if (status.refreshError !== null) {
+ // The catalog could not be read, so `updateAvailable` means "no update was
+ // visible", not "no update exists". Latching on that would spend the one
+ // shot on an answer we did not get.
+ log.warn('catchUpConnectors: could not refresh the catalog; will retry', {
+ event: 'connector_catch_up_deferred',
+ agentId,
+ reason: status.refreshError,
+ });
+ return 'failed';
+ }
+
+ if (!status.updateAvailable) return 'nothing-to-do';
+
+ const result = await switchSetupService.update(agentId);
+ if (!result.success) {
+ log.warn('catchUpConnectors: update failed', {
+ event: 'connector_catch_up_failed',
+ agentId,
+ from: status.installedVersion,
+ to: status.latestVersion,
+ reason: result.message,
+ });
+ return 'failed';
+ }
+
+ log.info('catchUpConnectors: connector updated', {
+ event: 'connector_catch_up_updated',
+ agentId,
+ from: status.installedVersion,
+ to: status.latestVersion,
+ });
+ return 'updated';
+}
+
+/**
+ * Run the catch-up if this install has not completed it.
+ *
+ * Best-effort and unawaited by its caller: it talks to a plugin marketplace
+ * over the network and must never hold up a launch or fail one.
+ */
+export async function catchUpConnectorsToCurrentVersion(): Promise<void> {
+ const marker = (await store.get('state')) ?? { generation: 0, attempts: 0 };
+ if (marker.generation >= CATCH_UP_GENERATION) return;
+ if (marker.attempts >= MAX_ATTEMPTS) return;
+
+ const agentIds = listPlugins()
+ .filter((plugin) => plugin.capabilities.switchSetup.kind !== 'none')
+ .map((plugin) => plugin.metadata.id);
+
+ let updated = 0;
+ let failed = 0;
+
+ for (const agentId of agentIds) {
+ try {
+ const step = await catchUpOne(agentId);
+ if (step === 'updated') updated += 1;
+ if (step === 'failed') failed += 1;
+ } catch (error) {
+ // One agent type's CLI being absent or broken says nothing about the
+ // others, so carry on and let the attempt count decide when to stop.
+ failed += 1;
+ log.warn('catchUpConnectors: connector threw during catch-up', {
+ event: 'connector_catch_up_failed',
+ agentId,
+ error: String(error),
+ });
+ }
+ }
+
+ if (failed > 0) {
+ await store.set('state', { generation: marker.generation, attempts: marker.attempts + 1 });
+ return;
+ }
+
+ await store.set('state', { generation: CATCH_UP_GENERATION, attempts: marker.attempts });
+ if (updated > 0) {
+ log.info('catchUpConnectors: connectors brought up to date', {
+ event: 'connector_catch_up_complete',
+ updated,
+ });
+ }
+}
diff --git a/console/apps/switch-console-desktop/src/main/index.ts b/console/apps/switch-console-desktop/src/main/index.ts
index 8860ef7d5..5046d1b3e 100644
--- a/console/apps/switch-console-desktop/src/main/index.ts
+++ b/console/apps/switch-console-desktop/src/main/index.ts
@@ -10,6 +10,7 @@ import { setupApplicationMenu } from './app/menu';
import { registerAppScheme, setupAppProtocol } from './app/protocol';
import { createMainWindow, getMainWindow } from './app/window';
import { agentHookService } from './core/agent-hooks/agent-hook-service';
+import { reapOrphanedAgentRuntimes } from './core/agent-runtime/reap-orphaned-runtimes';
import { migrateAgentStorage } from './core/agents/migrate-agent-storage';
import { initializeRemoteDiscovery, initializeRemoteWatchers } from './core/agents/remote-watcher';
import { resolveAgentServers } from './core/agents/resolve-servers';
@@ -32,6 +33,7 @@ import { registerSidecarDiagnostics } from './core/sidecar/sidecar-diagnostics';
import { sshConnectionManager } from './core/ssh/lifecycle/production-ssh-connection-manager';
import { autoSessionWatcher } from './core/switch-rooms/auto-session-watcher';
import { restoreSwitchRoomSessions } from './core/switch-rooms/restore-sessions';
+import { catchUpConnectorsToCurrentVersion } from './core/switch-setup/catch-up-connectors';
import { registerTelemetryListeners } from './core/telemetry/telemetry-listeners';
import { trackEvent } from './core/telemetry/telemetry-service';
import { updateService } from './core/updates/update-service';
@@ -165,6 +167,22 @@ void app.whenReady().then(async () => {
void reconcileResourceSampler();
+ // Before any session is relaunched below, so a runtime abandoned by a
+ // previous run is gone before its replacement starts — but not awaited: it
+ // scans every process on the machine and may remove thousands of stale
+ // directories, and the window must not wait for either.
+ void reapOrphanedAgentRuntimes().catch((e: unknown) => {
+ log.warn('agent-runtime: failed to reap orphaned runtimes at boot', { error: e });
+ });
+
+ // A one-shot, not a standing auto-updater — it latches on a generation marker
+ // and does nothing on every later launch. Unawaited because it talks to a
+ // plugin marketplace over the network: a session relaunched below may still
+ // start on the old pin, and picks the new one up next launch.
+ void catchUpConnectorsToCurrentVersion().catch((e: unknown) => {
+ log.warn('switch-setup: connector catch-up failed at boot', { error: e });
+ });
+
// Reflect a managed local Switch stack that survived the last quit, so the UI
// shows it running without the user restarting it.
void localServerService.initialize();
diff --git a/console/packages/switch-agent-runtime/src/bin.ts b/console/packages/switch-agent-runtime/src/bin.ts
index 866980c24..53cc422b4 100644
--- a/console/packages/switch-agent-runtime/src/bin.ts
+++ b/console/packages/switch-agent-runtime/src/bin.ts
@@ -1948,11 +1948,21 @@ serving = true;
// machine and may remove thousands of directories, and a host waiting on
// `initialize` must not pay for either. Needs no identity, so it runs degraded
// too — an orphan from a previous session is there to clear up regardless.
-void reapOrphanedRuntimes({
- sessionsRoot: SESSIONS_ROOT,
- ownSessionDir: SESSION_DIR,
- log: (message) => process.stderr.write(`switch: ${message}\n`),
-});
+void reapOrphanedRuntimes({ sessionsRoot: SESSIONS_ROOT, keepSessionDir: SESSION_DIR }).then(
+ (outcome) => {
+ if (outcome.reaped > 0) {
+ process.stderr.write(`switch: reaped ${outcome.reaped} runtime(s) whose host is gone\n`);
+ }
+ if (outcome.removedSessionDirs > 0) {
+ process.stderr.write(
+ `switch: removed ${outcome.removedSessionDirs} stale session director(ies)\n`
+ );
+ }
+ for (const failure of outcome.failures) {
+ process.stderr.write(`switch: reap ${failure.stage} failed: ${failure.error}\n`);
+ }
+ }
+);
// Degraded, none of the machinery below has anything to act on: no identity to
// open a connection as, no room to route a hook to. Publishing no port also
diff --git a/console/packages/switch-agent-runtime/src/index.ts b/console/packages/switch-agent-runtime/src/index.ts
index 4f0cc998b..dc9418f6d 100644
--- a/console/packages/switch-agent-runtime/src/index.ts
+++ b/console/packages/switch-agent-runtime/src/index.ts
@@ -33,6 +33,14 @@ export {
type StreamScope,
type SwitchEventStreamDeps,
} from './event-stream';
+export {
+ findOrphanedRuntimes,
+ parseProcessTable,
+ reapOrphanedRuntimes,
+ staleSessionDirs,
+ type ProcessRow,
+ type ReapOutcome,
+} from './reap';
export { readSse, type SseFrame } from './sse';
export type {
AgentBridgeEvent,
diff --git a/console/packages/switch-agent-runtime/src/reap.test.ts b/console/packages/switch-agent-runtime/src/reap.test.ts
index 71ff5090d..9fa88dc3b 100644
--- a/console/packages/switch-agent-runtime/src/reap.test.ts
+++ b/console/packages/switch-agent-runtime/src/reap.test.ts
@@ -174,4 +174,13 @@ describe('sweeping stale session directories', () => {
it('returns nothing for an empty directory', () => {
expect(staleSessionDirs([], alive, 'none')).toEqual([]);
});
+
+ // The app sweeps the same root and owns no session directory of its own.
+ it('spares nothing extra when the caller has no directory to keep', () => {
+ expect(staleSessionDirs(['900', '901'], alive, null)).toEqual(['901']);
+ });
+
+ it('still spares a live pid when there is nothing to keep', () => {
+ expect(staleSessionDirs(['900'], alive, null)).toEqual([]);
+ });
});
diff --git a/console/packages/switch-agent-runtime/src/reap.ts b/console/packages/switch-agent-runtime/src/reap.ts
index 69dcf9799..846d3cfbe 100644
--- a/console/packages/switch-agent-runtime/src/reap.ts
+++ b/console/packages/switch-agent-runtime/src/reap.ts
@@ -134,7 +134,7 @@ export function findOrphanedRuntimes(rows: ProcessRow[], selfPid: number): numbe
export function staleSessionDirs(
entries: string[],
isAlive: (pid: number) => boolean,
- keep: string
+ keep: string | null
): string[] {
return entries.filter(
(entry) => entry !== keep && /^\d+$/.test(entry) && !isAlive(Number(entry))
@@ -152,56 +152,73 @@ function pidIsAlive(pid: number): boolean {
}
}
+/** What one sweep did, for the caller to report however it reports things. */
+export type ReapOutcome = {
+ reaped: number;
+ removedSessionDirs: number;
+ /** Enumerated rather than free text: both callers log a code, not a sentence. */
+ failures: Array<{ stage: 'scan' | 'sweep'; error: unknown }>;
+};
+
/**
- * Clear up after previous runtimes. Best-effort throughout: this runs beside a
- * live session and must never be able to fail it.
+ * Clear up after runtimes that outlived their host.
+ *
+ * Returns what it did instead of logging it. There are two callers with two
+ * different ideas of a log line — the runtime writes prose to stderr, the app
+ * wants a structured event — and neither should have to be threaded through
+ * here as a callback.
+ *
+ * Best-effort throughout: this runs beside something already serving a user and
+ * must never be able to fail it, so every failure is collected rather than
+ * thrown.
*
* `ps` covers macOS and Linux; Windows has no equivalent worth reimplementing
- * for a population that will reboot away, so only the directory sweep runs
- * there.
+ * for a population that reboots away, so only the directory sweep runs there.
*/
export async function reapOrphanedRuntimes(options: {
sessionsRoot: string;
- ownSessionDir: string;
- log: (message: string) => void;
-}): Promise<void> {
- const { sessionsRoot, ownSessionDir, log } = options;
+ /**
+ * A session directory to spare — the caller's own, when the caller is itself
+ * a runtime. The app has none and passes null.
+ */
+ keepSessionDir: string | null;
+}): Promise<ReapOutcome> {
+ const { sessionsRoot, keepSessionDir } = options;
+ const outcome: ReapOutcome = { reaped: 0, removedSessionDirs: 0, failures: [] };
if (process.platform !== 'win32') {
try {
const { stdout } = await execFileAsync('ps', ['-axo', 'pid=,ppid=,command=']);
- const orphaned = findOrphanedRuntimes(parseProcessTable(stdout), process.pid);
-
- if (orphaned.length > 0) {
- log(`reaping ${orphaned.length} runtime process(es) whose host is gone`);
- for (const pid of orphaned) {
- try {
- // Every version being reaped predates the shutdown handlers, so
- // this lands on node's default disposition and terminates them.
- process.kill(pid, 'SIGTERM');
- } catch {
- // Already gone, or not ours to signal.
- }
+ for (const pid of findOrphanedRuntimes(parseProcessTable(stdout), process.pid)) {
+ try {
+ // Every version being reaped predates the shutdown handlers, so this
+ // lands on node's default disposition and terminates them.
+ process.kill(pid, 'SIGTERM');
+ outcome.reaped += 1;
+ } catch {
+ // Already gone, or not ours to signal.
}
}
- } catch (err) {
- log(`could not scan for orphaned runtimes: ${err}`);
+ } catch (error) {
+ outcome.failures.push({ stage: 'scan', error });
}
}
try {
const entries = await fs.readdir(sessionsRoot);
- const stale = staleSessionDirs(entries, pidIsAlive, path.basename(ownSessionDir));
- if (stale.length === 0) return;
-
- log(`removing ${stale.length} stale session director(ies)`);
- for (const entry of stale) {
+ const keep = keepSessionDir === null ? null : path.basename(keepSessionDir);
+ for (const entry of staleSessionDirs(entries, pidIsAlive, keep)) {
// Awaited one at a time rather than in bulk: there can be thousands, and
- // this shares an event loop with a session that is already serving.
+ // this shares an event loop with something already serving.
await fs.rm(path.join(sessionsRoot, entry), { recursive: true, force: true });
+ outcome.removedSessionDirs += 1;
+ }
+ } catch (error) {
+ // Nothing has ever run here; there is nothing to clean and nothing to say.
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+ outcome.failures.push({ stage: 'sweep', error });
}
- } catch (err) {
- if ((err as NodeJS.ErrnoException).code === 'ENOENT') return;
- log(`could not sweep stale session directories: ${err}`);
}
+
+ return outcome;
}