Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(runtime): no unattended server restarts under live runs; session lists always answer

ogsamrat8fdd5f1c1fe4
diff --git a/apps/app/src/app/lib/diagnostics-bundle.ts b/apps/app/src/app/lib/diagnostics-bundle.ts
index e19e8a29..41184cce 100644
--- a/apps/app/src/app/lib/diagnostics-bundle.ts
+++ b/apps/app/src/app/lib/diagnostics-bundle.ts
@@ -112,6 +112,9 @@ function pickHostInfo(info: OmniRushServerInfo | null) {
lanUrl: info.lanUrl ?? null,
lastStdout: info.lastStdout ?? null,
lastStderr: info.lastStderr ?? null,
+ // Built-in server/engine restarts with reason, trigger and time (newest first).
+ restarts: info.restarts ?? [],
+ pendingRestart: info.pendingRestart ?? null,
};
}
diff --git a/apps/app/src/react-app/domains/connections/omnirush-server-store.ts b/apps/app/src/react-app/domains/connections/omnirush-server-store.ts
index 0382867a..5bc8ca1c 100644
--- a/apps/app/src/react-app/domains/connections/omnirush-server-store.ts
+++ b/apps/app/src/react-app/domains/connections/omnirush-server-store.ts
@@ -807,7 +807,11 @@ export function createOmniRushServerStore(options: CreateOmniRushServerStoreOpti
}
try {
+ // Not a user action: the desktop keeps a server that still answers it
+ // and only replaces one that is really gone.
hostInfo = await omnirushServerRestart({
+ reason: "renderer_health_check_failed",
+ source: "omnirush-server-store",
remoteAccessEnabled: state.omnirushServerSettings.remoteAccessEnabled === true,
}) as OmniRushServerInfo;
mutateState((current) => ({ ...current, omnirushServerHostInfo: hostInfo }));
diff --git a/apps/app/src/react-app/domains/connections/provider-auth/store.ts b/apps/app/src/react-app/domains/connections/provider-auth/store.ts
index 7ff6b1e1..4fd50344 100644
--- a/apps/app/src/react-app/domains/connections/provider-auth/store.ts
+++ b/apps/app/src/react-app/domains/connections/provider-auth/store.ts
@@ -1481,7 +1481,7 @@ export function createProviderAuthStore(options: CreateProviderAuthStoreOptions)
if (!unreachable || !isDesktopRuntime()) {
throw error;
}
- await engineRestart({});
+ await engineRestart({ reason: "provider_reload_engine_unreachable", source: "provider-auth" });
}
reloaded = true;
}
@@ -2379,7 +2379,11 @@ export function createProviderAuthStore(options: CreateProviderAuthStoreOptions)
// Explicit desktop sign-out must replace that process so an
// account-scoped provider cannot remain connected in the UI.
if (isDesktopRuntime()) {
- await engineRestart({}).catch(() => undefined);
+ await engineRestart({
+ reason: "account_signed_out",
+ source: "provider-auth",
+ userInitiated: true,
+ }).catch(() => undefined);
}
})();
}
diff --git a/apps/app/src/react-app/domains/settings/state/debug-view-model.ts b/apps/app/src/react-app/domains/settings/state/debug-view-model.ts
index 2cec62e3..94d7bb61 100644
--- a/apps/app/src/react-app/domains/settings/state/debug-view-model.ts
+++ b/apps/app/src/react-app/domains/settings/state/debug-view-model.ts
@@ -736,6 +736,8 @@ export function useDebugViewModel(options: UseDebugViewModelOptions) {
}
const info = await engineStartCmd(workspacePath, {
+ reason: "debug_view_restart",
+ userInitiated: true,
runtime: "direct",
workspacePaths,
opencodeEnableExa: readOpencodeEnableExa(),
@@ -807,6 +809,8 @@ export function useDebugViewModel(options: UseDebugViewModelOptions) {
setServiceRestartError(null);
try {
await omnirushServerRestartCmd({
+ reason: "debug_view_restart",
+ userInitiated: true,
remoteAccessEnabled: omnirushServerSnapshot.omnirushServerSettings.remoteAccessEnabled === true,
});
setOmniRushServiceStatus({
diff --git a/apps/app/src/react-app/domains/workspace/remote-access-restart.ts b/apps/app/src/react-app/domains/workspace/remote-access-restart.ts
index 92aec605..c715a1f2 100644
--- a/apps/app/src/react-app/domains/workspace/remote-access-restart.ts
+++ b/apps/app/src/react-app/domains/workspace/remote-access-restart.ts
@@ -36,7 +36,11 @@ export function useRemoteAccessRestart(options: UseRemoteAccessRestartOptions) {
options.onSettingsChanged();
try {
- const info = await omnirushServerRestart({ remoteAccessEnabled: enabled }) as OmniRushServerInfo;
+ const info = await omnirushServerRestart({
+ reason: "remote_access_toggled",
+ userInitiated: true,
+ remoteAccessEnabled: enabled,
+ }) as OmniRushServerInfo;
writeOmniRushServerSettings({
urlOverride: info.baseUrl?.trim() || undefined,
token:
diff --git a/apps/app/src/react-app/shell/desktop-runtime-boot.ts b/apps/app/src/react-app/shell/desktop-runtime-boot.ts
index 007558c5..e9b328ec 100644
--- a/apps/app/src/react-app/shell/desktop-runtime-boot.ts
+++ b/apps/app/src/react-app/shell/desktop-runtime-boot.ts
@@ -118,7 +118,11 @@ export function useDesktopRuntimeBoot() {
const startServerWithoutDesktopWorkspace = async () => {
setPhase("starting-engine", "Starting omnirush.ai server");
- const serverInfo = await omnirushServerRestart({ remoteAccessEnabled: preferredRemoteAccess }).catch((error) => {
+ const serverInfo = await omnirushServerRestart({
+ reason: "renderer_boot_without_local_workspace",
+ source: "desktop-runtime-boot",
+ remoteAccessEnabled: preferredRemoteAccess,
+ }).catch((error) => {
console.warn("[desktop-boot] omnirushServerRestart failed:", error);
return null;
});
@@ -181,7 +185,11 @@ export function useDesktopRuntimeBoot() {
}
let serverInfo = boot.omnirushServer;
if (preferredRemoteAccess && serverInfo?.remoteAccessEnabled !== true) {
- const restarted = await omnirushServerRestart({ remoteAccessEnabled: true }).catch((error) => {
+ const restarted = await omnirushServerRestart({
+ reason: "renderer_boot_remote_access",
+ source: "desktop-runtime-boot",
+ remoteAccessEnabled: true,
+ }).catch((error) => {
console.warn("[desktop-boot] omnirushServerRestart failed:", error);
return null;
});
diff --git a/apps/app/src/react-app/shell/engine-reload-escalation.ts b/apps/app/src/react-app/shell/engine-reload-escalation.ts
index a2d02754..0e495d43 100644
--- a/apps/app/src/react-app/shell/engine-reload-escalation.ts
+++ b/apps/app/src/react-app/shell/engine-reload-escalation.ts
@@ -20,10 +20,6 @@ export function canRestartDesktopForReloadError(error: unknown) {
);
}
-function isEngineUnreachableError(error: unknown) {
- return error instanceof OmniRushServerError && error.code === "opencode_engine_unreachable";
-}
-
/**
* Aborted or timed-out fetches are client→server transport blips (including
* AbortSignal-cancelable fetch cancellations), not proof the engine is gone.
@@ -56,15 +52,18 @@ export type ReloadEngineFallbackOptions = {
/**
* Reload the workspace engine; escalate to a full desktop engine restart only
* when the reload keeps failing with a restartable server-reported code.
- * `opencode_engine_unreachable` gets one delayed retry first — if the engine
- * answers the second attempt, no session is disturbed.
+ * Both codes get one delayed retry first — an engine that is still starting
+ * or recovering answers the second attempt, and no session is disturbed.
+ * The desktop then defers the restart while any session is still running
+ * (it reports `restartDeferred`), so this never ends live runs by itself.
*/
export async function reloadEngineWithDesktopFallback(
client: Pick<OmniRushServerClient, "reloadEngine">,
workspaceId: string,
options?: ReloadEngineFallbackOptions,
): Promise<ReloadEngineFallbackResult> {
- const restartEngine = options?.restartEngine ?? (() => engineRestart({}));
+ const restartEngine = options?.restartEngine
+ ?? (() => engineRestart({ reason: "engine_reload_failed", source: "engine-reload" }));
const isDesktop = options?.isDesktop ?? isDesktopRuntime;
try {
await client.reloadEngine(workspaceId);
@@ -74,17 +73,17 @@ export async function reloadEngineWithDesktopFallback(
if (!canRestartDesktopForReloadError(error) || !isDesktop()) {
throw error;
}
- if (isEngineUnreachableError(error)) {
- await delay(options?.retryDelayMs ?? UNREACHABLE_RETRY_DELAY_MS);
- try {
- await client.reloadEngine(workspaceId);
- return { restartedEngine: false };
- } catch (retryError) {
- if (isTransientTransportError(retryError)) throw retryError;
- if (!canRestartDesktopForReloadError(retryError)) throw retryError;
- }
+ await delay(options?.retryDelayMs ?? UNREACHABLE_RETRY_DELAY_MS);
+ try {
+ await client.reloadEngine(workspaceId);
+ return { restartedEngine: false };
+ } catch (retryError) {
+ if (isTransientTransportError(retryError)) throw retryError;
+ if (!canRestartDesktopForReloadError(retryError)) throw retryError;
}
- await restartEngine();
- return { restartedEngine: true };
+ const restart = await restartEngine();
+ const deferred = typeof restart === "object" && restart !== null && "restartDeferred" in restart
+ && restart.restartDeferred === true;
+ return { restartedEngine: !deferred };
}
}
diff --git a/apps/app/src/react-app/shell/settings-route.tsx b/apps/app/src/react-app/shell/settings-route.tsx
index 70c9d3d9..95b1edaf 100644
--- a/apps/app/src/react-app/shell/settings-route.tsx
+++ b/apps/app/src/react-app/shell/settings-route.tsx
@@ -713,6 +713,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {
if (!isDesktopRuntime()) return false;
try {
await omnirushServerRestart({
+ reason: "share_remote_access_changed",
+ userInitiated: true,
remoteAccessEnabled:
readOmniRushServerSettings().remoteAccessEnabled === true,
});
@@ -1892,6 +1894,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {
if (!isDesktopRuntime()) return false;
try {
await omnirushServerRestart({
+ reason: "extension_settings_restart",
+ userInitiated: true,
remoteAccessEnabled:
readOmniRushServerSettings().remoteAccessEnabled === true,
});
@@ -2095,6 +2099,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {
workspacePaths.unshift(selectedWorkspaceRoot);
}
await engineStart(selectedWorkspaceRoot, {
+ reason: "environment_applied",
+ userInitiated: true,
preferSidecar: true,
runtime: "direct",
workspacePaths,
diff --git a/apps/app/tests/diagnostics-bundle.test.ts b/apps/app/tests/diagnostics-bundle.test.ts
index cef534e0..60b151eb 100644
--- a/apps/app/tests/diagnostics-bundle.test.ts
+++ b/apps/app/tests/diagnostics-bundle.test.ts
@@ -102,6 +102,43 @@ describe("diagnostics bundle", () => {
expect(json).not.toContain("opencodeUsername");
});
+ test("reports built-in server restarts with reason, trigger and time", () => {
+ const input = baseInputs();
+ input.desktopRuntime = true;
+ input.hostInfo = {
+ running: true,
+ generation: 3,
+ remoteAccessEnabled: false,
+ host: "127.0.0.1",
+ port: 4096,
+ baseUrl: "http://127.0.0.1:4096",
+ connectUrl: null,
+ mdnsUrl: null,
+ lanUrl: null,
+ clientToken: null,
+ ownerToken: null,
+ hostToken: null,
+ managedOpencodeBinPath: null,
+ managedOpencodeBinSource: null,
+ logFilePath: null,
+ pid: null,
+ lastStdout: null,
+ lastStderr: null,
+ managedOpencodeExecution: null,
+ restarts: [
+ { at: "2026-09-24T07:11:02.000Z", kind: "server", action: "deferred", reason: "engine_reload_failed", source: "engine-reload", busySessions: 3 },
+ { at: "2026-09-24T02:57:40.000Z", kind: "engine", action: "restarted", reason: "engine_unreachable:process_exited", source: "engine-pool-watchdog" },
+ ],
+ pendingRestart: { action: "engine-restart", reason: "engine_reload_failed", source: "engine-reload", requestedAt: "2026-09-24T07:11:02.000Z" },
+ };
+
+ const parsed = JSON.parse(composeDiagnosticsBundleJson(input));
+
+ expect(parsed.omnirushServer.host.restarts).toHaveLength(2);
+ expect(parsed.omnirushServer.host.restarts[0]).toMatchObject({ action: "deferred", reason: "engine_reload_failed", busySessions: 3 });
+ expect(parsed.omnirushServer.host.pendingRestart.reason).toBe("engine_reload_failed");
+ });
+
test("produces valid JSON without desktop info", () => {
const json = composeDiagnosticsBundleJson(baseInputs());
const parsed = JSON.parse(json);
diff --git a/apps/app/tests/engine-reload-escalation.test.ts b/apps/app/tests/engine-reload-escalation.test.ts
index bcfc8a63..3e9f497a 100644
--- a/apps/app/tests/engine-reload-escalation.test.ts
+++ b/apps/app/tests/engine-reload-escalation.test.ts
@@ -69,11 +69,32 @@ describe("reloadEngineWithDesktopFallback", () => {
expect(harness.restartCalls).toBe(1);
});
- test("an unconfigured engine restarts immediately — there is nothing to re-probe", async () => {
- const harness = makeHarness([unconfiguredError()]);
+ test("an engine that is still starting (unconfigured) recovers on the retry without a restart", async () => {
+ const harness = makeHarness([unconfiguredError(), null]);
+ const result = await reloadEngineWithDesktopFallback(harness.client, "ws", harness.options);
+ expect(result.restartedEngine).toBe(false);
+ expect(harness.reloadCalls).toBe(2);
+ expect(harness.restartCalls).toBe(0);
+ });
+
+ test("a persistently unconfigured engine escalates to a restart after one retry", async () => {
+ const harness = makeHarness([unconfiguredError(), unconfiguredError()]);
const result = await reloadEngineWithDesktopFallback(harness.client, "ws", harness.options);
expect(result.restartedEngine).toBe(true);
- expect(harness.reloadCalls).toBe(1);
+ expect(harness.reloadCalls).toBe(2);
+ expect(harness.restartCalls).toBe(1);
+ });
+
+ test("a restart the desktop deferred because sessions are running is not reported as a restart", async () => {
+ const harness = makeHarness([unreachableError(), unreachableError()]);
+ const result = await reloadEngineWithDesktopFallback(harness.client, "ws", {
+ ...harness.options,
+ restartEngine: async () => {
+ harness.restartCalls += 1;
+ return { running: true, restartDeferred: true, busySessions: 3 };
+ },
+ });
+ expect(result.restartedEngine).toBe(false);
expect(harness.restartCalls).toBe(1);
});
diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjs
index 5b53a9f6..568e834e 100644
--- a/apps/desktop/electron/main.mjs
+++ b/apps/desktop/electron/main.mjs
@@ -1221,7 +1221,7 @@ async function persistConnectLinkClaims(claims) {
await uiControlServer.start().catch((error) => {
console.warn("[ui-control] failed to start", error);
});
- await runtimeManager.prepareFreshRuntime();
+ await runtimeManager.prepareFreshRuntime({ reason: "desktop_activated", source: "connect-link" });
}
return config;
}
@@ -1899,11 +1899,11 @@ const desktopCommandHandlers = {
},
"engineStart": async (event, ...args) => {
const projectDir = String(args[0] ?? "").trim();
- const options = args[1] ?? {};
+ const options = { source: "renderer", ...(args[1] ?? {}) };
return runtimeManager.engineStart(projectDir, options);
},
"prepareFreshRuntime": async (event, ...args) => {
- return runtimeManager.prepareFreshRuntime();
+ return runtimeManager.prepareFreshRuntime({ source: "renderer" });
},
"runtimeBootstrap": async (event, ...args) => {
return ensureRuntimeBootstrap();
@@ -1912,10 +1912,10 @@ const desktopCommandHandlers = {
return runtimeManager.runtimeStatus();
},
"engineStop": async (event, ...args) => {
- return runtimeManager.engineStop();
+ return runtimeManager.engineStop({ source: "renderer" });
},
"engineRestart": async (event, ...args) => {
- return runtimeManager.engineRestart(args[0] ?? {});
+ return runtimeManager.engineRestart({ source: "renderer", ...(args[0] ?? {}) });
},
"engineInfo": async (event, ...args) => {
return runtimeManager.engineInfo();
@@ -2031,7 +2031,7 @@ const desktopCommandHandlers = {
await uiControlServer.start().catch((error) => {
console.warn("[ui-control] failed to start", error);
});
- await runtimeManager.prepareFreshRuntime();
+ await runtimeManager.prepareFreshRuntime({ reason: "desktop_activated", source: "bootstrap-config" });
}
return next;
},
@@ -2110,19 +2110,19 @@ const desktopCommandHandlers = {
deviceName: String(input.deviceName ?? "").trim() || `${app.getName()} on ${os.hostname()}`,
openVerification: (url) => shell.openExternal(url),
});
- await runtimeManager.omnirushServerRestart({});
+ await runtimeManager.omnirushServerRestart({ reason: "account_connected", source: "account", userInitiated: true });
return result;
},
"omnirushAccountSignOut": async (event, ...args) => {
const result = await omnirushAccountStore.clear();
- await runtimeManager.omnirushServerRestart({});
+ await runtimeManager.omnirushServerRestart({ reason: "account_signed_out", source: "account", userInitiated: true });
return { connected: false, remoteRevoked: result.remoteRevoked, reason: result.reason };
},
"automationRunnerConfigure": async (event, ...args) => {
return desktopAutomationRunner.configure(args[0] ?? null);
},
"omnirushServerRestart": async (event, ...args) => {
- return runtimeManager.omnirushServerRestart(args[0] ?? {});
+ return runtimeManager.omnirushServerRestart({ source: "renderer", ...(args[0] ?? {}) });
},
"pickDirectory": async (event, ...args) => {
const options = args[0] ?? {};
@@ -2949,7 +2949,7 @@ or use: pnpm dev:worktree`);
}
applicationMenu.install();
if (!desktopActivationRequired(DESKTOP_DISTRIBUTION, bootstrapConfig)) {
- await runtimeManager.prepareFreshRuntime().catch(() => undefined);
+ await runtimeManager.prepareFreshRuntime({ reason: "app_launch", source: "boot" }).catch(() => undefined);
}
// Use Tauri's existing workspace state file as canonical so rollback and
diff --git a/apps/desktop/electron/nuke.mjs b/apps/desktop/electron/nuke.mjs
index a5dff717..7e334c29 100644
--- a/apps/desktop/electron/nuke.mjs
+++ b/apps/desktop/electron/nuke.mjs
@@ -736,7 +736,7 @@ async function bestEffort(errors, label, task, timeoutMs) {
async function quiesceForNuke({ runtimeManager, uiControlServer, removeWindowsBrandShortcut }, errors, options = {}) {
await bestEffort(errors, "ui-control-server", () => uiControlServer.stop(), 3000);
await bestEffort(errors, "runtime-dispose", () => runtimeManager.dispose(), 12_000);
- await bestEffort(errors, "packaged-sidecar-reaper", () => runtimeManager.prepareFreshRuntime(), 16_000);
+ await bestEffort(errors, "packaged-sidecar-reaper", () => runtimeManager.prepareFreshRuntime({ reason: "reset_app_data", source: "nuke" }), 16_000);
// Container cleanup matches on name prefix across the whole Docker host, so it
// cannot tell this profile's containers from another profile's. Only the
// default profile may run it; isolated profiles leave containers alone rather
diff --git a/apps/desktop/electron/runtime-restart-log.mjs b/apps/desktop/electron/runtime-restart-log.mjs
new file mode 100644
index 00000000..1c613e41
--- /dev/null
+++ b/apps/desktop/electron/runtime-restart-log.mjs
@@ -0,0 +1,135 @@
+import { appendFileSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Restart record and restart policy for the built-in server and its engine.
+ *
+ * Every stop of a running server ends every live run, so each one is written
+ * with its reason and trigger to omnirush-server.log and to a small JSON-lines
+ * history next to it (runtime-restarts.jsonl, also appended by the server for
+ * engine recoveries it performs itself). omnirushServerInfo returns the tail
+ * of that history for the diagnostics bundle.
+ */
+
+export const RUNTIME_RESTART_LOG_FILE_NAME = "runtime-restarts.jsonl";
+const MAX_HISTORY_BYTES = 256 * 1024;
+const KEEP_AFTER_TRIM = 100;
+/** A deferred restart whose busy check stays unreadable this long runs anyway: the engine is gone. */
+export const DEFERRED_RESTART_UNKNOWN_MAX_MS = 10 * 60_000;
+
+export function runtimeRestartLogPath(serverLogFile) {
+ return path.join(path.dirname(serverLogFile), RUNTIME_RESTART_LOG_FILE_NAME);
+}
+
+function trimHistory(historyFile) {
+ try {
+ if (statSync(historyFile).size <= MAX_HISTORY_BYTES) return;
+ const lines = readFileSync(historyFile, "utf8").split("\n").filter(Boolean);
+ writeFileSync(historyFile, `${lines.slice(-KEEP_AFTER_TRIM).join("\n")}\n`, "utf8");
+ } catch {
+ // Diagnostics only.
+ }
+}
+
+/**
+ * Append one record to the history and one structured line to the server log.
+ * Best effort: diagnostics must never block or fail a restart.
+ *
+ * @param {{ serverLogFile: string | null, record: Record<string, unknown> & { kind: string, action: string,
+ * reason: string, source: string }, now?: () => Date }} input
+ */
+export function appendRuntimeRestartRecord({ serverLogFile, record, now = () => new Date() }) {
+ if (!serverLogFile) return null;
+ const at = now();
+ const entry = { at: at.toISOString(), ...record };
+ const historyFile = runtimeRestartLogPath(serverLogFile);
+ try {
+ mkdirSync(path.dirname(historyFile), { recursive: true });
+ appendFileSync(historyFile, `${JSON.stringify(entry)}\n`, "utf8");
+ trimHistory(historyFile);
+ } catch {
+ // Diagnostics only.
+ }
+ const attributes = {};
+ for (const [key, value] of Object.entries(record)) {
+ if (value !== undefined) attributes[`runtime.restart.${key.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`)}`] = value;
+ }
+ const line = {
+ timeUnixNano: `${BigInt(at.getTime()) * 1_000_000n}`,
+ severityText: record.action === "deferred" || record.action === "skipped" ? "INFO" : "WARN",
+ body: `Built-in ${record.kind === "engine" ? "engine" : "server"} ${record.action}: ${record.reason} (${record.source})`,
+ attributes,
+ resource: { "service.name": "omnirush-desktop-runtime" },
+ };
+ try {
+ mkdirSync(path.dirname(serverLogFile), { recursive: true });
+ appendFileSync(serverLogFile, `${JSON.stringify(line)}\n`, "utf8");
+ } catch {
+ // Diagnostics only.
+ }
+ return entry;
+}
+
+/** Newest first. Unreadable or partial lines are skipped. */
+export async function readRuntimeRestartRecords(serverLogFile, limit = 20) {
+ if (!serverLogFile) return [];
+ let text = "";
+ try {
+ text = await readFile(runtimeRestartLogPath(serverLogFile), "utf8");
+ } catch {
+ return [];
+ }
+ const records = [];
+ const lines = text.split("\n").filter(Boolean);
+ for (let index = lines.length - 1; index >= 0 && records.length < limit; index -= 1) {
+ try {
+ const value = JSON.parse(lines[index]);
+ if (value && typeof value === "object" && typeof value.at === "string") records.push(value);
+ } catch {
+ // Skip a torn line.
+ }
+ }
+ return records;
+}
+
+/**
+ * Whether a lifecycle request may stop the running server now.
+ *
+ * action: "engine-restart" | "server-restart" | "engine-start"
+ * busy: { sessions: number, unknown: number } — unknown probes count as busy
+ *
+ * Returns { verdict: "proceed" | "defer" | "skip", why }.
+ *
+ * @param {{ action: string, userInitiated?: boolean, serverRunning: boolean, serverHealthy: boolean,
+ * busy?: { sessions: number, unknown: number }, sameSettings?: boolean }} input
+ * @returns {{ verdict: "proceed" | "defer" | "skip", why: string }}
+ */
+export function decideRuntimeRestart({ action, userInitiated, serverRunning, serverHealthy, busy, sameSettings }) {
+ if (!serverRunning) return { verdict: "proceed", why: "server_not_running" };
+ if (userInitiated === true) return { verdict: "proceed", why: "user_initiated" };
+ if (!serverHealthy) return { verdict: "proceed", why: "server_unresponsive" };
+ // A healthy server already runs with what was asked for: a restart would
+ // only kill live runs. This covers renderer health checks that timed out
+ // while the server was merely busy, and renderer reloads re-running boot.
+ if (action === "engine-start") return { verdict: "skip", why: "server_healthy" };
+ if (action === "server-restart" && sameSettings === true) return { verdict: "skip", why: "server_healthy" };
+ if ((busy?.sessions ?? 0) > 0 || (busy?.unknown ?? 0) > 0) return { verdict: "defer", why: "sessions_busy" };
+ return { verdict: "proceed", why: "idle" };
+}
+
+/**
+ * Whether a deferred restart runs now: "run", "wait" or "drop" (the server
+ * it was meant for is gone; whoever starts the next one applies everything).
+ *
+ * @param {{ serverRunning: boolean, serverHealthy: boolean,
+ * busy?: { sessions: number, unknown: number }, deferredForMs: number }} input
+ * @returns {"run" | "wait" | "drop"}
+ */
+export function decideDeferredRestart({ serverRunning, serverHealthy, busy, deferredForMs }) {
+ if (!serverRunning) return "drop";
+ if (!serverHealthy) return "run";
+ if ((busy?.sessions ?? 0) > 0) return "wait";
+ if ((busy?.unknown ?? 0) > 0 && deferredForMs < DEFERRED_RESTART_UNKNOWN_MAX_MS) return "wait";
+ return "run";
+}
diff --git a/apps/desktop/electron/runtime.mjs b/apps/desktop/electron/runtime.mjs
index c2e71215..cda314ef 100644
--- a/apps/desktop/electron/runtime.mjs
+++ b/apps/desktop/electron/runtime.mjs
@@ -8,6 +8,12 @@ import path from "node:path";
import tls from "node:tls";
import { fileURLToPath } from "node:url";
import { pathToFileURL } from "node:url";
+import {
+ appendRuntimeRestartRecord,
+ decideDeferredRestart,
+ decideRuntimeRestart,
+ readRuntimeRestartRecords,
+} from "./runtime-restart-log.mjs";
import {
desktopBootstrapPath,
normalizeWorkspaceRootPath,
@@ -1888,6 +1894,187 @@ export function createRuntimeManager({
// In-process server handle. Kept alive across restarts so we can stop it.
let inProcessServer = null;
+ // ── Restart guard ───────────────────────────────────────────────────────
+ // Stopping the running server ends every live run (the collector closes
+ // each session and the engine is killed). Automatic triggers therefore never
+ // stop a healthy server that still has running sessions: the request waits
+ // until the engine is idle. Every stop is recorded with its reason.
+ const DEFERRED_RESTART_POLL_MS = 15_000;
+ let pendingRestart = null;
+ let pendingRestartTimer = null;
+ let pendingRestartChecking = false;
+ let nextStartReason = null;
+
+ function restartLogFile() {
+ return omnirushServerState.logFilePath ?? resolveOmniRushServerLogFile(userDataDir);
+ }
+
+ function recordRestart(record) {
+ appendRuntimeRestartRecord({
+ serverLogFile: restartLogFile(),
+ record: { kind: "server", generation: omnirushServerState.generation ?? null, ...record },
+ });
+ }
+
+ function restartRequest(options, fallbackSource) {
+ const reason = typeof options?.reason === "string" && options.reason.trim() ? options.reason.trim() : "unspecified";
+ const source = typeof options?.source === "string" && options.source.trim() ? options.source.trim() : fallbackSource;
+ return { reason, source, userInitiated: options?.userInitiated === true };
+ }
+
+ async function probeRunningServer() {
+ const baseUrl = omnirushServerState.baseUrl;
+ if (!inProcessServer || !baseUrl) return { running: false, healthy: false };
+ for (let attempt = 0; attempt < 2; attempt += 1) {
+ try {
+ await fetchJson(`${baseUrl.replace(/\/+$/, "")}/health`, {}, 5000);
+ return { running: true, healthy: true };
+ } catch {
+ // One retry: a single slow answer is not a dead server.
+ }
+ }
+ return { running: true, healthy: false };
+ }
+
+ async function readBusySessions() {
+ const server = inProcessServer;
+ if (typeof server?.busySessions !== "function") return { sessions: 0, unknown: 1 };
+ let timer = null;
+ try {
+ const result = await Promise.race([
+ server.busySessions(),
+ new Promise((_, reject) => {
+ timer = setTimeout(() => reject(new Error("busy session probe timed out")), 8000);
+ }),
+ ]);
+ return { sessions: result.sessions.length, unknown: result.unknown };
+ } catch {
+ return { sessions: 0, unknown: 1 };
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+ }
+
+ /**
+ * Returns { verdict, why, reason, source, userInitiated, busy }. Callers stop
+ * the server only on "proceed"; "defer" queues the request until idle and
+ * "skip" keeps the healthy server.
+ */
+ async function guardRestart(action, options, fallbackSource, extra = {}) {
+ const request = restartRequest(options, fallbackSource);
+ const probe = await probeRunningServer();
+ const busy = probe.running && probe.healthy ? await readBusySessions() : { sessions: 0, unknown: 0 };
+ const decision = decideRuntimeRestart({
+ action,
+ userInitiated: request.userInitiated,
+ serverRunning: probe.running,
+ serverHealthy: probe.healthy,
+ busy,
+ sameSettings: extra.sameSettings,
+ });
+ const result = { ...decision, ...request, busy };
+ if (decision.verdict === "proceed") {
+ if (probe.running) {
+ recordRestart({
+ action: "restarting",
+ reason: request.reason,
+ source: request.source,
+ trigger: action,
+ why: decision.why,
+ busySessions: busy.sessions,
+ busyUnknown: busy.unknown,
+ });
+ }
+ if (request.userInitiated) clearPendingRestart();
+ nextStartReason = request.reason;
+ } else {
+ recordRestart({
+ action: decision.verdict === "defer" ? "deferred" : "skipped",
+ reason: request.reason,
+ source: request.source,
+ trigger: action,
+ why: decision.why,
+ busySessions: busy.sessions,
+ busyUnknown: busy.unknown,
+ });
+ if (decision.verdict === "defer") queuePendingRestart(action, options, request);
+ }
+ return result;
+ }
+
+ function clearPendingRestart() {
+ pendingRestart = null;
+ if (pendingRestartTimer) clearInterval(pendingRestartTimer);
+ pendingRestartTimer = null;
+ }
+
+ function queuePendingRestart(action, options, request) {
+ pendingRestart = {
+ action,
+ options: { ...(options ?? {}) },
+ reason: request.reason,
+ source: request.source,
+ requestedAt: pendingRestart?.requestedAt ?? Date.now(),
+ };
+ if (pendingRestartTimer) return;
+ pendingRestartTimer = setInterval(() => void runPendingRestartIfIdle(), DEFERRED_RESTART_POLL_MS);
+ pendingRestartTimer.unref?.();
+ }
+
+ async function runPendingRestartIfIdle() {
+ const pending = pendingRestart;
+ if (!pending || pendingRestartChecking) return;
+ pendingRestartChecking = true;
+ try {
+ const probe = await probeRunningServer();
+ const busy = probe.running && probe.healthy ? await readBusySessions() : { sessions: 0, unknown: 0 };
+ const verdict = decideDeferredRestart({
+ serverRunning: probe.running,
+ serverHealthy: probe.healthy,
+ busy,
+ deferredForMs: Date.now() - pending.requestedAt,
+ });
+ if (verdict === "wait" || pendingRestart !== pending) return;
+ clearPendingRestart();
+ if (verdict === "drop") {
+ recordRestart({ action: "dropped", reason: pending.reason, source: pending.source, trigger: pending.action });
+ return;
+ }
+ const options = {
+ ...pending.options,
+ reason: `${pending.reason} (deferred until idle)`,
+ source: pending.source,
+ deferredRun: true,
+ };
+ await (pending.action === "engine-restart"
+ ? withRuntimeLifecycle(() => engineRestart(options))
+ : withRuntimeLifecycle(() => omnirushServerRestart(options))
+ ).catch((error) => {
+ appendOutput(omnirushServerState, "lastStderr", `Deferred restart failed: ${error instanceof Error ? error.message : String(error)}\n`);
+ });
+ } finally {
+ pendingRestartChecking = false;
+ }
+ }
+
+ function pendingRestartSummary() {
+ return pendingRestart
+ ? {
+ action: pendingRestart.action,
+ reason: pendingRestart.reason,
+ source: pendingRestart.source,
+ requestedAt: new Date(pendingRestart.requestedAt).toISOString(),
+ }
+ : null;
+ }
+
+ /** Record a stop that is not a restart request (quit, stop, fresh runtime). */
+ function recordStop(options, fallbackSource) {
+ if (!inProcessServer) return;
+ const request = restartRequest(options, fallbackSource);
+ recordRestart({ action: "stopping", reason: request.reason, source: request.source });
+ }
+
async function startOmniRushServer(options) {
// The inner start stops any previous runtime before mutating state, so a
// throw below always happens with nothing left running.
@@ -2044,6 +2231,9 @@ export function createRuntimeManager({
omnirushServerState.baseUrl = baseUrl;
omnirushServerState.clientToken = tokens.clientToken;
omnirushServerState.hostToken = tokens.hostToken;
+ recordRestart({ action: "started", reason: nextStartReason ?? "startup", source: "runtime" });
+ nextStartReason = null;
+ clearPendingRestart();
const connectUrls = options.remoteAccessEnabled ? buildConnectUrls(boundPort) : { connectUrl: null, mdnsUrl: null, lanUrl: null };
omnirushServerState.connectUrl = connectUrls.connectUrl;
@@ -2098,6 +2288,7 @@ export function createRuntimeManager({
async function stopAllRuntimeChildren() {
// Stop the in-process server (and its managed OpenCode child) if running.
+ clearPendingRestart();
if (inProcessServer) {
try { await inProcessServer.stop(); } catch { /* ignore */ }
inProcessServer = null;
@@ -2109,7 +2300,12 @@ export function createRuntimeManager({
Object.assign(omnirushServerState, createOmniRushServerState());
}
- async function prepareFreshRuntime() {
+ async function prepareFreshRuntime(options = {}) {
+ recordStop(options, "prepare-fresh-runtime");
+ await resetRuntime();
+ }
+
+ async function resetRuntime() {
lifecycleState = "cleaning";
await stopAllRuntimeChildren();
await cleanupPackagedSidecars();
@@ -2169,7 +2365,7 @@ export function createRuntimeManager({
// resolveOmniRushServerReuse also spans workspace switches: requesting a
// different projectDir retargets the running runtime instead of killing
// the process and every in-flight run with it.
- const reuseDecision = resolveOmniRushServerReuse({
+ let reuseDecision = resolveOmniRushServerReuse({
forceRestart: options.forceRestart,
inProcess: omnirushServerState.inProcess,
lifecycleState,
@@ -2179,6 +2375,20 @@ export function createRuntimeManager({
requestedProjectDir: safeProjectDir,
platform: workspacePlatform,
});
+ if (!reuseDecision.reuse && options.forceRestart !== true && inProcessServer) {
+ // Not a restart request, yet it would replace the running server
+ // (remote-access mismatch, or a lifecycle state left behind by an
+ // earlier failure). Only a user action may do that to a healthy one.
+ const guard = await guardRestart("engine-start", options, "engine-start");
+ if (guard.verdict === "skip") {
+ lifecycleState = "healthy";
+ reuseDecision = {
+ reuse: true,
+ retarget: normalizeWorkspaceKey(engineState.projectDir, workspacePlatform)
+ !== normalizeWorkspaceKey(safeProjectDir, workspacePlatform),
+ };
+ }
+ }
if (reuseDecision.reuse) {
const existing = snapshotOmniRushServerState(omnirushServerState);
if (existing.running && existing.baseUrl && (existing.ownerToken || existing.clientToken)) {
@@ -2211,7 +2421,7 @@ export function createRuntimeManager({
settleAfterWorkspacePreparationFailure();
throw error;
}
- await prepareFreshRuntime();
+ await resetRuntime();
const workspacePaths = prioritizeWorkspacePaths(safeProjectDir, options.workspacePaths, {
platform: workspacePlatform,
@@ -2241,7 +2451,8 @@ export function createRuntimeManager({
}
}
- async function engineStop() {
+ async function engineStop(options = {}) {
+ recordStop(options, "engine-stop");
lifecycleState = "stopping";
await stopAllRuntimeChildren();
lifecycleState = "idle";
@@ -2253,6 +2464,20 @@ export function createRuntimeManager({
if (!projectDir) {
throw new Error("OpenCode is not configured for a local workspace");
}
+ if (options.deferredRun === true) {
+ const request = restartRequest(options, "deferred");
+ recordRestart({ action: "restarting", reason: request.reason, source: request.source, trigger: "engine-restart", why: "idle" });
+ nextStartReason = request.reason;
+ } else {
+ const guard = await guardRestart("engine-restart", options, "engine-restart");
+ if (guard.verdict !== "proceed") {
+ return {
+ ...snapshotEngineState(engineState),
+ restartDeferred: guard.verdict === "defer",
+ busySessions: guard.busy.sessions,
+ };
+ }
+ }
const omnirushRemoteAccess = typeof options.omnirushRemoteAccess === "boolean"
? options.omnirushRemoteAccess
: omnirushServerState.remoteAccessEnabled;
@@ -2282,10 +2507,36 @@ export function createRuntimeManager({
}
async function omnirushServerInfo() {
- return snapshotOmniRushServerState(omnirushServerState);
+ return {
+ ...snapshotOmniRushServerState(omnirushServerState),
+ restarts: await readRuntimeRestartRecords(restartLogFile()),
+ pendingRestart: pendingRestartSummary(),
+ };
}
async function omnirushServerRestart(options = {}) {
+ // Keep the running host binding unless the caller asks to change it: a
+ // restart for another reason (account change) must not drop remote access.
+ const remoteAccessEnabled = typeof options.remoteAccessEnabled === "boolean"
+ ? options.remoteAccessEnabled
+ : omnirushServerState.remoteAccessEnabled === true;
+ if (options.deferredRun === true) {
+ const request = restartRequest(options, "deferred");
+ recordRestart({ action: "restarting", reason: request.reason, source: request.source, trigger: "server-restart", why: "idle" });
+ nextStartReason = request.reason;
+ } else {
+ const guard = await guardRestart("server-restart", options, "server-restart", {
+ sameSettings: omnirushServerState.remoteAccessEnabled === remoteAccessEnabled,
+ });
+ if (guard.verdict !== "proceed") {
+ return {
+ ...snapshotOmniRushServerState(omnirushServerState),
+ restartDeferred: guard.verdict === "defer",
+ restartSkipped: guard.verdict === "skip",
+ busySessions: guard.busy.sessions,
+ };
+ }
+ }
const workspacePaths = prioritizeWorkspacePaths(engineState.projectDir, await listLocalWorkspacePaths(), {
platform: workspacePlatform,
});
@@ -2297,7 +2548,7 @@ export function createRuntimeManager({
opencodeBaseUrl: shouldManageOpencode ? null : engineState.baseUrl,
opencodeUsername: shouldManageOpencode ? null : engineState.opencodeUsername,
opencodePassword: shouldManageOpencode ? null : engineState.opencodePassword,
- remoteAccessEnabled: options.remoteAccessEnabled === true,
+ remoteAccessEnabled,
manageOpencode: shouldManageOpencode,
opencodeBinPath: engineState.opencodeBinPath ?? omnirushServerState.managedOpencodeBinPath,
});
@@ -2382,10 +2633,13 @@ export function createRuntimeManager({
return {
systemCaCertificates: async () => (await systemCa()).trustedCertificates,
engineStart: (projectDir, options) => withRuntimeLifecycle(() => engineStart(projectDir, options)),
- engineStop: () => withRuntimeLifecycle(() => engineStop()),
+ engineStop: (options) => withRuntimeLifecycle(() => engineStop(options)),
engineRestart: (options) => withRuntimeLifecycle(() => engineRestart(options)),
- prepareFreshRuntime: () => withRuntimeLifecycle(() => prepareFreshRuntime()),
- dispose: () => withRuntimeLifecycle(() => stopAllRuntimeChildren()),
+ prepareFreshRuntime: (options) => withRuntimeLifecycle(() => prepareFreshRuntime(options)),
+ dispose: (options) => withRuntimeLifecycle(() => {
+ recordStop(options ?? { reason: "app_quit" }, "dispose");
+ return stopAllRuntimeChildren();
+ }),
runtimeStatus,
engineInfo,
engineDoctor,
diff --git a/apps/desktop/electron/runtime.test.mjs b/apps/desktop/electron/runtime.test.mjs
index e24221f8..44a4c30a 100644
--- a/apps/desktop/electron/runtime.test.mjs
+++ b/apps/desktop/electron/runtime.test.mjs
@@ -22,6 +22,14 @@ import {
snapshotEngineState,
snapshotOmniRushServerState,
} from "./runtime.mjs";
+import {
+ appendRuntimeRestartRecord,
+ decideDeferredRestart,
+ decideRuntimeRestart,
+ DEFERRED_RESTART_UNKNOWN_MAX_MS,
+ readRuntimeRestartRecords,
+ runtimeRestartLogPath,
+} from "./runtime-restart-log.mjs";
describe("workspace root preparation", () => {
it("reports an inaccessible Windows drive as a controlled recoverable error", async () => {
@@ -483,3 +491,132 @@ describe("resetRuntimeStatesAfterFailedServerStart", () => {
assert.equal(engineState.projectDir, "/workspace/current");
});
});
+
+describe("built-in server restart guard", () => {
+ const idle = { sessions: 0, unknown: 0 };
+ const busy = { sessions: 3, unknown: 0 };
+
+ it("never lets an automatic trigger stop a healthy server with running sessions", () => {
+ for (const action of ["engine-restart", "server-restart"]) {
+ assert.deepEqual(
+ decideRuntimeRestart({ action, userInitiated: false, serverRunning: true, serverHealthy: true, busy, sameSettings: false }),
+ { verdict: "defer", why: "sessions_busy" },
+ );
+ }
+ });
+
+ it("treats an unreadable busy probe as busy", () => {
+ assert.equal(
+ decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 } }).verdict,
+ "defer",
+ );
+ });
+
+ it("keeps a healthy server that already runs with the requested settings", () => {
+ // A renderer health check that timed out while the server was busy, or a
+ // renderer reload re-running boot, must not restart anything.
+ assert.equal(
+ decideRuntimeRestart({ action: "server-restart", serverRunning: true, serverHealthy: true, busy: idle, sameSettings: true }).verdict,
+ "skip",
+ );
+ assert.equal(
+ decideRuntimeRestart({ action: "engine-start", serverRunning: true, serverHealthy: true, busy }).verdict,
+ "skip",
+ );
+ });
+
+ it("restarts when the user asked, when nothing runs, when the server is gone, or when idle", () => {
+ assert.equal(decideRuntimeRestart({ action: "server-restart", userInitiated: true, serverRunning: true, serverHealthy: true, busy, sameSettings: true }).verdict, "proceed");
+ assert.equal(decideRuntimeRestart({ action: "server-restart", serverRunning: false, serverHealthy: false, busy: idle }).verdict, "proceed");
+ assert.deepEqual(
+ decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: false, busy }),
+ { verdict: "proceed", why: "server_unresponsive" },
+ );
+ assert.equal(decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: true, busy: idle }).verdict, "proceed");
+ });
+
+ it("runs a deferred restart once idle, and gives up waiting on an engine that stays unreadable", () => {
+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy, deferredForMs: 60 * 60_000 }), "wait");
+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: idle, deferredForMs: 0 }), "run");
+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 }, deferredForMs: 1_000 }), "wait");
+ assert.equal(
+ decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 }, deferredForMs: DEFERRED_RESTART_UNKNOWN_MAX_MS }),
+ "run",
+ );
+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: false, busy, deferredForMs: 0 }), "run");
+ assert.equal(decideDeferredRestart({ serverRunning: false, serverHealthy: false, busy: idle, deferredForMs: 0 }), "drop");
+ });
+});
+
+describe("built-in server restart record", () => {
+ it("writes each restart to omnirush-server.log and the history, newest first", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-restart-log-"));
+ try {
+ const serverLogFile = path.join(root, "logs", "omnirush-server.log");
+ appendRuntimeRestartRecord({
+ serverLogFile,
+ now: () => new Date("2026-09-24T07:11:00.000Z"),
+ record: { kind: "server", action: "deferred", reason: "engine_reload_failed", source: "engine-reload", busySessions: 3 },
+ });
+ appendRuntimeRestartRecord({
+ serverLogFile,
+ now: () => new Date("2026-09-24T07:40:00.000Z"),
+ record: { kind: "server", action: "restarting", reason: "engine_reload_failed (deferred until idle)", source: "engine-reload" },
+ });
+
+ const records = await readRuntimeRestartRecords(serverLogFile);
+ assert.deepEqual(records.map((record) => [record.at, record.action]), [
+ ["2026-09-24T07:40:00.000Z", "restarting"],
+ ["2026-09-24T07:11:00.000Z", "deferred"],
+ ]);
+ assert.equal(records[1].busySessions, 3);
+
+ const logLines = (await readFile(serverLogFile, "utf8")).trim().split("\n").map((line) => JSON.parse(line));
+ assert.equal(logLines.length, 2);
+ assert.equal(logLines[0].body, "Built-in server deferred: engine_reload_failed (engine-reload)");
+ assert.equal(logLines[0].attributes["runtime.restart.busy_sessions"], 3);
+ assert.equal(logLines[0].timeUnixNano, "1790233860000000000");
+ assert.equal(path.dirname(runtimeRestartLogPath(serverLogFile)), path.dirname(serverLogFile));
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+
+ it("skips torn lines and reports an empty history when there is none", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-restart-log-"));
+ try {
+ const serverLogFile = path.join(root, "omnirush-server.log");
+ assert.deepEqual(await readRuntimeRestartRecords(serverLogFile), []);
+ await writeFile(runtimeRestartLogPath(serverLogFile), '{"at":"2026-09-24T00:00:00.000Z","action":"started"}\n{"at":"2026-09-24T', "utf8");
+ assert.deepEqual((await readRuntimeRestartRecords(serverLogFile)).map((record) => record.action), ["started"]);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+
+ it("returns the history and any pending restart with the server info", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-runtime-restarts-"));
+ const previous = process.env.OMNIRUSH_SERVER_LOG_FILE;
+ delete process.env.OMNIRUSH_SERVER_LOG_FILE;
+ try {
+ appendRuntimeRestartRecord({
+ serverLogFile: resolveOmniRushServerLogFile(root),
+ record: { kind: "engine", action: "restarted", reason: "engine_unreachable:process_exited", source: "engine-pool-watchdog" },
+ });
+ const manager = createRuntimeManager({
+ app: { getPath: (name) => name === "exe" ? path.join(root, "OmniRush.ai.exe") : root, isPackaged: false },
+ desktopRoot: path.dirname(fileURLToPath(import.meta.url)),
+ listLocalWorkspacePaths: async () => [],
+ localManagedMcpVaultKey: "test-key",
+ });
+ const info = await manager.omnirushServerInfo();
+ assert.equal(info.running, false);
+ assert.equal(info.pendingRestart, null);
+ assert.deepEqual(info.restarts.map((record) => record.reason), ["engine_unreachable:process_exited"]);
+ } finally {
+ if (previous === undefined) delete process.env.OMNIRUSH_SERVER_LOG_FILE;
+ else process.env.OMNIRUSH_SERVER_LOG_FILE = previous;
+ await rm(root, { recursive: true, force: true });
+ }
+ });
+});
diff --git a/apps/server/src/embedded.ts b/apps/server/src/embedded.ts
index 9a2860a2..3b6ab60d 100644
--- a/apps/server/src/embedded.ts
+++ b/apps/server/src/embedded.ts
@@ -26,6 +26,7 @@ import {
import { createManagedOpencodeServer, type ManagedOpencodeServer, type OpencodeExecutionSnapshot } from "./managed-opencode.js";
import {
clearTrustedOpencodeProcess,
+ collectBusyEngineSessions,
createEnginePoolForConfig,
createServerLogger,
registerTrustedOpencodeProcess,
@@ -78,6 +79,11 @@ export type EmbeddedServerHandle = {
managedOpencode: { pid: number | null; isAlive: () => boolean } | null;
/** Current managed-engine generations for desktop diagnostics and acceptance checks. */
managedOpencodePool: () => EnginePoolSnapshot | null;
+ /**
+ * Sessions still running on the managed engine. The desktop asks before any
+ * unattended restart; `unknown` counts unreadable probes, which it treats as busy.
+ */
+ busySessions: () => Promise<{ sessions: string[]; unknown: number }>;
/** Stop the HTTP server and managed OpenCode (if any). */
stop: () => Promise<void>;
};
@@ -364,6 +370,7 @@ export async function startEmbeddedServer(options: EmbeddedServerOptions): Promi
}
: null,
managedOpencodePool: () => enginePool?.snapshot() ?? null,
+ busySessions: () => collectBusyEngineSessions(config),
stop,
};
}
diff --git a/apps/server/src/engine-pool.ts b/apps/server/src/engine-pool.ts
index 1a412858..f1287414 100644
--- a/apps/server/src/engine-pool.ts
+++ b/apps/server/src/engine-pool.ts
@@ -28,6 +28,7 @@ import {
updateEngineInstanceRole,
} from "./engine-registry.js";
import { createManagedOpencodeServer, type ManagedOpencodeServer } from "./managed-opencode.js";
+import { appendRuntimeRestartRecord } from "./runtime-restart-log.js";
import { loopbackFetch } from "./server-fetch.js";
import type { ServerConfig, WorkspaceInfo } from "./types.js";
@@ -81,6 +82,17 @@ export type EnginePoolHooks = {
now?: () => number;
schedule?: (operation: () => void, delayMs: number) => ReturnType<typeof setTimeout>;
waitForHealthy?: (handle: ManagedOpencodeServer) => Promise<void>;
+ /**
+ * One liveness probe of a running engine: "ok" when it answered at all,
+ * "refused" when nothing listens on its port, "failed" for anything else
+ * (timeout, reset). Defaults to GET /global/health.
+ */
+ probeHealth?: (handle: ManagedOpencodeServer) => Promise<"ok" | "refused" | "failed">;
+ /**
+ * Called right before the watchdog closes a primary it confirmed dead, so
+ * the runs it was carrying can be queued for task recovery.
+ */
+ onPrimaryLost?: (config: ServerConfig) => void;
/** Called once when the pool shuts down, before its engines are retired. */
onDisposed?: () => void;
logger?: EnginePoolLogger;
@@ -174,6 +186,17 @@ function minSpawnIntervalMs(): number {
return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS", 30_000);
}
+/**
+ * Spacing between the watchdog's liveness probes of a primary that stopped
+ * answering proxied requests. Three probes spread over this interval must all
+ * fail before a live engine process is closed.
+ */
+function recoveryProbeIntervalMs(): number {
+ return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS", 10_000);
+}
+
+const RECOVERY_PROBE_ATTEMPTS = 3;
+
function drainPollIntervalMs(): number {
return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_POLL_MS", 5_000);
}
@@ -216,6 +239,15 @@ export function isEngineConnectionFailure(error: unknown): boolean {
return false;
}
+function isConnectionRefused(error: unknown): boolean {
+ let current: unknown = error;
+ for (let depth = 0; depth < 6 && isRecord(current); depth += 1) {
+ if (current.code === "ECONNREFUSED") return true;
+ current = current.cause;
+ }
+ return false;
+}
+
export function isConnectionRefusedClassError(error: unknown): boolean {
return isEngineConnectionFailure(error);
}
@@ -447,9 +479,14 @@ export class EnginePool {
if (this.consecutiveConnectionFailures < 3) return;
const now = this.now();
if (now - this.lastRecoveryAt < minSpawnIntervalMs()) return;
+ const failures = this.consecutiveConnectionFailures;
this.consecutiveConnectionFailures = 0;
this.lastRecoveryAt = now;
this.recoveryWorkspace = workspace;
+ this.hooks.logger?.log("warn", "Managed engine stopped answering; checking whether it is still alive.", {
+ "engine.recovery.consecutive_failures": failures,
+ "engine.recovery.last_error": error instanceof Error ? error.message : String(error),
+ });
void this.recoverDeadPrimary(workspace).catch(() => undefined);
}
@@ -569,6 +606,41 @@ export class EnginePool {
};
}
+ /**
+ * Non-idle sessions on every routable engine, for the desktop's restart
+ * guard. `unknown` counts the probes that could not be read; callers treat
+ * those as busy. Only the given directories are probed, so an instance the
+ * engine never booted is not booted just to learn that it is idle.
+ */
+ async activeSessions(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {
+ const sessions = new Set<string>();
+ let unknown = 0;
+ const targets = this.generations.filter(isRoutableGeneration);
+ await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {
+ if (!generation.handle.isAlive()) return;
+ try {
+ const url = new URL("/session/status", generation.handle.url);
+ url.searchParams.set("directory", directory);
+ const response = await loopbackFetch(url.toString(), {
+ headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },
+ signal: AbortSignal.timeout(5_000),
+ });
+ if (!response.ok) throw new Error(`status ${response.status}`);
+ const payload: unknown = await response.json();
+ if (!isRecord(payload)) throw new Error("unreadable status");
+ for (const [sessionId, status] of Object.entries(payload)) {
+ if (isRecord(status) && status.type !== "idle") sessions.add(sessionId);
+ }
+ } catch {
+ unknown += 1;
+ }
+ })));
+ for (const owned of this.activeSessionsByGeneration.values()) {
+ for (const sessionId of owned) sessions.add(sessionId);
+ }
+ return { sessions: [...sessions], unknown };
+ }
+
/** Provider sync treats an active drain as busy so it defers instead of parking on waitForDrain(). */
hasDrainingGeneration(): boolean {
return this.generations.some((entry) => entry.status === "draining");
@@ -645,7 +717,9 @@ export class EnginePool {
const busy = forceStandby
? true
- : await this.hooks.engineBusy(this.config, workspace).catch(() => false);
+ // Unknown activity counts as busy: a busy engine rolls over to a
+ // standby, while an in-place dispose would abort whatever it runs.
+ : await this.hooks.engineBusy(this.config, workspace).catch(() => true);
if (!busy) {
await this.hooks.reloadInPlace(this.config, workspace, { awaitPostRefreshSync });
if (primary) primary.fingerprint = fingerprint;
@@ -1082,6 +1156,31 @@ export class EnginePool {
this.recoveryOrphan = null;
}
if (previous) {
+ const liveness = await this.confirmPrimaryDead(previous);
+ if (!liveness.dead) {
+ // A slow or overloaded engine still carries every live run; closing
+ // it here would abort them all. Only a dead one is replaced.
+ this.consecutiveConnectionFailures = 0;
+ this.hooks.logger?.log("warn", "Managed engine recovery skipped: the engine still answers its health check.", {
+ "engine.recovery.reason": liveness.reason,
+ });
+ return;
+ }
+ this.hooks.logger?.log("warn", "Managed engine recovery started.", {
+ "engine.recovery.reason": liveness.reason,
+ "engine.pid": previous.handle.pid ?? null,
+ });
+ appendRuntimeRestartRecord({
+ kind: "engine",
+ action: "restarted",
+ reason: `engine_unreachable:${liveness.reason}`,
+ source: "engine-pool-watchdog",
+ });
+ try {
+ this.hooks.onPrimaryLost?.(this.config);
+ } catch {
+ // Recovery bookkeeping must never block replacing a dead engine.
+ }
if (previous.trustedIdentity) {
try {
this.hooks.clearTrusted(this.config, previous.trustedIdentity);
@@ -1151,6 +1250,40 @@ export class EnginePool {
}
}
+ /**
+ * Decide whether a primary that failed several proxied requests is really
+ * gone. A process that exited or a port nobody listens on is dead at once;
+ * timeouts and resets are only trusted after every spaced probe fails.
+ */
+ private async confirmPrimaryDead(generation: Generation): Promise<{ dead: boolean; reason: string }> {
+ for (let attempt = 1; attempt <= RECOVERY_PROBE_ATTEMPTS; attempt += 1) {
+ if (!generation.handle.isAlive()) return { dead: true, reason: "process_exited" };
+ const verdict = await this.probeHealth(generation.handle);
+ if (verdict === "ok") return { dead: false, reason: "health_ok" };
+ if (verdict === "refused") return { dead: true, reason: "connection_refused" };
+ if (attempt < RECOVERY_PROBE_ATTEMPTS) {
+ await new Promise((resolve) => {
+ const timer = setTimeout(resolve, recoveryProbeIntervalMs());
+ timer.unref?.();
+ });
+ }
+ }
+ return { dead: true, reason: "health_probe_failed" };
+ }
+
+ private async probeHealth(handle: ManagedOpencodeServer): Promise<"ok" | "refused" | "failed"> {
+ if (this.hooks.probeHealth) return this.hooks.probeHealth(handle);
+ try {
+ await loopbackFetch(new URL("/global/health", handle.url).toString(), {
+ headers: { Authorization: buildEngineAuthProbeHeader(handle.username, handle.password) },
+ signal: AbortSignal.timeout(5_000),
+ });
+ return "ok";
+ } catch (error) {
+ return isConnectionRefused(error) ? "refused" : "failed";
+ }
+ }
+
private scheduleRecovery(workspace: WorkspaceInfo): void {
if (this.disposed || this.recoveryTimer) return;
this.recoveryWorkspace = workspace;
diff --git a/apps/server/src/engine-self-heal.test.ts b/apps/server/src/engine-self-heal.test.ts
index 50ef7ccd..0cd12872 100644
--- a/apps/server/src/engine-self-heal.test.ts
+++ b/apps/server/src/engine-self-heal.test.ts
@@ -5,10 +5,13 @@ import { createManagedProcessClose, type ManagedChildProcess, type ManagedOpenco
import type { ServerConfig, WorkspaceInfo } from "./types.js";
const previousSpawnInterval = process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS;
+const previousProbeInterval = process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS;
afterEach(() => {
if (previousSpawnInterval === undefined) delete process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS;
else process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = previousSpawnInterval;
+ if (previousProbeInterval === undefined) delete process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS;
+ else process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS = previousProbeInterval;
});
function managedHandle(port: number) {
@@ -29,7 +32,7 @@ function managedHandle(port: number) {
return { handle, closeCalls: () => closeCalls };
}
-function fixture(spawn: EnginePoolHooks["spawn"]) {
+function fixture(spawn: EnginePoolHooks["spawn"], overrides: Partial<EnginePoolHooks> = {}) {
let now = 0;
const scheduled: Array<() => void> = [];
const workspace: WorkspaceInfo = {
@@ -75,6 +78,9 @@ function fixture(spawn: EnginePoolHooks["spawn"]) {
return setTimeout(() => undefined, 60_000);
},
waitForHealthy: async () => undefined,
+ // Nothing listens on these fake engines' ports.
+ probeHealth: async () => "refused",
+ ...overrides,
};
const pool = new EnginePool({ config, template, hooks });
return { pool, config, workspace, scheduled, setNow: (value: number) => { now = value; } };
@@ -184,6 +190,105 @@ describe("managed engine self-heal", () => {
await testFixture.pool.disposeAll();
});
+ test("keeps a live primary that still answers its health check", async () => {
+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";
+ const old = managedHandle(44001);
+ let spawnCalls = 0;
+ let lost = 0;
+ const testFixture = fixture(async () => {
+ spawnCalls += 1;
+ return managedHandle(44002).handle;
+ }, { probeHealth: async () => "ok", onPrimaryLost: () => { lost += 1; } });
+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });
+
+ // An overloaded engine resets or times out proxied requests but is alive.
+ for (let failure = 0; failure < 3; failure += 1) {
+ testFixture.pool.reportRequestFailure(old.handle.url, connectionReset(), testFixture.workspace);
+ }
+ await settle();
+
+ expect(spawnCalls).toBe(0);
+ expect(lost).toBe(0);
+ expect(old.closeCalls()).toBe(0);
+ expect(testFixture.pool.primaryUrl()).toBe(old.handle.url);
+ await testFixture.pool.disposeAll();
+ });
+
+ test("replaces an unresponsive live primary only after every spaced probe fails", async () => {
+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";
+ process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS = "1";
+ const old = managedHandle(45001);
+ const replacement = managedHandle(45002);
+ let probes = 0;
+ let lost = 0;
+ const testFixture = fixture(async () => replacement.handle, {
+ probeHealth: async () => {
+ probes += 1;
+ return "failed";
+ },
+ onPrimaryLost: () => { lost += 1; },
+ });
+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });
+
+ for (let failure = 0; failure < 3; failure += 1) {
+ testFixture.pool.reportRequestFailure(old.handle.url, timedOut(), testFixture.workspace);
+ }
+ await settle();
+
+ expect(probes).toBe(3);
+ expect(lost).toBe(1);
+ expect(old.handle.isAlive()).toBe(false);
+ expect(testFixture.pool.primaryUrl()).toBe(replacement.handle.url);
+ await testFixture.pool.disposeAll();
+ });
+
+ test("replaces a primary whose process already exited without probing it", async () => {
+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";
+ const old = managedHandle(46001);
+ const replacement = managedHandle(46002);
+ let probes = 0;
+ const testFixture = fixture(async () => replacement.handle, {
+ probeHealth: async () => {
+ probes += 1;
+ return "ok";
+ },
+ });
+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });
+ await old.handle.close();
+
+ for (let failure = 0; failure < 3; failure += 1) {
+ testFixture.pool.reportRequestFailure(old.handle.url, refused(), testFixture.workspace);
+ }
+ await settle();
+
+ expect(probes).toBe(0);
+ expect(testFixture.pool.primaryUrl()).toBe(replacement.handle.url);
+ await testFixture.pool.disposeAll();
+ });
+
+ test("a reload whose busy probe fails rolls over instead of disposing the engine in place", async () => {
+ const old = managedHandle(47001);
+ const standby = managedHandle(47002);
+ let inPlace = 0;
+ let spawnCalls = 0;
+ const testFixture = fixture(async () => {
+ spawnCalls += 1;
+ return standby.handle;
+ }, {
+ engineBusy: async () => { throw new Error("status probe timed out"); },
+ reloadInPlace: async () => { inPlace += 1; },
+ });
+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });
+
+ const outcome = await testFixture.pool.requestRollover({ reason: "test", workspace: testFixture.workspace, manual: true });
+
+ expect(outcome.action).toBe("rolled_over");
+ expect(inPlace).toBe(0);
+ expect(spawnCalls).toBe(1);
+ expect(testFixture.pool.primaryUrl()).toBe(standby.handle.url);
+ await testFixture.pool.disposeAll();
+ });
+
test("dispose escalates an ignored SIGTERM and waits for the child exit", async () => {
const signals: Array<NodeJS.Signals | number | undefined> = [];
const exitListeners: Array<() => void> = [];
diff --git a/apps/server/src/runtime-restart-log.ts b/apps/server/src/runtime-restart-log.ts
new file mode 100644
index 00000000..7477739b
--- /dev/null
+++ b/apps/server/src/runtime-restart-log.ts
@@ -0,0 +1,43 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import { SERVER_LOG_FILE_ENV } from "./server-log-file.js";
+
+/**
+ * Persisted record of every built-in server and managed engine restart.
+ *
+ * The desktop runtime (apps/desktop/electron/runtime-restart-log.mjs) writes
+ * server restarts to the same JSON-lines file next to omnirush-server.log;
+ * the server adds the engine recoveries it performs on its own. The desktop
+ * reads the tail back into omnirushServerInfo for the diagnostics bundle.
+ */
+export const RUNTIME_RESTART_LOG_FILE_NAME = "runtime-restarts.jsonl";
+
+export type RuntimeRestartRecord = {
+ at: string;
+ kind: "server" | "engine";
+ action: string;
+ reason: string;
+ source: string;
+ busySessions?: number | null;
+ detail?: string | null;
+};
+
+export function runtimeRestartLogPath(env: NodeJS.ProcessEnv = process.env): string | null {
+ const logFile = env[SERVER_LOG_FILE_ENV]?.trim();
+ return logFile ? join(dirname(logFile), RUNTIME_RESTART_LOG_FILE_NAME) : null;
+}
+
+/** Best effort: a record that cannot be written never blocks a recovery. */
+export function appendRuntimeRestartRecord(
+ record: Omit<RuntimeRestartRecord, "at"> & { at?: string },
+ path: string | null = runtimeRestartLogPath(),
+): void {
+ if (!path) return;
+ try {
+ mkdirSync(dirname(path), { recursive: true });
+ appendFileSync(path, `${JSON.stringify({ ...record, at: record.at ?? new Date().toISOString() })}\n`, "utf8");
+ } catch {
+ // Diagnostics only.
+ }
+}
diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts
index fef14ed9..a73cd580 100644
--- a/apps/server/src/server.ts
+++ b/apps/server/src/server.ts
@@ -1,6 +1,15 @@
import { managedDesktopPolicy } from "./managed-desktop-policy.js";
import { parseApprovalMode, resolveApprovalMode } from "./approval-mode.js";
-import { createTaskRecovery, setTaskRecovery } from "./task-recovery.js";
+import { createTaskRecovery, requeueTaskRecoveryAfterEngineLoss, setTaskRecovery } from "./task-recovery.js";
+import {
+ directoryMatchKey,
+ EmptySessionDirectoryCache,
+ isSessionListRequest,
+ listSessionsByDirectory,
+ serveSessionListWithFallback,
+ sessionListFallbackDeadlineMs,
+ sessionListFallbackQuery,
+} from "./session-list-fallback.js";
import { managedPolicyActionSchema } from "./managed-policy-rules.js";
import { readFile, realpath, writeFile, rm, stat } from "node:fs/promises";
import { homedir, hostname } from "node:os";
@@ -659,7 +668,12 @@ async function assertWorkspaceOwnsProxiedSessionRead(
realpath(sessionDirectory).catch(() => sessionDirectory),
])
: [directory, sessionDirectory];
- if (!actualDirectory || actualDirectory !== expectedDirectory) {
+ // Local folders compare the way their file system does: a session created
+ // before a case-only rename (Windows, macOS) still belongs to the folder.
+ const sameDirectory = workspace.workspaceType === "local" && actualDirectory
+ ? directoryMatchKey(actualDirectory) === directoryMatchKey(expectedDirectory)
+ : actualDirectory === expectedDirectory;
+ if (!actualDirectory || !sameDirectory) {
throw new ApiError(404, "session_not_found", "Session not found");
}
}
@@ -1857,6 +1871,14 @@ export async function proxyOpencodeRequest(input: {
kind: engineAggregateKind(proxyPath) ?? "pending",
});
}
+ if (pool && workspace && workspace.workspaceType !== "remote" && method === "POST"
+ && normalizeOpencodeProxyPath(proxyPath) === "/instance/dispose") {
+ // A client-side dispose (provider refresh) would abort every run in this
+ // folder. The pool applies it the same way without that: in place when
+ // idle, through a standby while sessions are running.
+ await reloadOpencodeEngine(input.config, workspace, undefined, { reason: "client_instance_dispose", manual: true });
+ return jsonResponse(true);
+ }
const targetUrl = buildOpencodeProxyUrl(baseUrl, proxyPath, search);
// Managed OpenCode proxy traffic is loopback/engine I/O; keep streaming on Node fetch.
if (isSessionCommandProxyRequest(method, proxyPath)) {
@@ -1950,9 +1972,66 @@ export async function proxyOpencodeRequest(input: {
if (workspace && workspace.workspaceType !== "remote" && isPromptAsyncProxyRequest(method, proxyPath)) {
return withEngineDirectoryFence(input.config, workspace, forwardAndCollect);
}
+ if (pool && workspace && workspace.workspaceType !== "remote" && isSessionListRequest(method, normalizeOpencodeProxyPath(proxyPath))) {
+ return serveLocalSessionList(input.config, workspace, search, forwardAndCollect());
+ }
return forwardAndCollect();
}
+const emptySessionDirectoriesByServer = new WeakMap<ServerConfig, EmptySessionDirectoryCache>();
+
+/**
+ * A local workspace's session list must not depend on its engine instance
+ * finishing a cold boot, or on the stored project id and directory string
+ * still matching the folder exactly. See session-list-fallback.ts.
+ */
+async function serveLocalSessionList(
+ config: ServerConfig,
+ workspace: WorkspaceInfo,
+ search: string,
+ primary: Promise<Response>,
+): Promise<Response> {
+ const directory = resolveOpencodeDirectory(workspace) ?? workspace.path;
+ const cacheKey = directoryMatchKey(directory);
+ const cache = emptySessionDirectoriesByServer.get(config) ?? new EmptySessionDirectoryCache();
+ emptySessionDirectoriesByServer.set(config, cache);
+ const query = sessionListFallbackQuery(search);
+ return serveSessionListWithFallback({
+ primary,
+ deadlineMs: sessionListFallbackDeadlineMs(),
+ fallback: async (cause) => {
+ if (cause === "empty" && cache.has(cacheKey)) return [];
+ const engine = primaryManagedEngineConnection(config);
+ if (!engine) throw new Error("The managed engine is unavailable for the session index");
+ const canonical = await realpath(directory).catch(() => directory);
+ const items = await listSessionsByDirectory({
+ directories: [workspace.path, directory, canonical],
+ query,
+ fetchPage: async (params) => {
+ const response = await loopbackFetch(`${engine.baseUrl.replace(/\/+$/, "")}/experimental/session?${params.toString()}`, {
+ headers: { Authorization: buildEngineAuthProbeHeader(engine.username, engine.password) },
+ signal: AbortSignal.timeout(10_000),
+ });
+ if (!response.ok) throw new Error(`OpenCode session index failed with status ${response.status}`);
+ const payload: unknown = await response.json();
+ if (!Array.isArray(payload)) throw new Error("OpenCode session index returned an unreadable page");
+ return { items: payload, nextCursor: response.headers.get("x-next-cursor") };
+ },
+ });
+ if (items.length === 0) cache.remember(cacheKey);
+ else cache.forget(cacheKey);
+ return items;
+ },
+ onFallback: ({ cause, count }) => {
+ createServerLogger(config).log("warn", "Session list served from the engine session index.", {
+ "workspace.id": workspace.id,
+ "session_list.fallback_cause": cause,
+ "session_list.count": count,
+ });
+ },
+ });
+}
+
function isEngineEventPath(proxyPath: string): boolean {
const normalized = normalizeOpencodeProxyPath(proxyPath);
return normalized === "/event" || normalized === "/global/event" || normalized === "/api/event";
@@ -4788,19 +4867,37 @@ async function requireWorkspaceRunModeIdle(config: ServerConfig, workspace: Work
/**
* True when the managed engine reports any non-idle session (subagent child
* sessions carry their own ids and statuses, so they count too). Unknown
- * activity reports false: a reload against a dead engine fails loudly on its
- * own, and "unknown" must never park reloads forever.
+ * activity reports `unknownIsBusy`: false for callers that would otherwise
+ * park a reload forever against a dead engine, true for the rollover pool,
+ * where "busy" only means rolling over to a standby instead of disposing an
+ * engine that may still be running sessions.
*/
-async function engineHasActiveSessions(config: ServerConfig, workspace: WorkspaceInfo): Promise<boolean> {
+async function engineHasActiveSessions(
+ config: ServerConfig,
+ workspace: WorkspaceInfo,
+ unknownIsBusy = false,
+): Promise<boolean> {
try {
const opencode = createWorkspaceOpencodeClient(config, workspace);
const statuses = unwrapOpencodeResult(await opencode.session.status(), "/session/status");
return Object.values(statuses).some((status) => status.type !== "idle");
} catch {
- return false;
+ return unknownIsBusy;
}
}
+/**
+ * Sessions still running on the managed engine, for the desktop's restart
+ * guard. Only directories the engine already serves are probed; `unknown`
+ * counts probes that could not be read and callers treat them as busy.
+ */
+export async function collectBusyEngineSessions(config: ServerConfig): Promise<{ sessions: string[]; unknown: number }> {
+ const pool = enginePoolForConfig(config);
+ if (!pool) return { sessions: [], unknown: 0 };
+ const directories = new Set(engineInstanceReaperForConfig(config)?.snapshot().map((instance) => instance.directory) ?? []);
+ return pool.activeSessions([...directories]);
+}
+
function primaryManagedEngineConnection(config: ServerConfig): EnginePoolConnection | null {
return enginePoolForConfig(config)?.connections().find((entry) => entry.role === "primary") ?? null;
}
@@ -5675,7 +5772,8 @@ export function createEnginePoolForConfig(input: {
onDisposed: () => threadApprovals?.stop(),
reloadInPlace: (poolConfig, workspace, options) =>
reloadOpencodeEngineInPlace(poolConfig, workspace, undefined, options),
- engineBusy: (poolConfig, workspace) => engineHasActiveSessions(poolConfig, workspace),
+ engineBusy: (poolConfig, workspace) => engineHasActiveSessions(poolConfig, workspace, true),
+ onPrimaryLost: (poolConfig) => requeueTaskRecoveryAfterEngineLoss(poolConfig),
postRefreshSync: async (poolConfig, workspace) => {
await postEngineRefreshSync(poolConfig, workspace, activeEngineMcpServerState(poolConfig));
await syncAllWorkspacesRuntimeMcpToEngine(poolConfig);
diff --git a/apps/server/src/session-list-fallback.test.ts b/apps/server/src/session-list-fallback.test.ts
new file mode 100644
index 00000000..43d0f058
--- /dev/null
+++ b/apps/server/src/session-list-fallback.test.ts
@@ -0,0 +1,203 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+ directoryMatchKey,
+ EmptySessionDirectoryCache,
+ isSessionListRequest,
+ listSessionsByDirectory,
+ serveSessionListWithFallback,
+ sessionListFallbackQuery,
+ type GlobalSessionPage,
+} from "./session-list-fallback.js";
+
+function session(id: string, directory: string, updated: number) {
+ return { id, directory, parentID: undefined, title: id, time: { created: updated, updated }, project: { id: "p", worktree: directory } };
+}
+
+/** The engine's global index: newest first, `cursor` = last item's time.updated. */
+function globalIndex(items: ReturnType<typeof session>[]) {
+ const calls: URLSearchParams[] = [];
+ const fetchPage = async (params: URLSearchParams): Promise<GlobalSessionPage> => {
+ calls.push(params);
+ const limit = Number(params.get("limit"));
+ const cursor = params.get("cursor");
+ const sorted = [...items].sort((left, right) => right.time.updated - left.time.updated);
+ const eligible = cursor ? sorted.filter((item) => item.time.updated < Number(cursor)) : sorted;
+ const page = eligible.slice(0, limit);
+ const more = eligible.length > limit && page.length > 0;
+ return { items: page, nextCursor: more ? String(page[page.length - 1]!.time.updated) : null };
+ };
+ return { fetchPage, calls };
+}
+
+describe("directoryMatchKey", () => {
+ test("Windows folders compare case-insensitively across slash styles, prefixes and trailing separators", () => {
+ const current = "C:\\Users\\vedant\\Desktop\\RepoRadar";
+ for (const stored of [
+ "C:/Users/vedant/Desktop/reporadar",
+ "c:/users/vedant/desktop/RepoRadar/",
+ "\\\\?\\C:\\Users\\vedant\\Desktop\\RepoRadar",
+ "C:\\Users\\vedant\\\\Desktop\\RepoRadar\\",
+ ]) {
+ expect(directoryMatchKey(stored, "win32")).toBe(directoryMatchKey(current, "win32"));
+ }
+ });
+
+ test("Windows keeps different folders, drives and UNC shares apart", () => {
+ const key = directoryMatchKey("C:\\Users\\vedant\\taskforge", "win32");
+ expect(directoryMatchKey("D:\\Users\\vedant\\taskforge", "win32")).not.toBe(key);
+ expect(directoryMatchKey("C:\\Users\\vedant\\taskforge2", "win32")).not.toBe(key);
+ expect(directoryMatchKey("\\\\?\\UNC\\server\\share\\repo", "win32")).toBe(directoryMatchKey("\\\\server\\share\\repo", "win32"));
+ expect(directoryMatchKey("\\\\server\\share\\repo", "win32")).not.toBe(directoryMatchKey("C:\\share\\repo", "win32"));
+ expect(directoryMatchKey("C:\\", "win32")).toBe("c:/");
+ });
+
+ test("Linux stays case-sensitive; macOS default volumes do not", () => {
+ expect(directoryMatchKey("/home/a/Repo", "linux")).not.toBe(directoryMatchKey("/home/a/repo", "linux"));
+ expect(directoryMatchKey("/home/a/repo/", "linux")).toBe(directoryMatchKey("/home/a/repo", "linux"));
+ expect(directoryMatchKey("/Users/a/RepoRadar", "darwin")).toBe(directoryMatchKey("/Users/a/reporadar", "darwin"));
+ expect(directoryMatchKey("/", "linux")).toBe("/");
+ });
+});
+
+describe("session list request detection", () => {
+ test("only the bare list is served with the fallback", () => {
+ expect(isSessionListRequest("GET", "/session")).toBe(true);
+ expect(isSessionListRequest("POST", "/session")).toBe(false);
+ expect(isSessionListRequest("GET", "/session/status")).toBe(false);
+ expect(isSessionListRequest("GET", "/session/ses_1")).toBe(false);
+ });
+
+ test("the query mirrors the list's roots, start, search and limit", () => {
+ expect(sessionListFallbackQuery("?limit=200&directory=C%3A%5Cx")).toEqual({ roots: undefined, start: undefined, search: undefined, limit: 200 });
+ expect(sessionListFallbackQuery("?roots=true&start=5&search=%20fix%20")).toEqual({ roots: true, start: 5, search: "fix", limit: 100 });
+ });
+});
+
+describe("listSessionsByDirectory", () => {
+ test("finds a re-cased Windows folder's sessions across pages and drops the project join", async () => {
+ const items = [
+ ...Array.from({ length: 700 }, (_, index) => session(`other_${index}`, "C:/Users/vedant/helm", 10_000 - index)),
+ session("ses_old_case", "C:/Users/vedant/reporadar", 1_000),
+ session("ses_new_case", "C:/Users/vedant/RepoRadar", 2_000),
+ ];
+ const index = globalIndex(items);
+ const result = await listSessionsByDirectory({
+ fetchPage: index.fetchPage,
+ directories: ["C:\\Users\\vedant\\RepoRadar"],
+ query: { limit: 200 },
+ platform: "win32",
+ });
+ expect(result.map((item) => item.id)).toEqual(["ses_new_case", "ses_old_case"]);
+ expect(result[0]).not.toHaveProperty("project");
+ expect(index.calls).toHaveLength(2);
+ expect(index.calls[0]!.get("archived")).toBe("true");
+ });
+
+ test("stops at the limit and at the scan budget", async () => {
+ const items = Array.from({ length: 50 }, (_, index) => session(`ses_${index}`, "/w", 100 - index));
+ const limited = await listSessionsByDirectory({ fetchPage: globalIndex(items).fetchPage, directories: ["/w"], query: { limit: 3 }, platform: "linux" });
+ expect(limited.map((item) => item.id)).toEqual(["ses_0", "ses_1", "ses_2"]);
+
+ const index = globalIndex(Array.from({ length: 2_000 }, (_, index) => session(`x_${index}`, "/other", 5_000 - index)));
+ const none = await listSessionsByDirectory({ fetchPage: index.fetchPage, directories: ["/w"], query: { limit: 10 }, platform: "linux", maxScanned: 1_000 });
+ expect(none).toEqual([]);
+ expect(index.calls).toHaveLength(2);
+ });
+});
+
+function delayed<T>(value: T, ms: number): Promise<T> {
+ return new Promise((resolve) => setTimeout(() => resolve(value), ms));
+}
+
+describe("serveSessionListWithFallback", () => {
+ const indexed = [{ id: "ses_a", directory: "C:/w" }];
+
+ test("a quick non-empty engine answer passes through untouched", async () => {
+ let fallbackCalls = 0;
+ const response = await serveSessionListWithFallback({
+ primary: Promise.resolve(Response.json([{ id: "ses_engine" }])),
+ fallback: async () => { fallbackCalls += 1; return indexed; },
+ deadlineMs: 1_000,
+ });
+ expect(await response.json()).toEqual([{ id: "ses_engine" }]);
+ expect(fallbackCalls).toBe(0);
+ });
+
+ test("an instance still booting past the deadline is answered from the index", async () => {
+ const events: string[] = [];
+ const response = await serveSessionListWithFallback({
+ primary: delayed(Response.json([{ id: "late" }]), 200),
+ fallback: async (cause) => { events.push(cause); return indexed; },
+ deadlineMs: 10,
+ onFallback: ({ cause, count }) => events.push(`${cause}:${count}`),
+ });
+ expect(await response.json()).toEqual(indexed);
+ expect(events).toEqual(["slow", "slow:1"]);
+ });
+
+ test("an empty engine answer for a folder whose sessions live under another spelling uses the index", async () => {
+ const response = await serveSessionListWithFallback({
+ primary: Promise.resolve(Response.json([])),
+ fallback: async (cause) => (cause === "empty" ? indexed : []),
+ deadlineMs: 1_000,
+ });
+ expect(await response.json()).toEqual(indexed);
+ });
+
+ test("a truly empty folder stays empty", async () => {
+ const response = await serveSessionListWithFallback({
+ primary: Promise.resolve(Response.json([])),
+ fallback: async () => [],
+ deadlineMs: 1_000,
+ });
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual([]);
+ });
+
+ test("a failed engine request is answered from the index, or rethrown when the index fails too", async () => {
+ const answered = await serveSessionListWithFallback({
+ primary: Promise.reject(new Error("fetch failed")),
+ fallback: async () => indexed,
+ deadlineMs: 1_000,
+ });
+ expect(await answered.json()).toEqual(indexed);
+
+ const failure = new Error("engine down");
+ await expect(serveSessionListWithFallback({
+ primary: Promise.reject(failure),
+ fallback: async () => { throw new Error("index down"); },
+ deadlineMs: 1_000,
+ })).rejects.toBe(failure);
+ });
+
+ test("a slow engine still wins when the index fails", async () => {
+ const response = await serveSessionListWithFallback({
+ primary: delayed(Response.json([{ id: "late" }]), 30),
+ fallback: async () => { throw new Error("index down"); },
+ deadlineMs: 5,
+ });
+ expect(await response.json()).toEqual([{ id: "late" }]);
+ });
+
+ test("an engine error status is replaced by the index answer", async () => {
+ const response = await serveSessionListWithFallback({
+ primary: Promise.resolve(Response.json({ error: "boot failed" }, { status: 500 })),
+ fallback: async () => indexed,
+ deadlineMs: 1_000,
+ });
+ expect(response.status).toBe(200);
+ expect(await response.json()).toEqual(indexed);
+ });
+});
+
+describe("EmptySessionDirectoryCache", () => {
+ test("remembers an empty folder for the TTL only", () => {
+ let now = 0;
+ const cache = new EmptySessionDirectoryCache(() => now, 1_000);
+ cache.remember("c:/w");
+ expect(cache.has("c:/w")).toBe(true);
+ now = 1_001;
+ expect(cache.has("c:/w")).toBe(false);
+ });
+});
diff --git a/apps/server/src/session-list-fallback.ts b/apps/server/src/session-list-fallback.ts
new file mode 100644
index 00000000..06fbb444
--- /dev/null
+++ b/apps/server/src/session-list-fallback.ts
@@ -0,0 +1,224 @@
+/**
+ * Keep a local workspace's session list answerable.
+ *
+ * The engine's GET /session is instance-scoped: the first request for a
+ * directory waits for that directory's instance to boot (git probes, config,
+ * plugin installs), which can take well over the app's request timeout on a
+ * cold Windows start. It also matches sessions by the stored project id AND
+ * the exact canonical directory string, so a renamed, re-cased or re-rooted
+ * folder, or a recreated .git, lists as empty although its sessions exist.
+ *
+ * The fallback answers from the engine's global session index instead
+ * (GET /experimental/session, served by the engine's already-booted default
+ * instance), filtered here by directory with platform-aware comparison.
+ */
+
+export const SESSION_LIST_FALLBACK_DEADLINE_ENV = "OMNIRUSH_SESSION_LIST_FALLBACK_MS";
+const DEFAULT_DEADLINE_MS = 6_000;
+const PAGE_SIZE = 500;
+const MAX_SCANNED = 5_000;
+const EMPTY_RESULT_TTL_MS = 60_000;
+
+export type SessionListFallbackCause = "slow" | "empty" | "failed";
+
+export function sessionListFallbackDeadlineMs(env: NodeJS.ProcessEnv = process.env): number {
+ const raw = Number(env[SESSION_LIST_FALLBACK_DEADLINE_ENV]);
+ return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DEADLINE_MS;
+}
+
+export function isSessionListRequest(method: string, normalizedPath: string): boolean {
+ return method.toUpperCase() === "GET" && normalizedPath === "/session";
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+/**
+ * Comparable form of a directory. Windows paths are case-insensitive and may
+ * arrive with backslashes, a `\\?\` prefix, a lower-case drive letter or a
+ * trailing separator; the engine stores forward slashes. macOS volumes are
+ * case-insensitive by default, so a case-only rename must still match there.
+ */
+export function directoryMatchKey(directory: string, platform: NodeJS.Platform = process.platform): string {
+ let value = directory.trim();
+ if (platform === "win32") {
+ value = value.replace(/\\/g, "/");
+ value = value.replace(/^\/\/\?\/UNC\//i, "//").replace(/^\/\/\?\//, "");
+ const unc = value.startsWith("//");
+ value = (unc ? "//" : "") + value.slice(unc ? 2 : 0).replace(/\/{2,}/g, "/");
+ } else {
+ value = value.replace(/\/{2,}/g, "/");
+ }
+ if (value.length > 1 && value.endsWith("/") && !/^[A-Za-z]:\/$/.test(value)) value = value.replace(/\/+$/, "");
+ return platform === "win32" || platform === "darwin" ? value.toLowerCase() : value;
+}
+
+export type SessionListFallbackQuery = {
+ roots?: boolean;
+ start?: number;
+ search?: string;
+ limit: number;
+};
+
+export function sessionListFallbackQuery(search: string): SessionListFallbackQuery {
+ const params = new URLSearchParams(search);
+ const limit = Number(params.get("limit"));
+ const start = Number(params.get("start"));
+ const searchText = params.get("search")?.trim();
+ return {
+ roots: params.get("roots") === "true" ? true : undefined,
+ start: Number.isFinite(start) && start > 0 ? start : undefined,
+ search: searchText ? searchText : undefined,
+ limit: Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : 100,
+ };
+}
+
+export type GlobalSessionPage = { items: unknown[]; nextCursor: string | null };
+
+/**
+ * Page through the engine's global session index (newest first) and keep the
+ * sessions whose directory matches one of `directories`. Archived sessions are
+ * included because the instance list includes them too.
+ */
+export async function listSessionsByDirectory(input: {
+ fetchPage: (params: URLSearchParams) => Promise<GlobalSessionPage>;
+ directories: string[];
+ query: SessionListFallbackQuery;
+ platform?: NodeJS.Platform;
+ maxScanned?: number;
+}): Promise<Record<string, unknown>[]> {
+ const platform = input.platform ?? process.platform;
+ const keys = new Set(input.directories.filter((entry) => entry.trim()).map((entry) => directoryMatchKey(entry, platform)));
+ const matches: Record<string, unknown>[] = [];
+ const maxScanned = input.maxScanned ?? MAX_SCANNED;
+ let scanned = 0;
+ let cursor: string | null = null;
+ while (matches.length < input.query.limit && scanned < maxScanned) {
+ const params = new URLSearchParams({ limit: String(PAGE_SIZE), archived: "true" });
+ if (input.query.roots) params.set("roots", "true");
+ if (input.query.start !== undefined) params.set("start", String(input.query.start));
+ if (input.query.search) params.set("search", input.query.search);
+ if (cursor) params.set("cursor", cursor);
+ const page = await input.fetchPage(params);
+ scanned += page.items.length;
+ for (const item of page.items) {
+ if (!isRecord(item) || typeof item.directory !== "string") continue;
+ if (!keys.has(directoryMatchKey(item.directory, platform))) continue;
+ const { project: _project, ...session } = item;
+ matches.push(session);
+ if (matches.length >= input.query.limit) break;
+ }
+ if (!page.nextCursor || page.items.length === 0) break;
+ cursor = page.nextCursor;
+ }
+ return matches;
+}
+
+type PrimaryOutcome = { ok: true; response: Response } | { ok: false; error: unknown };
+
+function jsonList(items: unknown[]): Response {
+ return new Response(JSON.stringify(items), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ });
+}
+
+/**
+ * Serve a session list: the engine's own answer when it is quick and
+ * non-empty, otherwise the directory index. A slow instance boot keeps
+ * running in the background, so the next list is served by the engine again.
+ */
+export async function serveSessionListWithFallback(input: {
+ primary: Promise<Response>;
+ /** `cause` says why the index is consulted; "empty" may answer from a negative cache. */
+ fallback: (cause: SessionListFallbackCause) => Promise<Record<string, unknown>[]>;
+ deadlineMs: number;
+ onFallback?: (event: { cause: SessionListFallbackCause; count: number }) => void;
+}): Promise<Response> {
+ const primary: Promise<PrimaryOutcome> = input.primary.then(
+ (response) => ({ ok: true, response }),
+ (error: unknown) => ({ ok: false, error }),
+ );
+ let fallbackRun: Promise<Record<string, unknown>[] | null> | null = null;
+ const fallback = (cause: SessionListFallbackCause) => {
+ fallbackRun ??= input.fallback(cause).catch(() => null);
+ return fallbackRun;
+ };
+ const served = (cause: SessionListFallbackCause, items: Record<string, unknown>[]) => {
+ input.onFallback?.({ cause, count: items.length });
+ return jsonList(items);
+ };
+
+ const finish = async (outcome: PrimaryOutcome): Promise<Response> => {
+ if (!outcome.ok) {
+ const items = await fallback("failed");
+ if (items) return served("failed", items);
+ throw outcome.error;
+ }
+ const { response } = outcome;
+ if (!response.ok) {
+ const items = await fallback("failed");
+ return items ? served("failed", items) : response;
+ }
+ const text = await response.text();
+ let parsed: unknown = null;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ parsed = null;
+ }
+ if (Array.isArray(parsed) && parsed.length === 0) {
+ const items = await fallback("empty");
+ if (items && items.length > 0) return served("empty", items);
+ }
+ return new Response(text, { status: response.status, statusText: response.statusText, headers: response.headers });
+ };
+
+ let timer: ReturnType<typeof setTimeout> | null = null;
+ const deadline = new Promise<null>((resolve) => {
+ timer = setTimeout(() => resolve(null), input.deadlineMs);
+ timer.unref?.();
+ });
+ const first = await Promise.race([primary, deadline]);
+ if (timer) clearTimeout(timer);
+ if (first) return finish(first);
+
+ type Winner = { source: "primary"; outcome: PrimaryOutcome } | { source: "fallback"; items: Record<string, unknown>[] };
+ const fromPrimary = primary.then((outcome): Winner => ({ source: "primary", outcome }));
+ const winner = await Promise.race<Winner>([
+ fromPrimary,
+ fallback("slow").then((items): Winner | Promise<Winner> => items ? { source: "fallback", items } : fromPrimary),
+ ]);
+ if (winner.source === "primary") return finish(winner.outcome);
+ // The engine keeps booting the instance; release its eventual answer.
+ void primary.then((outcome) => {
+ if (outcome.ok) void outcome.response.body?.cancel().catch(() => undefined);
+ });
+ return served("slow", winner.items);
+}
+
+/** Directories known to have no sessions anywhere, so repeated empty lists skip the index scan. */
+export class EmptySessionDirectoryCache {
+ private readonly entries = new Map<string, number>();
+
+ constructor(private readonly now: () => number = Date.now, private readonly ttlMs = EMPTY_RESULT_TTL_MS) {}
+
+ has(key: string): boolean {
+ const at = this.entries.get(key);
+ if (at === undefined) return false;
+ if (this.now() - at > this.ttlMs) {
+ this.entries.delete(key);
+ return false;
+ }
+ return true;
+ }
+
+ remember(key: string): void {
+ this.entries.set(key, this.now());
+ }
+
+ forget(key: string): void {
+ this.entries.delete(key);
+ }
+}
diff --git a/apps/server/src/task-recovery.test.ts b/apps/server/src/task-recovery.test.ts
index 1fbae52b..7f4712d7 100644
--- a/apps/server/src/task-recovery.test.ts
+++ b/apps/server/src/task-recovery.test.ts
@@ -92,6 +92,7 @@ async function fixture(engine: "v1" | "v2") {
return {
tasks, resumes, tick, config,
readHook(hook?: () => Promise<void>) { beforeRead = hook; },
+ engineLost() { recovery.requeueAfterEngineLoss(); },
policy(ready: boolean) { policyReady = ready; },
loseAck() { loseAcknowledgement = true; },
async send(id: string, stop = false, optOut = false) {
@@ -199,6 +200,28 @@ for (const engine of ["v1", "v2"] as const) {
expect(f.resumes).toHaveLength(1);
});
+ test(`${engine}: a running turn whose engine was replaced without a server restart resumes once`, async () => {
+ const f = await fixture(engine);
+ await f.send("ses_work"); await f.send("ses_done"); await f.tick();
+ // The watchdog replaced a dead engine: both runs stopped, one had finished.
+ f.tasks.get("ses_work")!.active = false;
+ f.tasks.get("ses_done")!.active = false;
+ f.tasks.get("ses_done")!.completed = true;
+ f.engineLost();
+ await f.tick(); await f.tick();
+ expect(f.resumes.map((resume) => resume.id)).toEqual(["ses_work"]);
+ await f.tick(); await f.tick();
+ expect(f.resumes).toHaveLength(1);
+ });
+
+ test(`${engine}: without an engine replacement an idle turn is never re-prompted`, async () => {
+ const f = await fixture(engine);
+ await f.send("ses_work"); await f.tick();
+ f.tasks.get("ses_work")!.active = false;
+ await f.tick(); await f.tick();
+ expect(f.resumes).toEqual([]);
+ });
+
test(`${engine}: shutdown checkpoints a just-admitted task before the first poll`, async () => {
const f = await fixture(engine);
await f.send("ses_work"); await f.restart(true); await f.tick();
diff --git a/apps/server/src/task-recovery.ts b/apps/server/src/task-recovery.ts
index f8cba641..d37111dd 100644
--- a/apps/server/src/task-recovery.ts
+++ b/apps/server/src/task-recovery.ts
@@ -204,6 +204,24 @@ export async function createTaskRecovery(
return {
tick,
owns: (request: Request) => ownedRequests.has(request),
+ /**
+ * The managed engine died under running turns and is being replaced
+ * without a server restart. Treat those turns exactly like the ones a
+ * restart interrupted: the startup path re-observes each one against the
+ * new engine and resumes it only if the same turn is still unfinished.
+ */
+ requeueAfterEngineLoss() {
+ if (stopped) return;
+ let changed = false;
+ for (const record of records.values()) {
+ if (record.phase !== "running" || record.userId === null) continue;
+ record.shutdown = true;
+ startup.add(key(record));
+ recovered.delete(key(record));
+ changed = true;
+ }
+ if (changed) void persist().catch(() => undefined);
+ },
async forward(workspace: WorkspaceInfo, engine: Engine, path: string, req: Request, send: () => Promise<Response>) {
if (workspace.workspaceType !== "local" || !["POST", "PUT", "PATCH", "DELETE"].includes(req.method)) return send();
const match = path.replace(/^\/opencode2\/api|^\/opencode/, "").match(/^\/session\/([^/]+)(.*)$/);
@@ -276,3 +294,4 @@ type TaskRecovery = Awaited<ReturnType<typeof createTaskRecovery>>;
const coordinators = new WeakMap<ServerConfig, TaskRecovery>();
export function setTaskRecovery(config: ServerConfig, recovery: TaskRecovery) { coordinators.set(config, recovery); }
export async function stopTaskRecovery(config: ServerConfig) { await coordinators.get(config)?.stop(); }
+export function requeueTaskRecoveryAfterEngineLoss(config: ServerConfig) { coordinators.get(config)?.requeueAfterEngineLoss(); }
diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.ts
index b16914ae..0219b1fe 100644
--- a/packages/types/src/desktop-ipc.ts
+++ b/packages/types/src/desktop-ipc.ts
@@ -85,6 +85,17 @@ export type DesktopIntegrationResult = {
error?: string;
};
+/** One built-in server or engine restart, as recorded in runtime-restarts.jsonl. */
+export type OmniRushRuntimeRestartRecord = {
+ at: string;
+ kind: "server" | "engine";
+ /** restarting, started, deferred, skipped, dropped, stopping, or restarted (engine watchdog). */
+ action: string;
+ reason: string;
+ source: string;
+ [key: string]: unknown;
+};
+
export type OmniRushServerInfo = {
running: boolean;
/**
@@ -113,6 +124,13 @@ export type OmniRushServerInfo = {
lastStdout: string | null;
lastStderr: string | null;
managedOpencodeExecution: OpencodeExecutionSnapshot | null;
+ /** Recent restarts, newest first (omnirushServerInfo only). */
+ restarts?: OmniRushRuntimeRestartRecord[];
+ /** An automatic restart that waits until no session is running. */
+ pendingRestart?: { action: string; reason: string; source: string; requestedAt: string } | null;
+ /** Set on an omnirushServerRestart answer that kept the running server. */
+ restartDeferred?: boolean;
+ restartSkipped?: boolean;
};
/** Outcome of the remote device-session revocation attempted during sign-out. */