A desktop coding agent with free access to frontier models.
TypeScript
fix(runtime): no unattended server restarts under live runs; session lists always answer
ogsamrat8fdd5f1c1fe4
diff --git a/apps/app/src/app/lib/diagnostics-bundle.ts b/apps/app/src/app/lib/diagnostics-bundle.tsindex e19e8a29..41184cce 100644--- a/apps/app/src/app/lib/diagnostics-bundle.ts+++ b/apps/app/src/app/lib/diagnostics-bundle.ts@@ -112,6 +112,9 @@ function pickHostInfo(info: OmniRushServerInfo | null) {lanUrl: info.lanUrl ?? null,lastStdout: info.lastStdout ?? null,lastStderr: info.lastStderr ?? null,+ // Built-in server/engine restarts with reason, trigger and time (newest first).+ restarts: info.restarts ?? [],+ pendingRestart: info.pendingRestart ?? null,};}diff --git a/apps/app/src/react-app/domains/connections/omnirush-server-store.ts b/apps/app/src/react-app/domains/connections/omnirush-server-store.tsindex 0382867a..5bc8ca1c 100644--- a/apps/app/src/react-app/domains/connections/omnirush-server-store.ts+++ b/apps/app/src/react-app/domains/connections/omnirush-server-store.ts@@ -807,7 +807,11 @@ export function createOmniRushServerStore(options: CreateOmniRushServerStoreOpti}try {+ // Not a user action: the desktop keeps a server that still answers it+ // and only replaces one that is really gone.hostInfo = await omnirushServerRestart({+ reason: "renderer_health_check_failed",+ source: "omnirush-server-store",remoteAccessEnabled: state.omnirushServerSettings.remoteAccessEnabled === true,}) as OmniRushServerInfo;mutateState((current) => ({ ...current, omnirushServerHostInfo: hostInfo }));diff --git a/apps/app/src/react-app/domains/connections/provider-auth/store.ts b/apps/app/src/react-app/domains/connections/provider-auth/store.tsindex 7ff6b1e1..4fd50344 100644--- a/apps/app/src/react-app/domains/connections/provider-auth/store.ts+++ b/apps/app/src/react-app/domains/connections/provider-auth/store.ts@@ -1481,7 +1481,7 @@ export function createProviderAuthStore(options: CreateProviderAuthStoreOptions)if (!unreachable || !isDesktopRuntime()) {throw error;}- await engineRestart({});+ await engineRestart({ reason: "provider_reload_engine_unreachable", source: "provider-auth" });}reloaded = true;}@@ -2379,7 +2379,11 @@ export function createProviderAuthStore(options: CreateProviderAuthStoreOptions)// Explicit desktop sign-out must replace that process so an// account-scoped provider cannot remain connected in the UI.if (isDesktopRuntime()) {- await engineRestart({}).catch(() => undefined);+ await engineRestart({+ reason: "account_signed_out",+ source: "provider-auth",+ userInitiated: true,+ }).catch(() => undefined);}})();}diff --git a/apps/app/src/react-app/domains/settings/state/debug-view-model.ts b/apps/app/src/react-app/domains/settings/state/debug-view-model.tsindex 2cec62e3..94d7bb61 100644--- a/apps/app/src/react-app/domains/settings/state/debug-view-model.ts+++ b/apps/app/src/react-app/domains/settings/state/debug-view-model.ts@@ -736,6 +736,8 @@ export function useDebugViewModel(options: UseDebugViewModelOptions) {}const info = await engineStartCmd(workspacePath, {+ reason: "debug_view_restart",+ userInitiated: true,runtime: "direct",workspacePaths,opencodeEnableExa: readOpencodeEnableExa(),@@ -807,6 +809,8 @@ export function useDebugViewModel(options: UseDebugViewModelOptions) {setServiceRestartError(null);try {await omnirushServerRestartCmd({+ reason: "debug_view_restart",+ userInitiated: true,remoteAccessEnabled: omnirushServerSnapshot.omnirushServerSettings.remoteAccessEnabled === true,});setOmniRushServiceStatus({diff --git a/apps/app/src/react-app/domains/workspace/remote-access-restart.ts b/apps/app/src/react-app/domains/workspace/remote-access-restart.tsindex 92aec605..c715a1f2 100644--- a/apps/app/src/react-app/domains/workspace/remote-access-restart.ts+++ b/apps/app/src/react-app/domains/workspace/remote-access-restart.ts@@ -36,7 +36,11 @@ export function useRemoteAccessRestart(options: UseRemoteAccessRestartOptions) {options.onSettingsChanged();try {- const info = await omnirushServerRestart({ remoteAccessEnabled: enabled }) as OmniRushServerInfo;+ const info = await omnirushServerRestart({+ reason: "remote_access_toggled",+ userInitiated: true,+ remoteAccessEnabled: enabled,+ }) as OmniRushServerInfo;writeOmniRushServerSettings({urlOverride: info.baseUrl?.trim() || undefined,token:diff --git a/apps/app/src/react-app/shell/desktop-runtime-boot.ts b/apps/app/src/react-app/shell/desktop-runtime-boot.tsindex 007558c5..e9b328ec 100644--- a/apps/app/src/react-app/shell/desktop-runtime-boot.ts+++ b/apps/app/src/react-app/shell/desktop-runtime-boot.ts@@ -118,7 +118,11 @@ export function useDesktopRuntimeBoot() {const startServerWithoutDesktopWorkspace = async () => {setPhase("starting-engine", "Starting omnirush.ai server");- const serverInfo = await omnirushServerRestart({ remoteAccessEnabled: preferredRemoteAccess }).catch((error) => {+ const serverInfo = await omnirushServerRestart({+ reason: "renderer_boot_without_local_workspace",+ source: "desktop-runtime-boot",+ remoteAccessEnabled: preferredRemoteAccess,+ }).catch((error) => {console.warn("[desktop-boot] omnirushServerRestart failed:", error);return null;});@@ -181,7 +185,11 @@ export function useDesktopRuntimeBoot() {}let serverInfo = boot.omnirushServer;if (preferredRemoteAccess && serverInfo?.remoteAccessEnabled !== true) {- const restarted = await omnirushServerRestart({ remoteAccessEnabled: true }).catch((error) => {+ const restarted = await omnirushServerRestart({+ reason: "renderer_boot_remote_access",+ source: "desktop-runtime-boot",+ remoteAccessEnabled: true,+ }).catch((error) => {console.warn("[desktop-boot] omnirushServerRestart failed:", error);return null;});diff --git a/apps/app/src/react-app/shell/engine-reload-escalation.ts b/apps/app/src/react-app/shell/engine-reload-escalation.tsindex a2d02754..0e495d43 100644--- a/apps/app/src/react-app/shell/engine-reload-escalation.ts+++ b/apps/app/src/react-app/shell/engine-reload-escalation.ts@@ -20,10 +20,6 @@ export function canRestartDesktopForReloadError(error: unknown) {);}-function isEngineUnreachableError(error: unknown) {- return error instanceof OmniRushServerError && error.code === "opencode_engine_unreachable";-}-/*** Aborted or timed-out fetches are client→server transport blips (including* AbortSignal-cancelable fetch cancellations), not proof the engine is gone.@@ -56,15 +52,18 @@ export type ReloadEngineFallbackOptions = {/*** Reload the workspace engine; escalate to a full desktop engine restart only* when the reload keeps failing with a restartable server-reported code.- * `opencode_engine_unreachable` gets one delayed retry first — if the engine- * answers the second attempt, no session is disturbed.+ * Both codes get one delayed retry first — an engine that is still starting+ * or recovering answers the second attempt, and no session is disturbed.+ * The desktop then defers the restart while any session is still running+ * (it reports `restartDeferred`), so this never ends live runs by itself.*/export async function reloadEngineWithDesktopFallback(client: Pick<OmniRushServerClient, "reloadEngine">,workspaceId: string,options?: ReloadEngineFallbackOptions,): Promise<ReloadEngineFallbackResult> {- const restartEngine = options?.restartEngine ?? (() => engineRestart({}));+ const restartEngine = options?.restartEngine+ ?? (() => engineRestart({ reason: "engine_reload_failed", source: "engine-reload" }));const isDesktop = options?.isDesktop ?? isDesktopRuntime;try {await client.reloadEngine(workspaceId);@@ -74,17 +73,17 @@ export async function reloadEngineWithDesktopFallback(if (!canRestartDesktopForReloadError(error) || !isDesktop()) {throw error;}- if (isEngineUnreachableError(error)) {- await delay(options?.retryDelayMs ?? UNREACHABLE_RETRY_DELAY_MS);- try {- await client.reloadEngine(workspaceId);- return { restartedEngine: false };- } catch (retryError) {- if (isTransientTransportError(retryError)) throw retryError;- if (!canRestartDesktopForReloadError(retryError)) throw retryError;- }+ await delay(options?.retryDelayMs ?? UNREACHABLE_RETRY_DELAY_MS);+ try {+ await client.reloadEngine(workspaceId);+ return { restartedEngine: false };+ } catch (retryError) {+ if (isTransientTransportError(retryError)) throw retryError;+ if (!canRestartDesktopForReloadError(retryError)) throw retryError;}- await restartEngine();- return { restartedEngine: true };+ const restart = await restartEngine();+ const deferred = typeof restart === "object" && restart !== null && "restartDeferred" in restart+ && restart.restartDeferred === true;+ return { restartedEngine: !deferred };}}diff --git a/apps/app/src/react-app/shell/settings-route.tsx b/apps/app/src/react-app/shell/settings-route.tsxindex 70c9d3d9..95b1edaf 100644--- a/apps/app/src/react-app/shell/settings-route.tsx+++ b/apps/app/src/react-app/shell/settings-route.tsx@@ -713,6 +713,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {if (!isDesktopRuntime()) return false;try {await omnirushServerRestart({+ reason: "share_remote_access_changed",+ userInitiated: true,remoteAccessEnabled:readOmniRushServerSettings().remoteAccessEnabled === true,});@@ -1892,6 +1894,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {if (!isDesktopRuntime()) return false;try {await omnirushServerRestart({+ reason: "extension_settings_restart",+ userInitiated: true,remoteAccessEnabled:readOmniRushServerSettings().remoteAccessEnabled === true,});@@ -2095,6 +2099,8 @@ function SettingsRouteContent(props: SettingsSurfaceProps = {}) {workspacePaths.unshift(selectedWorkspaceRoot);}await engineStart(selectedWorkspaceRoot, {+ reason: "environment_applied",+ userInitiated: true,preferSidecar: true,runtime: "direct",workspacePaths,diff --git a/apps/app/tests/diagnostics-bundle.test.ts b/apps/app/tests/diagnostics-bundle.test.tsindex cef534e0..60b151eb 100644--- a/apps/app/tests/diagnostics-bundle.test.ts+++ b/apps/app/tests/diagnostics-bundle.test.ts@@ -102,6 +102,43 @@ describe("diagnostics bundle", () => {expect(json).not.toContain("opencodeUsername");});+ test("reports built-in server restarts with reason, trigger and time", () => {+ const input = baseInputs();+ input.desktopRuntime = true;+ input.hostInfo = {+ running: true,+ generation: 3,+ remoteAccessEnabled: false,+ host: "127.0.0.1",+ port: 4096,+ baseUrl: "http://127.0.0.1:4096",+ connectUrl: null,+ mdnsUrl: null,+ lanUrl: null,+ clientToken: null,+ ownerToken: null,+ hostToken: null,+ managedOpencodeBinPath: null,+ managedOpencodeBinSource: null,+ logFilePath: null,+ pid: null,+ lastStdout: null,+ lastStderr: null,+ managedOpencodeExecution: null,+ restarts: [+ { at: "2026-09-24T07:11:02.000Z", kind: "server", action: "deferred", reason: "engine_reload_failed", source: "engine-reload", busySessions: 3 },+ { at: "2026-09-24T02:57:40.000Z", kind: "engine", action: "restarted", reason: "engine_unreachable:process_exited", source: "engine-pool-watchdog" },+ ],+ pendingRestart: { action: "engine-restart", reason: "engine_reload_failed", source: "engine-reload", requestedAt: "2026-09-24T07:11:02.000Z" },+ };++ const parsed = JSON.parse(composeDiagnosticsBundleJson(input));++ expect(parsed.omnirushServer.host.restarts).toHaveLength(2);+ expect(parsed.omnirushServer.host.restarts[0]).toMatchObject({ action: "deferred", reason: "engine_reload_failed", busySessions: 3 });+ expect(parsed.omnirushServer.host.pendingRestart.reason).toBe("engine_reload_failed");+ });+test("produces valid JSON without desktop info", () => {const json = composeDiagnosticsBundleJson(baseInputs());const parsed = JSON.parse(json);diff --git a/apps/app/tests/engine-reload-escalation.test.ts b/apps/app/tests/engine-reload-escalation.test.tsindex bcfc8a63..3e9f497a 100644--- a/apps/app/tests/engine-reload-escalation.test.ts+++ b/apps/app/tests/engine-reload-escalation.test.ts@@ -69,11 +69,32 @@ describe("reloadEngineWithDesktopFallback", () => {expect(harness.restartCalls).toBe(1);});- test("an unconfigured engine restarts immediately — there is nothing to re-probe", async () => {- const harness = makeHarness([unconfiguredError()]);+ test("an engine that is still starting (unconfigured) recovers on the retry without a restart", async () => {+ const harness = makeHarness([unconfiguredError(), null]);+ const result = await reloadEngineWithDesktopFallback(harness.client, "ws", harness.options);+ expect(result.restartedEngine).toBe(false);+ expect(harness.reloadCalls).toBe(2);+ expect(harness.restartCalls).toBe(0);+ });++ test("a persistently unconfigured engine escalates to a restart after one retry", async () => {+ const harness = makeHarness([unconfiguredError(), unconfiguredError()]);const result = await reloadEngineWithDesktopFallback(harness.client, "ws", harness.options);expect(result.restartedEngine).toBe(true);- expect(harness.reloadCalls).toBe(1);+ expect(harness.reloadCalls).toBe(2);+ expect(harness.restartCalls).toBe(1);+ });++ test("a restart the desktop deferred because sessions are running is not reported as a restart", async () => {+ const harness = makeHarness([unreachableError(), unreachableError()]);+ const result = await reloadEngineWithDesktopFallback(harness.client, "ws", {+ ...harness.options,+ restartEngine: async () => {+ harness.restartCalls += 1;+ return { running: true, restartDeferred: true, busySessions: 3 };+ },+ });+ expect(result.restartedEngine).toBe(false);expect(harness.restartCalls).toBe(1);});diff --git a/apps/desktop/electron/main.mjs b/apps/desktop/electron/main.mjsindex 5b53a9f6..568e834e 100644--- a/apps/desktop/electron/main.mjs+++ b/apps/desktop/electron/main.mjs@@ -1221,7 +1221,7 @@ async function persistConnectLinkClaims(claims) {await uiControlServer.start().catch((error) => {console.warn("[ui-control] failed to start", error);});- await runtimeManager.prepareFreshRuntime();+ await runtimeManager.prepareFreshRuntime({ reason: "desktop_activated", source: "connect-link" });}return config;}@@ -1899,11 +1899,11 @@ const desktopCommandHandlers = {},"engineStart": async (event, ...args) => {const projectDir = String(args[0] ?? "").trim();- const options = args[1] ?? {};+ const options = { source: "renderer", ...(args[1] ?? {}) };return runtimeManager.engineStart(projectDir, options);},"prepareFreshRuntime": async (event, ...args) => {- return runtimeManager.prepareFreshRuntime();+ return runtimeManager.prepareFreshRuntime({ source: "renderer" });},"runtimeBootstrap": async (event, ...args) => {return ensureRuntimeBootstrap();@@ -1912,10 +1912,10 @@ const desktopCommandHandlers = {return runtimeManager.runtimeStatus();},"engineStop": async (event, ...args) => {- return runtimeManager.engineStop();+ return runtimeManager.engineStop({ source: "renderer" });},"engineRestart": async (event, ...args) => {- return runtimeManager.engineRestart(args[0] ?? {});+ return runtimeManager.engineRestart({ source: "renderer", ...(args[0] ?? {}) });},"engineInfo": async (event, ...args) => {return runtimeManager.engineInfo();@@ -2031,7 +2031,7 @@ const desktopCommandHandlers = {await uiControlServer.start().catch((error) => {console.warn("[ui-control] failed to start", error);});- await runtimeManager.prepareFreshRuntime();+ await runtimeManager.prepareFreshRuntime({ reason: "desktop_activated", source: "bootstrap-config" });}return next;},@@ -2110,19 +2110,19 @@ const desktopCommandHandlers = {deviceName: String(input.deviceName ?? "").trim() || `${app.getName()} on ${os.hostname()}`,openVerification: (url) => shell.openExternal(url),});- await runtimeManager.omnirushServerRestart({});+ await runtimeManager.omnirushServerRestart({ reason: "account_connected", source: "account", userInitiated: true });return result;},"omnirushAccountSignOut": async (event, ...args) => {const result = await omnirushAccountStore.clear();- await runtimeManager.omnirushServerRestart({});+ await runtimeManager.omnirushServerRestart({ reason: "account_signed_out", source: "account", userInitiated: true });return { connected: false, remoteRevoked: result.remoteRevoked, reason: result.reason };},"automationRunnerConfigure": async (event, ...args) => {return desktopAutomationRunner.configure(args[0] ?? null);},"omnirushServerRestart": async (event, ...args) => {- return runtimeManager.omnirushServerRestart(args[0] ?? {});+ return runtimeManager.omnirushServerRestart({ source: "renderer", ...(args[0] ?? {}) });},"pickDirectory": async (event, ...args) => {const options = args[0] ?? {};@@ -2949,7 +2949,7 @@ or use: pnpm dev:worktree`);}applicationMenu.install();if (!desktopActivationRequired(DESKTOP_DISTRIBUTION, bootstrapConfig)) {- await runtimeManager.prepareFreshRuntime().catch(() => undefined);+ await runtimeManager.prepareFreshRuntime({ reason: "app_launch", source: "boot" }).catch(() => undefined);}// Use Tauri's existing workspace state file as canonical so rollback anddiff --git a/apps/desktop/electron/nuke.mjs b/apps/desktop/electron/nuke.mjsindex a5dff717..7e334c29 100644--- a/apps/desktop/electron/nuke.mjs+++ b/apps/desktop/electron/nuke.mjs@@ -736,7 +736,7 @@ async function bestEffort(errors, label, task, timeoutMs) {async function quiesceForNuke({ runtimeManager, uiControlServer, removeWindowsBrandShortcut }, errors, options = {}) {await bestEffort(errors, "ui-control-server", () => uiControlServer.stop(), 3000);await bestEffort(errors, "runtime-dispose", () => runtimeManager.dispose(), 12_000);- await bestEffort(errors, "packaged-sidecar-reaper", () => runtimeManager.prepareFreshRuntime(), 16_000);+ await bestEffort(errors, "packaged-sidecar-reaper", () => runtimeManager.prepareFreshRuntime({ reason: "reset_app_data", source: "nuke" }), 16_000);// Container cleanup matches on name prefix across the whole Docker host, so it// cannot tell this profile's containers from another profile's. Only the// default profile may run it; isolated profiles leave containers alone ratherdiff --git a/apps/desktop/electron/runtime-restart-log.mjs b/apps/desktop/electron/runtime-restart-log.mjsnew file mode 100644index 00000000..1c613e41--- /dev/null+++ b/apps/desktop/electron/runtime-restart-log.mjs@@ -0,0 +1,135 @@+import { appendFileSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";+import { readFile } from "node:fs/promises";+import path from "node:path";++/**+ * Restart record and restart policy for the built-in server and its engine.+ *+ * Every stop of a running server ends every live run, so each one is written+ * with its reason and trigger to omnirush-server.log and to a small JSON-lines+ * history next to it (runtime-restarts.jsonl, also appended by the server for+ * engine recoveries it performs itself). omnirushServerInfo returns the tail+ * of that history for the diagnostics bundle.+ */++export const RUNTIME_RESTART_LOG_FILE_NAME = "runtime-restarts.jsonl";+const MAX_HISTORY_BYTES = 256 * 1024;+const KEEP_AFTER_TRIM = 100;+/** A deferred restart whose busy check stays unreadable this long runs anyway: the engine is gone. */+export const DEFERRED_RESTART_UNKNOWN_MAX_MS = 10 * 60_000;++export function runtimeRestartLogPath(serverLogFile) {+ return path.join(path.dirname(serverLogFile), RUNTIME_RESTART_LOG_FILE_NAME);+}++function trimHistory(historyFile) {+ try {+ if (statSync(historyFile).size <= MAX_HISTORY_BYTES) return;+ const lines = readFileSync(historyFile, "utf8").split("\n").filter(Boolean);+ writeFileSync(historyFile, `${lines.slice(-KEEP_AFTER_TRIM).join("\n")}\n`, "utf8");+ } catch {+ // Diagnostics only.+ }+}++/**+ * Append one record to the history and one structured line to the server log.+ * Best effort: diagnostics must never block or fail a restart.+ *+ * @param {{ serverLogFile: string | null, record: Record<string, unknown> & { kind: string, action: string,+ * reason: string, source: string }, now?: () => Date }} input+ */+export function appendRuntimeRestartRecord({ serverLogFile, record, now = () => new Date() }) {+ if (!serverLogFile) return null;+ const at = now();+ const entry = { at: at.toISOString(), ...record };+ const historyFile = runtimeRestartLogPath(serverLogFile);+ try {+ mkdirSync(path.dirname(historyFile), { recursive: true });+ appendFileSync(historyFile, `${JSON.stringify(entry)}\n`, "utf8");+ trimHistory(historyFile);+ } catch {+ // Diagnostics only.+ }+ const attributes = {};+ for (const [key, value] of Object.entries(record)) {+ if (value !== undefined) attributes[`runtime.restart.${key.replace(/[A-Z]/g, (letter) => `_${letter.toLowerCase()}`)}`] = value;+ }+ const line = {+ timeUnixNano: `${BigInt(at.getTime()) * 1_000_000n}`,+ severityText: record.action === "deferred" || record.action === "skipped" ? "INFO" : "WARN",+ body: `Built-in ${record.kind === "engine" ? "engine" : "server"} ${record.action}: ${record.reason} (${record.source})`,+ attributes,+ resource: { "service.name": "omnirush-desktop-runtime" },+ };+ try {+ mkdirSync(path.dirname(serverLogFile), { recursive: true });+ appendFileSync(serverLogFile, `${JSON.stringify(line)}\n`, "utf8");+ } catch {+ // Diagnostics only.+ }+ return entry;+}++/** Newest first. Unreadable or partial lines are skipped. */+export async function readRuntimeRestartRecords(serverLogFile, limit = 20) {+ if (!serverLogFile) return [];+ let text = "";+ try {+ text = await readFile(runtimeRestartLogPath(serverLogFile), "utf8");+ } catch {+ return [];+ }+ const records = [];+ const lines = text.split("\n").filter(Boolean);+ for (let index = lines.length - 1; index >= 0 && records.length < limit; index -= 1) {+ try {+ const value = JSON.parse(lines[index]);+ if (value && typeof value === "object" && typeof value.at === "string") records.push(value);+ } catch {+ // Skip a torn line.+ }+ }+ return records;+}++/**+ * Whether a lifecycle request may stop the running server now.+ *+ * action: "engine-restart" | "server-restart" | "engine-start"+ * busy: { sessions: number, unknown: number } — unknown probes count as busy+ *+ * Returns { verdict: "proceed" | "defer" | "skip", why }.+ *+ * @param {{ action: string, userInitiated?: boolean, serverRunning: boolean, serverHealthy: boolean,+ * busy?: { sessions: number, unknown: number }, sameSettings?: boolean }} input+ * @returns {{ verdict: "proceed" | "defer" | "skip", why: string }}+ */+export function decideRuntimeRestart({ action, userInitiated, serverRunning, serverHealthy, busy, sameSettings }) {+ if (!serverRunning) return { verdict: "proceed", why: "server_not_running" };+ if (userInitiated === true) return { verdict: "proceed", why: "user_initiated" };+ if (!serverHealthy) return { verdict: "proceed", why: "server_unresponsive" };+ // A healthy server already runs with what was asked for: a restart would+ // only kill live runs. This covers renderer health checks that timed out+ // while the server was merely busy, and renderer reloads re-running boot.+ if (action === "engine-start") return { verdict: "skip", why: "server_healthy" };+ if (action === "server-restart" && sameSettings === true) return { verdict: "skip", why: "server_healthy" };+ if ((busy?.sessions ?? 0) > 0 || (busy?.unknown ?? 0) > 0) return { verdict: "defer", why: "sessions_busy" };+ return { verdict: "proceed", why: "idle" };+}++/**+ * Whether a deferred restart runs now: "run", "wait" or "drop" (the server+ * it was meant for is gone; whoever starts the next one applies everything).+ *+ * @param {{ serverRunning: boolean, serverHealthy: boolean,+ * busy?: { sessions: number, unknown: number }, deferredForMs: number }} input+ * @returns {"run" | "wait" | "drop"}+ */+export function decideDeferredRestart({ serverRunning, serverHealthy, busy, deferredForMs }) {+ if (!serverRunning) return "drop";+ if (!serverHealthy) return "run";+ if ((busy?.sessions ?? 0) > 0) return "wait";+ if ((busy?.unknown ?? 0) > 0 && deferredForMs < DEFERRED_RESTART_UNKNOWN_MAX_MS) return "wait";+ return "run";+}diff --git a/apps/desktop/electron/runtime.mjs b/apps/desktop/electron/runtime.mjsindex c2e71215..cda314ef 100644--- a/apps/desktop/electron/runtime.mjs+++ b/apps/desktop/electron/runtime.mjs@@ -8,6 +8,12 @@ import path from "node:path";import tls from "node:tls";import { fileURLToPath } from "node:url";import { pathToFileURL } from "node:url";+import {+ appendRuntimeRestartRecord,+ decideDeferredRestart,+ decideRuntimeRestart,+ readRuntimeRestartRecords,+} from "./runtime-restart-log.mjs";import {desktopBootstrapPath,normalizeWorkspaceRootPath,@@ -1888,6 +1894,187 @@ export function createRuntimeManager({// In-process server handle. Kept alive across restarts so we can stop it.let inProcessServer = null;+ // ── Restart guard ───────────────────────────────────────────────────────+ // Stopping the running server ends every live run (the collector closes+ // each session and the engine is killed). Automatic triggers therefore never+ // stop a healthy server that still has running sessions: the request waits+ // until the engine is idle. Every stop is recorded with its reason.+ const DEFERRED_RESTART_POLL_MS = 15_000;+ let pendingRestart = null;+ let pendingRestartTimer = null;+ let pendingRestartChecking = false;+ let nextStartReason = null;++ function restartLogFile() {+ return omnirushServerState.logFilePath ?? resolveOmniRushServerLogFile(userDataDir);+ }++ function recordRestart(record) {+ appendRuntimeRestartRecord({+ serverLogFile: restartLogFile(),+ record: { kind: "server", generation: omnirushServerState.generation ?? null, ...record },+ });+ }++ function restartRequest(options, fallbackSource) {+ const reason = typeof options?.reason === "string" && options.reason.trim() ? options.reason.trim() : "unspecified";+ const source = typeof options?.source === "string" && options.source.trim() ? options.source.trim() : fallbackSource;+ return { reason, source, userInitiated: options?.userInitiated === true };+ }++ async function probeRunningServer() {+ const baseUrl = omnirushServerState.baseUrl;+ if (!inProcessServer || !baseUrl) return { running: false, healthy: false };+ for (let attempt = 0; attempt < 2; attempt += 1) {+ try {+ await fetchJson(`${baseUrl.replace(/\/+$/, "")}/health`, {}, 5000);+ return { running: true, healthy: true };+ } catch {+ // One retry: a single slow answer is not a dead server.+ }+ }+ return { running: true, healthy: false };+ }++ async function readBusySessions() {+ const server = inProcessServer;+ if (typeof server?.busySessions !== "function") return { sessions: 0, unknown: 1 };+ let timer = null;+ try {+ const result = await Promise.race([+ server.busySessions(),+ new Promise((_, reject) => {+ timer = setTimeout(() => reject(new Error("busy session probe timed out")), 8000);+ }),+ ]);+ return { sessions: result.sessions.length, unknown: result.unknown };+ } catch {+ return { sessions: 0, unknown: 1 };+ } finally {+ if (timer) clearTimeout(timer);+ }+ }++ /**+ * Returns { verdict, why, reason, source, userInitiated, busy }. Callers stop+ * the server only on "proceed"; "defer" queues the request until idle and+ * "skip" keeps the healthy server.+ */+ async function guardRestart(action, options, fallbackSource, extra = {}) {+ const request = restartRequest(options, fallbackSource);+ const probe = await probeRunningServer();+ const busy = probe.running && probe.healthy ? await readBusySessions() : { sessions: 0, unknown: 0 };+ const decision = decideRuntimeRestart({+ action,+ userInitiated: request.userInitiated,+ serverRunning: probe.running,+ serverHealthy: probe.healthy,+ busy,+ sameSettings: extra.sameSettings,+ });+ const result = { ...decision, ...request, busy };+ if (decision.verdict === "proceed") {+ if (probe.running) {+ recordRestart({+ action: "restarting",+ reason: request.reason,+ source: request.source,+ trigger: action,+ why: decision.why,+ busySessions: busy.sessions,+ busyUnknown: busy.unknown,+ });+ }+ if (request.userInitiated) clearPendingRestart();+ nextStartReason = request.reason;+ } else {+ recordRestart({+ action: decision.verdict === "defer" ? "deferred" : "skipped",+ reason: request.reason,+ source: request.source,+ trigger: action,+ why: decision.why,+ busySessions: busy.sessions,+ busyUnknown: busy.unknown,+ });+ if (decision.verdict === "defer") queuePendingRestart(action, options, request);+ }+ return result;+ }++ function clearPendingRestart() {+ pendingRestart = null;+ if (pendingRestartTimer) clearInterval(pendingRestartTimer);+ pendingRestartTimer = null;+ }++ function queuePendingRestart(action, options, request) {+ pendingRestart = {+ action,+ options: { ...(options ?? {}) },+ reason: request.reason,+ source: request.source,+ requestedAt: pendingRestart?.requestedAt ?? Date.now(),+ };+ if (pendingRestartTimer) return;+ pendingRestartTimer = setInterval(() => void runPendingRestartIfIdle(), DEFERRED_RESTART_POLL_MS);+ pendingRestartTimer.unref?.();+ }++ async function runPendingRestartIfIdle() {+ const pending = pendingRestart;+ if (!pending || pendingRestartChecking) return;+ pendingRestartChecking = true;+ try {+ const probe = await probeRunningServer();+ const busy = probe.running && probe.healthy ? await readBusySessions() : { sessions: 0, unknown: 0 };+ const verdict = decideDeferredRestart({+ serverRunning: probe.running,+ serverHealthy: probe.healthy,+ busy,+ deferredForMs: Date.now() - pending.requestedAt,+ });+ if (verdict === "wait" || pendingRestart !== pending) return;+ clearPendingRestart();+ if (verdict === "drop") {+ recordRestart({ action: "dropped", reason: pending.reason, source: pending.source, trigger: pending.action });+ return;+ }+ const options = {+ ...pending.options,+ reason: `${pending.reason} (deferred until idle)`,+ source: pending.source,+ deferredRun: true,+ };+ await (pending.action === "engine-restart"+ ? withRuntimeLifecycle(() => engineRestart(options))+ : withRuntimeLifecycle(() => omnirushServerRestart(options))+ ).catch((error) => {+ appendOutput(omnirushServerState, "lastStderr", `Deferred restart failed: ${error instanceof Error ? error.message : String(error)}\n`);+ });+ } finally {+ pendingRestartChecking = false;+ }+ }++ function pendingRestartSummary() {+ return pendingRestart+ ? {+ action: pendingRestart.action,+ reason: pendingRestart.reason,+ source: pendingRestart.source,+ requestedAt: new Date(pendingRestart.requestedAt).toISOString(),+ }+ : null;+ }++ /** Record a stop that is not a restart request (quit, stop, fresh runtime). */+ function recordStop(options, fallbackSource) {+ if (!inProcessServer) return;+ const request = restartRequest(options, fallbackSource);+ recordRestart({ action: "stopping", reason: request.reason, source: request.source });+ }+async function startOmniRushServer(options) {// The inner start stops any previous runtime before mutating state, so a// throw below always happens with nothing left running.@@ -2044,6 +2231,9 @@ export function createRuntimeManager({omnirushServerState.baseUrl = baseUrl;omnirushServerState.clientToken = tokens.clientToken;omnirushServerState.hostToken = tokens.hostToken;+ recordRestart({ action: "started", reason: nextStartReason ?? "startup", source: "runtime" });+ nextStartReason = null;+ clearPendingRestart();const connectUrls = options.remoteAccessEnabled ? buildConnectUrls(boundPort) : { connectUrl: null, mdnsUrl: null, lanUrl: null };omnirushServerState.connectUrl = connectUrls.connectUrl;@@ -2098,6 +2288,7 @@ export function createRuntimeManager({async function stopAllRuntimeChildren() {// Stop the in-process server (and its managed OpenCode child) if running.+ clearPendingRestart();if (inProcessServer) {try { await inProcessServer.stop(); } catch { /* ignore */ }inProcessServer = null;@@ -2109,7 +2300,12 @@ export function createRuntimeManager({Object.assign(omnirushServerState, createOmniRushServerState());}- async function prepareFreshRuntime() {+ async function prepareFreshRuntime(options = {}) {+ recordStop(options, "prepare-fresh-runtime");+ await resetRuntime();+ }++ async function resetRuntime() {lifecycleState = "cleaning";await stopAllRuntimeChildren();await cleanupPackagedSidecars();@@ -2169,7 +2365,7 @@ export function createRuntimeManager({// resolveOmniRushServerReuse also spans workspace switches: requesting a// different projectDir retargets the running runtime instead of killing// the process and every in-flight run with it.- const reuseDecision = resolveOmniRushServerReuse({+ let reuseDecision = resolveOmniRushServerReuse({forceRestart: options.forceRestart,inProcess: omnirushServerState.inProcess,lifecycleState,@@ -2179,6 +2375,20 @@ export function createRuntimeManager({requestedProjectDir: safeProjectDir,platform: workspacePlatform,});+ if (!reuseDecision.reuse && options.forceRestart !== true && inProcessServer) {+ // Not a restart request, yet it would replace the running server+ // (remote-access mismatch, or a lifecycle state left behind by an+ // earlier failure). Only a user action may do that to a healthy one.+ const guard = await guardRestart("engine-start", options, "engine-start");+ if (guard.verdict === "skip") {+ lifecycleState = "healthy";+ reuseDecision = {+ reuse: true,+ retarget: normalizeWorkspaceKey(engineState.projectDir, workspacePlatform)+ !== normalizeWorkspaceKey(safeProjectDir, workspacePlatform),+ };+ }+ }if (reuseDecision.reuse) {const existing = snapshotOmniRushServerState(omnirushServerState);if (existing.running && existing.baseUrl && (existing.ownerToken || existing.clientToken)) {@@ -2211,7 +2421,7 @@ export function createRuntimeManager({settleAfterWorkspacePreparationFailure();throw error;}- await prepareFreshRuntime();+ await resetRuntime();const workspacePaths = prioritizeWorkspacePaths(safeProjectDir, options.workspacePaths, {platform: workspacePlatform,@@ -2241,7 +2451,8 @@ export function createRuntimeManager({}}- async function engineStop() {+ async function engineStop(options = {}) {+ recordStop(options, "engine-stop");lifecycleState = "stopping";await stopAllRuntimeChildren();lifecycleState = "idle";@@ -2253,6 +2464,20 @@ export function createRuntimeManager({if (!projectDir) {throw new Error("OpenCode is not configured for a local workspace");}+ if (options.deferredRun === true) {+ const request = restartRequest(options, "deferred");+ recordRestart({ action: "restarting", reason: request.reason, source: request.source, trigger: "engine-restart", why: "idle" });+ nextStartReason = request.reason;+ } else {+ const guard = await guardRestart("engine-restart", options, "engine-restart");+ if (guard.verdict !== "proceed") {+ return {+ ...snapshotEngineState(engineState),+ restartDeferred: guard.verdict === "defer",+ busySessions: guard.busy.sessions,+ };+ }+ }const omnirushRemoteAccess = typeof options.omnirushRemoteAccess === "boolean"? options.omnirushRemoteAccess: omnirushServerState.remoteAccessEnabled;@@ -2282,10 +2507,36 @@ export function createRuntimeManager({}async function omnirushServerInfo() {- return snapshotOmniRushServerState(omnirushServerState);+ return {+ ...snapshotOmniRushServerState(omnirushServerState),+ restarts: await readRuntimeRestartRecords(restartLogFile()),+ pendingRestart: pendingRestartSummary(),+ };}async function omnirushServerRestart(options = {}) {+ // Keep the running host binding unless the caller asks to change it: a+ // restart for another reason (account change) must not drop remote access.+ const remoteAccessEnabled = typeof options.remoteAccessEnabled === "boolean"+ ? options.remoteAccessEnabled+ : omnirushServerState.remoteAccessEnabled === true;+ if (options.deferredRun === true) {+ const request = restartRequest(options, "deferred");+ recordRestart({ action: "restarting", reason: request.reason, source: request.source, trigger: "server-restart", why: "idle" });+ nextStartReason = request.reason;+ } else {+ const guard = await guardRestart("server-restart", options, "server-restart", {+ sameSettings: omnirushServerState.remoteAccessEnabled === remoteAccessEnabled,+ });+ if (guard.verdict !== "proceed") {+ return {+ ...snapshotOmniRushServerState(omnirushServerState),+ restartDeferred: guard.verdict === "defer",+ restartSkipped: guard.verdict === "skip",+ busySessions: guard.busy.sessions,+ };+ }+ }const workspacePaths = prioritizeWorkspacePaths(engineState.projectDir, await listLocalWorkspacePaths(), {platform: workspacePlatform,});@@ -2297,7 +2548,7 @@ export function createRuntimeManager({opencodeBaseUrl: shouldManageOpencode ? null : engineState.baseUrl,opencodeUsername: shouldManageOpencode ? null : engineState.opencodeUsername,opencodePassword: shouldManageOpencode ? null : engineState.opencodePassword,- remoteAccessEnabled: options.remoteAccessEnabled === true,+ remoteAccessEnabled,manageOpencode: shouldManageOpencode,opencodeBinPath: engineState.opencodeBinPath ?? omnirushServerState.managedOpencodeBinPath,});@@ -2382,10 +2633,13 @@ export function createRuntimeManager({return {systemCaCertificates: async () => (await systemCa()).trustedCertificates,engineStart: (projectDir, options) => withRuntimeLifecycle(() => engineStart(projectDir, options)),- engineStop: () => withRuntimeLifecycle(() => engineStop()),+ engineStop: (options) => withRuntimeLifecycle(() => engineStop(options)),engineRestart: (options) => withRuntimeLifecycle(() => engineRestart(options)),- prepareFreshRuntime: () => withRuntimeLifecycle(() => prepareFreshRuntime()),- dispose: () => withRuntimeLifecycle(() => stopAllRuntimeChildren()),+ prepareFreshRuntime: (options) => withRuntimeLifecycle(() => prepareFreshRuntime(options)),+ dispose: (options) => withRuntimeLifecycle(() => {+ recordStop(options ?? { reason: "app_quit" }, "dispose");+ return stopAllRuntimeChildren();+ }),runtimeStatus,engineInfo,engineDoctor,diff --git a/apps/desktop/electron/runtime.test.mjs b/apps/desktop/electron/runtime.test.mjsindex e24221f8..44a4c30a 100644--- a/apps/desktop/electron/runtime.test.mjs+++ b/apps/desktop/electron/runtime.test.mjs@@ -22,6 +22,14 @@ import {snapshotEngineState,snapshotOmniRushServerState,} from "./runtime.mjs";+import {+ appendRuntimeRestartRecord,+ decideDeferredRestart,+ decideRuntimeRestart,+ DEFERRED_RESTART_UNKNOWN_MAX_MS,+ readRuntimeRestartRecords,+ runtimeRestartLogPath,+} from "./runtime-restart-log.mjs";describe("workspace root preparation", () => {it("reports an inaccessible Windows drive as a controlled recoverable error", async () => {@@ -483,3 +491,132 @@ describe("resetRuntimeStatesAfterFailedServerStart", () => {assert.equal(engineState.projectDir, "/workspace/current");});});++describe("built-in server restart guard", () => {+ const idle = { sessions: 0, unknown: 0 };+ const busy = { sessions: 3, unknown: 0 };++ it("never lets an automatic trigger stop a healthy server with running sessions", () => {+ for (const action of ["engine-restart", "server-restart"]) {+ assert.deepEqual(+ decideRuntimeRestart({ action, userInitiated: false, serverRunning: true, serverHealthy: true, busy, sameSettings: false }),+ { verdict: "defer", why: "sessions_busy" },+ );+ }+ });++ it("treats an unreadable busy probe as busy", () => {+ assert.equal(+ decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 } }).verdict,+ "defer",+ );+ });++ it("keeps a healthy server that already runs with the requested settings", () => {+ // A renderer health check that timed out while the server was busy, or a+ // renderer reload re-running boot, must not restart anything.+ assert.equal(+ decideRuntimeRestart({ action: "server-restart", serverRunning: true, serverHealthy: true, busy: idle, sameSettings: true }).verdict,+ "skip",+ );+ assert.equal(+ decideRuntimeRestart({ action: "engine-start", serverRunning: true, serverHealthy: true, busy }).verdict,+ "skip",+ );+ });++ it("restarts when the user asked, when nothing runs, when the server is gone, or when idle", () => {+ assert.equal(decideRuntimeRestart({ action: "server-restart", userInitiated: true, serverRunning: true, serverHealthy: true, busy, sameSettings: true }).verdict, "proceed");+ assert.equal(decideRuntimeRestart({ action: "server-restart", serverRunning: false, serverHealthy: false, busy: idle }).verdict, "proceed");+ assert.deepEqual(+ decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: false, busy }),+ { verdict: "proceed", why: "server_unresponsive" },+ );+ assert.equal(decideRuntimeRestart({ action: "engine-restart", serverRunning: true, serverHealthy: true, busy: idle }).verdict, "proceed");+ });++ it("runs a deferred restart once idle, and gives up waiting on an engine that stays unreadable", () => {+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy, deferredForMs: 60 * 60_000 }), "wait");+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: idle, deferredForMs: 0 }), "run");+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 }, deferredForMs: 1_000 }), "wait");+ assert.equal(+ decideDeferredRestart({ serverRunning: true, serverHealthy: true, busy: { sessions: 0, unknown: 1 }, deferredForMs: DEFERRED_RESTART_UNKNOWN_MAX_MS }),+ "run",+ );+ assert.equal(decideDeferredRestart({ serverRunning: true, serverHealthy: false, busy, deferredForMs: 0 }), "run");+ assert.equal(decideDeferredRestart({ serverRunning: false, serverHealthy: false, busy: idle, deferredForMs: 0 }), "drop");+ });+});++describe("built-in server restart record", () => {+ it("writes each restart to omnirush-server.log and the history, newest first", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-restart-log-"));+ try {+ const serverLogFile = path.join(root, "logs", "omnirush-server.log");+ appendRuntimeRestartRecord({+ serverLogFile,+ now: () => new Date("2026-09-24T07:11:00.000Z"),+ record: { kind: "server", action: "deferred", reason: "engine_reload_failed", source: "engine-reload", busySessions: 3 },+ });+ appendRuntimeRestartRecord({+ serverLogFile,+ now: () => new Date("2026-09-24T07:40:00.000Z"),+ record: { kind: "server", action: "restarting", reason: "engine_reload_failed (deferred until idle)", source: "engine-reload" },+ });++ const records = await readRuntimeRestartRecords(serverLogFile);+ assert.deepEqual(records.map((record) => [record.at, record.action]), [+ ["2026-09-24T07:40:00.000Z", "restarting"],+ ["2026-09-24T07:11:00.000Z", "deferred"],+ ]);+ assert.equal(records[1].busySessions, 3);++ const logLines = (await readFile(serverLogFile, "utf8")).trim().split("\n").map((line) => JSON.parse(line));+ assert.equal(logLines.length, 2);+ assert.equal(logLines[0].body, "Built-in server deferred: engine_reload_failed (engine-reload)");+ assert.equal(logLines[0].attributes["runtime.restart.busy_sessions"], 3);+ assert.equal(logLines[0].timeUnixNano, "1790233860000000000");+ assert.equal(path.dirname(runtimeRestartLogPath(serverLogFile)), path.dirname(serverLogFile));+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ });++ it("skips torn lines and reports an empty history when there is none", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-restart-log-"));+ try {+ const serverLogFile = path.join(root, "omnirush-server.log");+ assert.deepEqual(await readRuntimeRestartRecords(serverLogFile), []);+ await writeFile(runtimeRestartLogPath(serverLogFile), '{"at":"2026-09-24T00:00:00.000Z","action":"started"}\n{"at":"2026-09-24T', "utf8");+ assert.deepEqual((await readRuntimeRestartRecords(serverLogFile)).map((record) => record.action), ["started"]);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ });++ it("returns the history and any pending restart with the server info", async () => {+ const root = await mkdtemp(path.join(os.tmpdir(), "omnirush-runtime-restarts-"));+ const previous = process.env.OMNIRUSH_SERVER_LOG_FILE;+ delete process.env.OMNIRUSH_SERVER_LOG_FILE;+ try {+ appendRuntimeRestartRecord({+ serverLogFile: resolveOmniRushServerLogFile(root),+ record: { kind: "engine", action: "restarted", reason: "engine_unreachable:process_exited", source: "engine-pool-watchdog" },+ });+ const manager = createRuntimeManager({+ app: { getPath: (name) => name === "exe" ? path.join(root, "OmniRush.ai.exe") : root, isPackaged: false },+ desktopRoot: path.dirname(fileURLToPath(import.meta.url)),+ listLocalWorkspacePaths: async () => [],+ localManagedMcpVaultKey: "test-key",+ });+ const info = await manager.omnirushServerInfo();+ assert.equal(info.running, false);+ assert.equal(info.pendingRestart, null);+ assert.deepEqual(info.restarts.map((record) => record.reason), ["engine_unreachable:process_exited"]);+ } finally {+ if (previous === undefined) delete process.env.OMNIRUSH_SERVER_LOG_FILE;+ else process.env.OMNIRUSH_SERVER_LOG_FILE = previous;+ await rm(root, { recursive: true, force: true });+ }+ });+});diff --git a/apps/server/src/embedded.ts b/apps/server/src/embedded.tsindex 9a2860a2..3b6ab60d 100644--- a/apps/server/src/embedded.ts+++ b/apps/server/src/embedded.ts@@ -26,6 +26,7 @@ import {import { createManagedOpencodeServer, type ManagedOpencodeServer, type OpencodeExecutionSnapshot } from "./managed-opencode.js";import {clearTrustedOpencodeProcess,+ collectBusyEngineSessions,createEnginePoolForConfig,createServerLogger,registerTrustedOpencodeProcess,@@ -78,6 +79,11 @@ export type EmbeddedServerHandle = {managedOpencode: { pid: number | null; isAlive: () => boolean } | null;/** Current managed-engine generations for desktop diagnostics and acceptance checks. */managedOpencodePool: () => EnginePoolSnapshot | null;+ /**+ * Sessions still running on the managed engine. The desktop asks before any+ * unattended restart; `unknown` counts unreadable probes, which it treats as busy.+ */+ busySessions: () => Promise<{ sessions: string[]; unknown: number }>;/** Stop the HTTP server and managed OpenCode (if any). */stop: () => Promise<void>;};@@ -364,6 +370,7 @@ export async function startEmbeddedServer(options: EmbeddedServerOptions): Promi}: null,managedOpencodePool: () => enginePool?.snapshot() ?? null,+ busySessions: () => collectBusyEngineSessions(config),stop,};}diff --git a/apps/server/src/engine-pool.ts b/apps/server/src/engine-pool.tsindex 1a412858..f1287414 100644--- a/apps/server/src/engine-pool.ts+++ b/apps/server/src/engine-pool.ts@@ -28,6 +28,7 @@ import {updateEngineInstanceRole,} from "./engine-registry.js";import { createManagedOpencodeServer, type ManagedOpencodeServer } from "./managed-opencode.js";+import { appendRuntimeRestartRecord } from "./runtime-restart-log.js";import { loopbackFetch } from "./server-fetch.js";import type { ServerConfig, WorkspaceInfo } from "./types.js";@@ -81,6 +82,17 @@ export type EnginePoolHooks = {now?: () => number;schedule?: (operation: () => void, delayMs: number) => ReturnType<typeof setTimeout>;waitForHealthy?: (handle: ManagedOpencodeServer) => Promise<void>;+ /**+ * One liveness probe of a running engine: "ok" when it answered at all,+ * "refused" when nothing listens on its port, "failed" for anything else+ * (timeout, reset). Defaults to GET /global/health.+ */+ probeHealth?: (handle: ManagedOpencodeServer) => Promise<"ok" | "refused" | "failed">;+ /**+ * Called right before the watchdog closes a primary it confirmed dead, so+ * the runs it was carrying can be queued for task recovery.+ */+ onPrimaryLost?: (config: ServerConfig) => void;/** Called once when the pool shuts down, before its engines are retired. */onDisposed?: () => void;logger?: EnginePoolLogger;@@ -174,6 +186,17 @@ function minSpawnIntervalMs(): number {return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS", 30_000);}+/**+ * Spacing between the watchdog's liveness probes of a primary that stopped+ * answering proxied requests. Three probes spread over this interval must all+ * fail before a live engine process is closed.+ */+function recoveryProbeIntervalMs(): number {+ return nonNegativeIntFromEnv("OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS", 10_000);+}++const RECOVERY_PROBE_ATTEMPTS = 3;+function drainPollIntervalMs(): number {return positiveIntFromEnv("OMNIRUSH_ENGINE_DRAIN_POLL_MS", 5_000);}@@ -216,6 +239,15 @@ export function isEngineConnectionFailure(error: unknown): boolean {return false;}+function isConnectionRefused(error: unknown): boolean {+ let current: unknown = error;+ for (let depth = 0; depth < 6 && isRecord(current); depth += 1) {+ if (current.code === "ECONNREFUSED") return true;+ current = current.cause;+ }+ return false;+}+export function isConnectionRefusedClassError(error: unknown): boolean {return isEngineConnectionFailure(error);}@@ -447,9 +479,14 @@ export class EnginePool {if (this.consecutiveConnectionFailures < 3) return;const now = this.now();if (now - this.lastRecoveryAt < minSpawnIntervalMs()) return;+ const failures = this.consecutiveConnectionFailures;this.consecutiveConnectionFailures = 0;this.lastRecoveryAt = now;this.recoveryWorkspace = workspace;+ this.hooks.logger?.log("warn", "Managed engine stopped answering; checking whether it is still alive.", {+ "engine.recovery.consecutive_failures": failures,+ "engine.recovery.last_error": error instanceof Error ? error.message : String(error),+ });void this.recoverDeadPrimary(workspace).catch(() => undefined);}@@ -569,6 +606,41 @@ export class EnginePool {};}+ /**+ * Non-idle sessions on every routable engine, for the desktop's restart+ * guard. `unknown` counts the probes that could not be read; callers treat+ * those as busy. Only the given directories are probed, so an instance the+ * engine never booted is not booted just to learn that it is idle.+ */+ async activeSessions(directories: string[]): Promise<{ sessions: string[]; unknown: number }> {+ const sessions = new Set<string>();+ let unknown = 0;+ const targets = this.generations.filter(isRoutableGeneration);+ await Promise.all(targets.flatMap((generation) => directories.map(async (directory) => {+ if (!generation.handle.isAlive()) return;+ try {+ const url = new URL("/session/status", generation.handle.url);+ url.searchParams.set("directory", directory);+ const response = await loopbackFetch(url.toString(), {+ headers: { Authorization: buildEngineAuthProbeHeader(generation.handle.username, generation.handle.password) },+ signal: AbortSignal.timeout(5_000),+ });+ if (!response.ok) throw new Error(`status ${response.status}`);+ const payload: unknown = await response.json();+ if (!isRecord(payload)) throw new Error("unreadable status");+ for (const [sessionId, status] of Object.entries(payload)) {+ if (isRecord(status) && status.type !== "idle") sessions.add(sessionId);+ }+ } catch {+ unknown += 1;+ }+ })));+ for (const owned of this.activeSessionsByGeneration.values()) {+ for (const sessionId of owned) sessions.add(sessionId);+ }+ return { sessions: [...sessions], unknown };+ }+/** Provider sync treats an active drain as busy so it defers instead of parking on waitForDrain(). */hasDrainingGeneration(): boolean {return this.generations.some((entry) => entry.status === "draining");@@ -645,7 +717,9 @@ export class EnginePool {const busy = forceStandby? true- : await this.hooks.engineBusy(this.config, workspace).catch(() => false);+ // Unknown activity counts as busy: a busy engine rolls over to a+ // standby, while an in-place dispose would abort whatever it runs.+ : await this.hooks.engineBusy(this.config, workspace).catch(() => true);if (!busy) {await this.hooks.reloadInPlace(this.config, workspace, { awaitPostRefreshSync });if (primary) primary.fingerprint = fingerprint;@@ -1082,6 +1156,31 @@ export class EnginePool {this.recoveryOrphan = null;}if (previous) {+ const liveness = await this.confirmPrimaryDead(previous);+ if (!liveness.dead) {+ // A slow or overloaded engine still carries every live run; closing+ // it here would abort them all. Only a dead one is replaced.+ this.consecutiveConnectionFailures = 0;+ this.hooks.logger?.log("warn", "Managed engine recovery skipped: the engine still answers its health check.", {+ "engine.recovery.reason": liveness.reason,+ });+ return;+ }+ this.hooks.logger?.log("warn", "Managed engine recovery started.", {+ "engine.recovery.reason": liveness.reason,+ "engine.pid": previous.handle.pid ?? null,+ });+ appendRuntimeRestartRecord({+ kind: "engine",+ action: "restarted",+ reason: `engine_unreachable:${liveness.reason}`,+ source: "engine-pool-watchdog",+ });+ try {+ this.hooks.onPrimaryLost?.(this.config);+ } catch {+ // Recovery bookkeeping must never block replacing a dead engine.+ }if (previous.trustedIdentity) {try {this.hooks.clearTrusted(this.config, previous.trustedIdentity);@@ -1151,6 +1250,40 @@ export class EnginePool {}}+ /**+ * Decide whether a primary that failed several proxied requests is really+ * gone. A process that exited or a port nobody listens on is dead at once;+ * timeouts and resets are only trusted after every spaced probe fails.+ */+ private async confirmPrimaryDead(generation: Generation): Promise<{ dead: boolean; reason: string }> {+ for (let attempt = 1; attempt <= RECOVERY_PROBE_ATTEMPTS; attempt += 1) {+ if (!generation.handle.isAlive()) return { dead: true, reason: "process_exited" };+ const verdict = await this.probeHealth(generation.handle);+ if (verdict === "ok") return { dead: false, reason: "health_ok" };+ if (verdict === "refused") return { dead: true, reason: "connection_refused" };+ if (attempt < RECOVERY_PROBE_ATTEMPTS) {+ await new Promise((resolve) => {+ const timer = setTimeout(resolve, recoveryProbeIntervalMs());+ timer.unref?.();+ });+ }+ }+ return { dead: true, reason: "health_probe_failed" };+ }++ private async probeHealth(handle: ManagedOpencodeServer): Promise<"ok" | "refused" | "failed"> {+ if (this.hooks.probeHealth) return this.hooks.probeHealth(handle);+ try {+ await loopbackFetch(new URL("/global/health", handle.url).toString(), {+ headers: { Authorization: buildEngineAuthProbeHeader(handle.username, handle.password) },+ signal: AbortSignal.timeout(5_000),+ });+ return "ok";+ } catch (error) {+ return isConnectionRefused(error) ? "refused" : "failed";+ }+ }+private scheduleRecovery(workspace: WorkspaceInfo): void {if (this.disposed || this.recoveryTimer) return;this.recoveryWorkspace = workspace;diff --git a/apps/server/src/engine-self-heal.test.ts b/apps/server/src/engine-self-heal.test.tsindex 50ef7ccd..0cd12872 100644--- a/apps/server/src/engine-self-heal.test.ts+++ b/apps/server/src/engine-self-heal.test.ts@@ -5,10 +5,13 @@ import { createManagedProcessClose, type ManagedChildProcess, type ManagedOpencoimport type { ServerConfig, WorkspaceInfo } from "./types.js";const previousSpawnInterval = process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS;+const previousProbeInterval = process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS;afterEach(() => {if (previousSpawnInterval === undefined) delete process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS;else process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = previousSpawnInterval;+ if (previousProbeInterval === undefined) delete process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS;+ else process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS = previousProbeInterval;});function managedHandle(port: number) {@@ -29,7 +32,7 @@ function managedHandle(port: number) {return { handle, closeCalls: () => closeCalls };}-function fixture(spawn: EnginePoolHooks["spawn"]) {+function fixture(spawn: EnginePoolHooks["spawn"], overrides: Partial<EnginePoolHooks> = {}) {let now = 0;const scheduled: Array<() => void> = [];const workspace: WorkspaceInfo = {@@ -75,6 +78,9 @@ function fixture(spawn: EnginePoolHooks["spawn"]) {return setTimeout(() => undefined, 60_000);},waitForHealthy: async () => undefined,+ // Nothing listens on these fake engines' ports.+ probeHealth: async () => "refused",+ ...overrides,};const pool = new EnginePool({ config, template, hooks });return { pool, config, workspace, scheduled, setNow: (value: number) => { now = value; } };@@ -184,6 +190,105 @@ describe("managed engine self-heal", () => {await testFixture.pool.disposeAll();});+ test("keeps a live primary that still answers its health check", async () => {+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";+ const old = managedHandle(44001);+ let spawnCalls = 0;+ let lost = 0;+ const testFixture = fixture(async () => {+ spawnCalls += 1;+ return managedHandle(44002).handle;+ }, { probeHealth: async () => "ok", onPrimaryLost: () => { lost += 1; } });+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });++ // An overloaded engine resets or times out proxied requests but is alive.+ for (let failure = 0; failure < 3; failure += 1) {+ testFixture.pool.reportRequestFailure(old.handle.url, connectionReset(), testFixture.workspace);+ }+ await settle();++ expect(spawnCalls).toBe(0);+ expect(lost).toBe(0);+ expect(old.closeCalls()).toBe(0);+ expect(testFixture.pool.primaryUrl()).toBe(old.handle.url);+ await testFixture.pool.disposeAll();+ });++ test("replaces an unresponsive live primary only after every spaced probe fails", async () => {+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";+ process.env.OMNIRUSH_ENGINE_RECOVERY_PROBE_INTERVAL_MS = "1";+ const old = managedHandle(45001);+ const replacement = managedHandle(45002);+ let probes = 0;+ let lost = 0;+ const testFixture = fixture(async () => replacement.handle, {+ probeHealth: async () => {+ probes += 1;+ return "failed";+ },+ onPrimaryLost: () => { lost += 1; },+ });+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });++ for (let failure = 0; failure < 3; failure += 1) {+ testFixture.pool.reportRequestFailure(old.handle.url, timedOut(), testFixture.workspace);+ }+ await settle();++ expect(probes).toBe(3);+ expect(lost).toBe(1);+ expect(old.handle.isAlive()).toBe(false);+ expect(testFixture.pool.primaryUrl()).toBe(replacement.handle.url);+ await testFixture.pool.disposeAll();+ });++ test("replaces a primary whose process already exited without probing it", async () => {+ process.env.OMNIRUSH_ENGINE_MIN_SPAWN_INTERVAL_MS = "30000";+ const old = managedHandle(46001);+ const replacement = managedHandle(46002);+ let probes = 0;+ const testFixture = fixture(async () => replacement.handle, {+ probeHealth: async () => {+ probes += 1;+ return "ok";+ },+ });+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });+ await old.handle.close();++ for (let failure = 0; failure < 3; failure += 1) {+ testFixture.pool.reportRequestFailure(old.handle.url, refused(), testFixture.workspace);+ }+ await settle();++ expect(probes).toBe(0);+ expect(testFixture.pool.primaryUrl()).toBe(replacement.handle.url);+ await testFixture.pool.disposeAll();+ });++ test("a reload whose busy probe fails rolls over instead of disposing the engine in place", async () => {+ const old = managedHandle(47001);+ const standby = managedHandle(47002);+ let inPlace = 0;+ let spawnCalls = 0;+ const testFixture = fixture(async () => {+ spawnCalls += 1;+ return standby.handle;+ }, {+ engineBusy: async () => { throw new Error("status probe timed out"); },+ reloadInPlace: async () => { inPlace += 1; },+ });+ testFixture.pool.adoptPrimary({ handle: old.handle, fingerprint: "one", registryId: null, trustedIdentity: null });++ const outcome = await testFixture.pool.requestRollover({ reason: "test", workspace: testFixture.workspace, manual: true });++ expect(outcome.action).toBe("rolled_over");+ expect(inPlace).toBe(0);+ expect(spawnCalls).toBe(1);+ expect(testFixture.pool.primaryUrl()).toBe(standby.handle.url);+ await testFixture.pool.disposeAll();+ });+test("dispose escalates an ignored SIGTERM and waits for the child exit", async () => {const signals: Array<NodeJS.Signals | number | undefined> = [];const exitListeners: Array<() => void> = [];diff --git a/apps/server/src/runtime-restart-log.ts b/apps/server/src/runtime-restart-log.tsnew file mode 100644index 00000000..7477739b--- /dev/null+++ b/apps/server/src/runtime-restart-log.ts@@ -0,0 +1,43 @@+import { appendFileSync, mkdirSync } from "node:fs";+import { dirname, join } from "node:path";++import { SERVER_LOG_FILE_ENV } from "./server-log-file.js";++/**+ * Persisted record of every built-in server and managed engine restart.+ *+ * The desktop runtime (apps/desktop/electron/runtime-restart-log.mjs) writes+ * server restarts to the same JSON-lines file next to omnirush-server.log;+ * the server adds the engine recoveries it performs on its own. The desktop+ * reads the tail back into omnirushServerInfo for the diagnostics bundle.+ */+export const RUNTIME_RESTART_LOG_FILE_NAME = "runtime-restarts.jsonl";++export type RuntimeRestartRecord = {+ at: string;+ kind: "server" | "engine";+ action: string;+ reason: string;+ source: string;+ busySessions?: number | null;+ detail?: string | null;+};++export function runtimeRestartLogPath(env: NodeJS.ProcessEnv = process.env): string | null {+ const logFile = env[SERVER_LOG_FILE_ENV]?.trim();+ return logFile ? join(dirname(logFile), RUNTIME_RESTART_LOG_FILE_NAME) : null;+}++/** Best effort: a record that cannot be written never blocks a recovery. */+export function appendRuntimeRestartRecord(+ record: Omit<RuntimeRestartRecord, "at"> & { at?: string },+ path: string | null = runtimeRestartLogPath(),+): void {+ if (!path) return;+ try {+ mkdirSync(dirname(path), { recursive: true });+ appendFileSync(path, `${JSON.stringify({ ...record, at: record.at ?? new Date().toISOString() })}\n`, "utf8");+ } catch {+ // Diagnostics only.+ }+}diff --git a/apps/server/src/server.ts b/apps/server/src/server.tsindex fef14ed9..a73cd580 100644--- a/apps/server/src/server.ts+++ b/apps/server/src/server.ts@@ -1,6 +1,15 @@import { managedDesktopPolicy } from "./managed-desktop-policy.js";import { parseApprovalMode, resolveApprovalMode } from "./approval-mode.js";-import { createTaskRecovery, setTaskRecovery } from "./task-recovery.js";+import { createTaskRecovery, requeueTaskRecoveryAfterEngineLoss, setTaskRecovery } from "./task-recovery.js";+import {+ directoryMatchKey,+ EmptySessionDirectoryCache,+ isSessionListRequest,+ listSessionsByDirectory,+ serveSessionListWithFallback,+ sessionListFallbackDeadlineMs,+ sessionListFallbackQuery,+} from "./session-list-fallback.js";import { managedPolicyActionSchema } from "./managed-policy-rules.js";import { readFile, realpath, writeFile, rm, stat } from "node:fs/promises";import { homedir, hostname } from "node:os";@@ -659,7 +668,12 @@ async function assertWorkspaceOwnsProxiedSessionRead(realpath(sessionDirectory).catch(() => sessionDirectory),]): [directory, sessionDirectory];- if (!actualDirectory || actualDirectory !== expectedDirectory) {+ // Local folders compare the way their file system does: a session created+ // before a case-only rename (Windows, macOS) still belongs to the folder.+ const sameDirectory = workspace.workspaceType === "local" && actualDirectory+ ? directoryMatchKey(actualDirectory) === directoryMatchKey(expectedDirectory)+ : actualDirectory === expectedDirectory;+ if (!actualDirectory || !sameDirectory) {throw new ApiError(404, "session_not_found", "Session not found");}}@@ -1857,6 +1871,14 @@ export async function proxyOpencodeRequest(input: {kind: engineAggregateKind(proxyPath) ?? "pending",});}+ if (pool && workspace && workspace.workspaceType !== "remote" && method === "POST"+ && normalizeOpencodeProxyPath(proxyPath) === "/instance/dispose") {+ // A client-side dispose (provider refresh) would abort every run in this+ // folder. The pool applies it the same way without that: in place when+ // idle, through a standby while sessions are running.+ await reloadOpencodeEngine(input.config, workspace, undefined, { reason: "client_instance_dispose", manual: true });+ return jsonResponse(true);+ }const targetUrl = buildOpencodeProxyUrl(baseUrl, proxyPath, search);// Managed OpenCode proxy traffic is loopback/engine I/O; keep streaming on Node fetch.if (isSessionCommandProxyRequest(method, proxyPath)) {@@ -1950,9 +1972,66 @@ export async function proxyOpencodeRequest(input: {if (workspace && workspace.workspaceType !== "remote" && isPromptAsyncProxyRequest(method, proxyPath)) {return withEngineDirectoryFence(input.config, workspace, forwardAndCollect);}+ if (pool && workspace && workspace.workspaceType !== "remote" && isSessionListRequest(method, normalizeOpencodeProxyPath(proxyPath))) {+ return serveLocalSessionList(input.config, workspace, search, forwardAndCollect());+ }return forwardAndCollect();}+const emptySessionDirectoriesByServer = new WeakMap<ServerConfig, EmptySessionDirectoryCache>();++/**+ * A local workspace's session list must not depend on its engine instance+ * finishing a cold boot, or on the stored project id and directory string+ * still matching the folder exactly. See session-list-fallback.ts.+ */+async function serveLocalSessionList(+ config: ServerConfig,+ workspace: WorkspaceInfo,+ search: string,+ primary: Promise<Response>,+): Promise<Response> {+ const directory = resolveOpencodeDirectory(workspace) ?? workspace.path;+ const cacheKey = directoryMatchKey(directory);+ const cache = emptySessionDirectoriesByServer.get(config) ?? new EmptySessionDirectoryCache();+ emptySessionDirectoriesByServer.set(config, cache);+ const query = sessionListFallbackQuery(search);+ return serveSessionListWithFallback({+ primary,+ deadlineMs: sessionListFallbackDeadlineMs(),+ fallback: async (cause) => {+ if (cause === "empty" && cache.has(cacheKey)) return [];+ const engine = primaryManagedEngineConnection(config);+ if (!engine) throw new Error("The managed engine is unavailable for the session index");+ const canonical = await realpath(directory).catch(() => directory);+ const items = await listSessionsByDirectory({+ directories: [workspace.path, directory, canonical],+ query,+ fetchPage: async (params) => {+ const response = await loopbackFetch(`${engine.baseUrl.replace(/\/+$/, "")}/experimental/session?${params.toString()}`, {+ headers: { Authorization: buildEngineAuthProbeHeader(engine.username, engine.password) },+ signal: AbortSignal.timeout(10_000),+ });+ if (!response.ok) throw new Error(`OpenCode session index failed with status ${response.status}`);+ const payload: unknown = await response.json();+ if (!Array.isArray(payload)) throw new Error("OpenCode session index returned an unreadable page");+ return { items: payload, nextCursor: response.headers.get("x-next-cursor") };+ },+ });+ if (items.length === 0) cache.remember(cacheKey);+ else cache.forget(cacheKey);+ return items;+ },+ onFallback: ({ cause, count }) => {+ createServerLogger(config).log("warn", "Session list served from the engine session index.", {+ "workspace.id": workspace.id,+ "session_list.fallback_cause": cause,+ "session_list.count": count,+ });+ },+ });+}+function isEngineEventPath(proxyPath: string): boolean {const normalized = normalizeOpencodeProxyPath(proxyPath);return normalized === "/event" || normalized === "/global/event" || normalized === "/api/event";@@ -4788,19 +4867,37 @@ async function requireWorkspaceRunModeIdle(config: ServerConfig, workspace: Work/*** True when the managed engine reports any non-idle session (subagent child* sessions carry their own ids and statuses, so they count too). Unknown- * activity reports false: a reload against a dead engine fails loudly on its- * own, and "unknown" must never park reloads forever.+ * activity reports `unknownIsBusy`: false for callers that would otherwise+ * park a reload forever against a dead engine, true for the rollover pool,+ * where "busy" only means rolling over to a standby instead of disposing an+ * engine that may still be running sessions.*/-async function engineHasActiveSessions(config: ServerConfig, workspace: WorkspaceInfo): Promise<boolean> {+async function engineHasActiveSessions(+ config: ServerConfig,+ workspace: WorkspaceInfo,+ unknownIsBusy = false,+): Promise<boolean> {try {const opencode = createWorkspaceOpencodeClient(config, workspace);const statuses = unwrapOpencodeResult(await opencode.session.status(), "/session/status");return Object.values(statuses).some((status) => status.type !== "idle");} catch {- return false;+ return unknownIsBusy;}}+/**+ * Sessions still running on the managed engine, for the desktop's restart+ * guard. Only directories the engine already serves are probed; `unknown`+ * counts probes that could not be read and callers treat them as busy.+ */+export async function collectBusyEngineSessions(config: ServerConfig): Promise<{ sessions: string[]; unknown: number }> {+ const pool = enginePoolForConfig(config);+ if (!pool) return { sessions: [], unknown: 0 };+ const directories = new Set(engineInstanceReaperForConfig(config)?.snapshot().map((instance) => instance.directory) ?? []);+ return pool.activeSessions([...directories]);+}+function primaryManagedEngineConnection(config: ServerConfig): EnginePoolConnection | null {return enginePoolForConfig(config)?.connections().find((entry) => entry.role === "primary") ?? null;}@@ -5675,7 +5772,8 @@ export function createEnginePoolForConfig(input: {onDisposed: () => threadApprovals?.stop(),reloadInPlace: (poolConfig, workspace, options) =>reloadOpencodeEngineInPlace(poolConfig, workspace, undefined, options),- engineBusy: (poolConfig, workspace) => engineHasActiveSessions(poolConfig, workspace),+ engineBusy: (poolConfig, workspace) => engineHasActiveSessions(poolConfig, workspace, true),+ onPrimaryLost: (poolConfig) => requeueTaskRecoveryAfterEngineLoss(poolConfig),postRefreshSync: async (poolConfig, workspace) => {await postEngineRefreshSync(poolConfig, workspace, activeEngineMcpServerState(poolConfig));await syncAllWorkspacesRuntimeMcpToEngine(poolConfig);diff --git a/apps/server/src/session-list-fallback.test.ts b/apps/server/src/session-list-fallback.test.tsnew file mode 100644index 00000000..43d0f058--- /dev/null+++ b/apps/server/src/session-list-fallback.test.ts@@ -0,0 +1,203 @@+import { describe, expect, test } from "bun:test";++import {+ directoryMatchKey,+ EmptySessionDirectoryCache,+ isSessionListRequest,+ listSessionsByDirectory,+ serveSessionListWithFallback,+ sessionListFallbackQuery,+ type GlobalSessionPage,+} from "./session-list-fallback.js";++function session(id: string, directory: string, updated: number) {+ return { id, directory, parentID: undefined, title: id, time: { created: updated, updated }, project: { id: "p", worktree: directory } };+}++/** The engine's global index: newest first, `cursor` = last item's time.updated. */+function globalIndex(items: ReturnType<typeof session>[]) {+ const calls: URLSearchParams[] = [];+ const fetchPage = async (params: URLSearchParams): Promise<GlobalSessionPage> => {+ calls.push(params);+ const limit = Number(params.get("limit"));+ const cursor = params.get("cursor");+ const sorted = [...items].sort((left, right) => right.time.updated - left.time.updated);+ const eligible = cursor ? sorted.filter((item) => item.time.updated < Number(cursor)) : sorted;+ const page = eligible.slice(0, limit);+ const more = eligible.length > limit && page.length > 0;+ return { items: page, nextCursor: more ? String(page[page.length - 1]!.time.updated) : null };+ };+ return { fetchPage, calls };+}++describe("directoryMatchKey", () => {+ test("Windows folders compare case-insensitively across slash styles, prefixes and trailing separators", () => {+ const current = "C:\\Users\\vedant\\Desktop\\RepoRadar";+ for (const stored of [+ "C:/Users/vedant/Desktop/reporadar",+ "c:/users/vedant/desktop/RepoRadar/",+ "\\\\?\\C:\\Users\\vedant\\Desktop\\RepoRadar",+ "C:\\Users\\vedant\\\\Desktop\\RepoRadar\\",+ ]) {+ expect(directoryMatchKey(stored, "win32")).toBe(directoryMatchKey(current, "win32"));+ }+ });++ test("Windows keeps different folders, drives and UNC shares apart", () => {+ const key = directoryMatchKey("C:\\Users\\vedant\\taskforge", "win32");+ expect(directoryMatchKey("D:\\Users\\vedant\\taskforge", "win32")).not.toBe(key);+ expect(directoryMatchKey("C:\\Users\\vedant\\taskforge2", "win32")).not.toBe(key);+ expect(directoryMatchKey("\\\\?\\UNC\\server\\share\\repo", "win32")).toBe(directoryMatchKey("\\\\server\\share\\repo", "win32"));+ expect(directoryMatchKey("\\\\server\\share\\repo", "win32")).not.toBe(directoryMatchKey("C:\\share\\repo", "win32"));+ expect(directoryMatchKey("C:\\", "win32")).toBe("c:/");+ });++ test("Linux stays case-sensitive; macOS default volumes do not", () => {+ expect(directoryMatchKey("/home/a/Repo", "linux")).not.toBe(directoryMatchKey("/home/a/repo", "linux"));+ expect(directoryMatchKey("/home/a/repo/", "linux")).toBe(directoryMatchKey("/home/a/repo", "linux"));+ expect(directoryMatchKey("/Users/a/RepoRadar", "darwin")).toBe(directoryMatchKey("/Users/a/reporadar", "darwin"));+ expect(directoryMatchKey("/", "linux")).toBe("/");+ });+});++describe("session list request detection", () => {+ test("only the bare list is served with the fallback", () => {+ expect(isSessionListRequest("GET", "/session")).toBe(true);+ expect(isSessionListRequest("POST", "/session")).toBe(false);+ expect(isSessionListRequest("GET", "/session/status")).toBe(false);+ expect(isSessionListRequest("GET", "/session/ses_1")).toBe(false);+ });++ test("the query mirrors the list's roots, start, search and limit", () => {+ expect(sessionListFallbackQuery("?limit=200&directory=C%3A%5Cx")).toEqual({ roots: undefined, start: undefined, search: undefined, limit: 200 });+ expect(sessionListFallbackQuery("?roots=true&start=5&search=%20fix%20")).toEqual({ roots: true, start: 5, search: "fix", limit: 100 });+ });+});++describe("listSessionsByDirectory", () => {+ test("finds a re-cased Windows folder's sessions across pages and drops the project join", async () => {+ const items = [+ ...Array.from({ length: 700 }, (_, index) => session(`other_${index}`, "C:/Users/vedant/helm", 10_000 - index)),+ session("ses_old_case", "C:/Users/vedant/reporadar", 1_000),+ session("ses_new_case", "C:/Users/vedant/RepoRadar", 2_000),+ ];+ const index = globalIndex(items);+ const result = await listSessionsByDirectory({+ fetchPage: index.fetchPage,+ directories: ["C:\\Users\\vedant\\RepoRadar"],+ query: { limit: 200 },+ platform: "win32",+ });+ expect(result.map((item) => item.id)).toEqual(["ses_new_case", "ses_old_case"]);+ expect(result[0]).not.toHaveProperty("project");+ expect(index.calls).toHaveLength(2);+ expect(index.calls[0]!.get("archived")).toBe("true");+ });++ test("stops at the limit and at the scan budget", async () => {+ const items = Array.from({ length: 50 }, (_, index) => session(`ses_${index}`, "/w", 100 - index));+ const limited = await listSessionsByDirectory({ fetchPage: globalIndex(items).fetchPage, directories: ["/w"], query: { limit: 3 }, platform: "linux" });+ expect(limited.map((item) => item.id)).toEqual(["ses_0", "ses_1", "ses_2"]);++ const index = globalIndex(Array.from({ length: 2_000 }, (_, index) => session(`x_${index}`, "/other", 5_000 - index)));+ const none = await listSessionsByDirectory({ fetchPage: index.fetchPage, directories: ["/w"], query: { limit: 10 }, platform: "linux", maxScanned: 1_000 });+ expect(none).toEqual([]);+ expect(index.calls).toHaveLength(2);+ });+});++function delayed<T>(value: T, ms: number): Promise<T> {+ return new Promise((resolve) => setTimeout(() => resolve(value), ms));+}++describe("serveSessionListWithFallback", () => {+ const indexed = [{ id: "ses_a", directory: "C:/w" }];++ test("a quick non-empty engine answer passes through untouched", async () => {+ let fallbackCalls = 0;+ const response = await serveSessionListWithFallback({+ primary: Promise.resolve(Response.json([{ id: "ses_engine" }])),+ fallback: async () => { fallbackCalls += 1; return indexed; },+ deadlineMs: 1_000,+ });+ expect(await response.json()).toEqual([{ id: "ses_engine" }]);+ expect(fallbackCalls).toBe(0);+ });++ test("an instance still booting past the deadline is answered from the index", async () => {+ const events: string[] = [];+ const response = await serveSessionListWithFallback({+ primary: delayed(Response.json([{ id: "late" }]), 200),+ fallback: async (cause) => { events.push(cause); return indexed; },+ deadlineMs: 10,+ onFallback: ({ cause, count }) => events.push(`${cause}:${count}`),+ });+ expect(await response.json()).toEqual(indexed);+ expect(events).toEqual(["slow", "slow:1"]);+ });++ test("an empty engine answer for a folder whose sessions live under another spelling uses the index", async () => {+ const response = await serveSessionListWithFallback({+ primary: Promise.resolve(Response.json([])),+ fallback: async (cause) => (cause === "empty" ? indexed : []),+ deadlineMs: 1_000,+ });+ expect(await response.json()).toEqual(indexed);+ });++ test("a truly empty folder stays empty", async () => {+ const response = await serveSessionListWithFallback({+ primary: Promise.resolve(Response.json([])),+ fallback: async () => [],+ deadlineMs: 1_000,+ });+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual([]);+ });++ test("a failed engine request is answered from the index, or rethrown when the index fails too", async () => {+ const answered = await serveSessionListWithFallback({+ primary: Promise.reject(new Error("fetch failed")),+ fallback: async () => indexed,+ deadlineMs: 1_000,+ });+ expect(await answered.json()).toEqual(indexed);++ const failure = new Error("engine down");+ await expect(serveSessionListWithFallback({+ primary: Promise.reject(failure),+ fallback: async () => { throw new Error("index down"); },+ deadlineMs: 1_000,+ })).rejects.toBe(failure);+ });++ test("a slow engine still wins when the index fails", async () => {+ const response = await serveSessionListWithFallback({+ primary: delayed(Response.json([{ id: "late" }]), 30),+ fallback: async () => { throw new Error("index down"); },+ deadlineMs: 5,+ });+ expect(await response.json()).toEqual([{ id: "late" }]);+ });++ test("an engine error status is replaced by the index answer", async () => {+ const response = await serveSessionListWithFallback({+ primary: Promise.resolve(Response.json({ error: "boot failed" }, { status: 500 })),+ fallback: async () => indexed,+ deadlineMs: 1_000,+ });+ expect(response.status).toBe(200);+ expect(await response.json()).toEqual(indexed);+ });+});++describe("EmptySessionDirectoryCache", () => {+ test("remembers an empty folder for the TTL only", () => {+ let now = 0;+ const cache = new EmptySessionDirectoryCache(() => now, 1_000);+ cache.remember("c:/w");+ expect(cache.has("c:/w")).toBe(true);+ now = 1_001;+ expect(cache.has("c:/w")).toBe(false);+ });+});diff --git a/apps/server/src/session-list-fallback.ts b/apps/server/src/session-list-fallback.tsnew file mode 100644index 00000000..06fbb444--- /dev/null+++ b/apps/server/src/session-list-fallback.ts@@ -0,0 +1,224 @@+/**+ * Keep a local workspace's session list answerable.+ *+ * The engine's GET /session is instance-scoped: the first request for a+ * directory waits for that directory's instance to boot (git probes, config,+ * plugin installs), which can take well over the app's request timeout on a+ * cold Windows start. It also matches sessions by the stored project id AND+ * the exact canonical directory string, so a renamed, re-cased or re-rooted+ * folder, or a recreated .git, lists as empty although its sessions exist.+ *+ * The fallback answers from the engine's global session index instead+ * (GET /experimental/session, served by the engine's already-booted default+ * instance), filtered here by directory with platform-aware comparison.+ */++export const SESSION_LIST_FALLBACK_DEADLINE_ENV = "OMNIRUSH_SESSION_LIST_FALLBACK_MS";+const DEFAULT_DEADLINE_MS = 6_000;+const PAGE_SIZE = 500;+const MAX_SCANNED = 5_000;+const EMPTY_RESULT_TTL_MS = 60_000;++export type SessionListFallbackCause = "slow" | "empty" | "failed";++export function sessionListFallbackDeadlineMs(env: NodeJS.ProcessEnv = process.env): number {+ const raw = Number(env[SESSION_LIST_FALLBACK_DEADLINE_ENV]);+ return Number.isFinite(raw) && raw > 0 ? raw : DEFAULT_DEADLINE_MS;+}++export function isSessionListRequest(method: string, normalizedPath: string): boolean {+ return method.toUpperCase() === "GET" && normalizedPath === "/session";+}++function isRecord(value: unknown): value is Record<string, unknown> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++/**+ * Comparable form of a directory. Windows paths are case-insensitive and may+ * arrive with backslashes, a `\\?\` prefix, a lower-case drive letter or a+ * trailing separator; the engine stores forward slashes. macOS volumes are+ * case-insensitive by default, so a case-only rename must still match there.+ */+export function directoryMatchKey(directory: string, platform: NodeJS.Platform = process.platform): string {+ let value = directory.trim();+ if (platform === "win32") {+ value = value.replace(/\\/g, "/");+ value = value.replace(/^\/\/\?\/UNC\//i, "//").replace(/^\/\/\?\//, "");+ const unc = value.startsWith("//");+ value = (unc ? "//" : "") + value.slice(unc ? 2 : 0).replace(/\/{2,}/g, "/");+ } else {+ value = value.replace(/\/{2,}/g, "/");+ }+ if (value.length > 1 && value.endsWith("/") && !/^[A-Za-z]:\/$/.test(value)) value = value.replace(/\/+$/, "");+ return platform === "win32" || platform === "darwin" ? value.toLowerCase() : value;+}++export type SessionListFallbackQuery = {+ roots?: boolean;+ start?: number;+ search?: string;+ limit: number;+};++export function sessionListFallbackQuery(search: string): SessionListFallbackQuery {+ const params = new URLSearchParams(search);+ const limit = Number(params.get("limit"));+ const start = Number(params.get("start"));+ const searchText = params.get("search")?.trim();+ return {+ roots: params.get("roots") === "true" ? true : undefined,+ start: Number.isFinite(start) && start > 0 ? start : undefined,+ search: searchText ? searchText : undefined,+ limit: Number.isFinite(limit) && limit > 0 ? Math.floor(limit) : 100,+ };+}++export type GlobalSessionPage = { items: unknown[]; nextCursor: string | null };++/**+ * Page through the engine's global session index (newest first) and keep the+ * sessions whose directory matches one of `directories`. Archived sessions are+ * included because the instance list includes them too.+ */+export async function listSessionsByDirectory(input: {+ fetchPage: (params: URLSearchParams) => Promise<GlobalSessionPage>;+ directories: string[];+ query: SessionListFallbackQuery;+ platform?: NodeJS.Platform;+ maxScanned?: number;+}): Promise<Record<string, unknown>[]> {+ const platform = input.platform ?? process.platform;+ const keys = new Set(input.directories.filter((entry) => entry.trim()).map((entry) => directoryMatchKey(entry, platform)));+ const matches: Record<string, unknown>[] = [];+ const maxScanned = input.maxScanned ?? MAX_SCANNED;+ let scanned = 0;+ let cursor: string | null = null;+ while (matches.length < input.query.limit && scanned < maxScanned) {+ const params = new URLSearchParams({ limit: String(PAGE_SIZE), archived: "true" });+ if (input.query.roots) params.set("roots", "true");+ if (input.query.start !== undefined) params.set("start", String(input.query.start));+ if (input.query.search) params.set("search", input.query.search);+ if (cursor) params.set("cursor", cursor);+ const page = await input.fetchPage(params);+ scanned += page.items.length;+ for (const item of page.items) {+ if (!isRecord(item) || typeof item.directory !== "string") continue;+ if (!keys.has(directoryMatchKey(item.directory, platform))) continue;+ const { project: _project, ...session } = item;+ matches.push(session);+ if (matches.length >= input.query.limit) break;+ }+ if (!page.nextCursor || page.items.length === 0) break;+ cursor = page.nextCursor;+ }+ return matches;+}++type PrimaryOutcome = { ok: true; response: Response } | { ok: false; error: unknown };++function jsonList(items: unknown[]): Response {+ return new Response(JSON.stringify(items), {+ status: 200,+ headers: { "Content-Type": "application/json" },+ });+}++/**+ * Serve a session list: the engine's own answer when it is quick and+ * non-empty, otherwise the directory index. A slow instance boot keeps+ * running in the background, so the next list is served by the engine again.+ */+export async function serveSessionListWithFallback(input: {+ primary: Promise<Response>;+ /** `cause` says why the index is consulted; "empty" may answer from a negative cache. */+ fallback: (cause: SessionListFallbackCause) => Promise<Record<string, unknown>[]>;+ deadlineMs: number;+ onFallback?: (event: { cause: SessionListFallbackCause; count: number }) => void;+}): Promise<Response> {+ const primary: Promise<PrimaryOutcome> = input.primary.then(+ (response) => ({ ok: true, response }),+ (error: unknown) => ({ ok: false, error }),+ );+ let fallbackRun: Promise<Record<string, unknown>[] | null> | null = null;+ const fallback = (cause: SessionListFallbackCause) => {+ fallbackRun ??= input.fallback(cause).catch(() => null);+ return fallbackRun;+ };+ const served = (cause: SessionListFallbackCause, items: Record<string, unknown>[]) => {+ input.onFallback?.({ cause, count: items.length });+ return jsonList(items);+ };++ const finish = async (outcome: PrimaryOutcome): Promise<Response> => {+ if (!outcome.ok) {+ const items = await fallback("failed");+ if (items) return served("failed", items);+ throw outcome.error;+ }+ const { response } = outcome;+ if (!response.ok) {+ const items = await fallback("failed");+ return items ? served("failed", items) : response;+ }+ const text = await response.text();+ let parsed: unknown = null;+ try {+ parsed = JSON.parse(text);+ } catch {+ parsed = null;+ }+ if (Array.isArray(parsed) && parsed.length === 0) {+ const items = await fallback("empty");+ if (items && items.length > 0) return served("empty", items);+ }+ return new Response(text, { status: response.status, statusText: response.statusText, headers: response.headers });+ };++ let timer: ReturnType<typeof setTimeout> | null = null;+ const deadline = new Promise<null>((resolve) => {+ timer = setTimeout(() => resolve(null), input.deadlineMs);+ timer.unref?.();+ });+ const first = await Promise.race([primary, deadline]);+ if (timer) clearTimeout(timer);+ if (first) return finish(first);++ type Winner = { source: "primary"; outcome: PrimaryOutcome } | { source: "fallback"; items: Record<string, unknown>[] };+ const fromPrimary = primary.then((outcome): Winner => ({ source: "primary", outcome }));+ const winner = await Promise.race<Winner>([+ fromPrimary,+ fallback("slow").then((items): Winner | Promise<Winner> => items ? { source: "fallback", items } : fromPrimary),+ ]);+ if (winner.source === "primary") return finish(winner.outcome);+ // The engine keeps booting the instance; release its eventual answer.+ void primary.then((outcome) => {+ if (outcome.ok) void outcome.response.body?.cancel().catch(() => undefined);+ });+ return served("slow", winner.items);+}++/** Directories known to have no sessions anywhere, so repeated empty lists skip the index scan. */+export class EmptySessionDirectoryCache {+ private readonly entries = new Map<string, number>();++ constructor(private readonly now: () => number = Date.now, private readonly ttlMs = EMPTY_RESULT_TTL_MS) {}++ has(key: string): boolean {+ const at = this.entries.get(key);+ if (at === undefined) return false;+ if (this.now() - at > this.ttlMs) {+ this.entries.delete(key);+ return false;+ }+ return true;+ }++ remember(key: string): void {+ this.entries.set(key, this.now());+ }++ forget(key: string): void {+ this.entries.delete(key);+ }+}diff --git a/apps/server/src/task-recovery.test.ts b/apps/server/src/task-recovery.test.tsindex 1fbae52b..7f4712d7 100644--- a/apps/server/src/task-recovery.test.ts+++ b/apps/server/src/task-recovery.test.ts@@ -92,6 +92,7 @@ async function fixture(engine: "v1" | "v2") {return {tasks, resumes, tick, config,readHook(hook?: () => Promise<void>) { beforeRead = hook; },+ engineLost() { recovery.requeueAfterEngineLoss(); },policy(ready: boolean) { policyReady = ready; },loseAck() { loseAcknowledgement = true; },async send(id: string, stop = false, optOut = false) {@@ -199,6 +200,28 @@ for (const engine of ["v1", "v2"] as const) {expect(f.resumes).toHaveLength(1);});+ test(`${engine}: a running turn whose engine was replaced without a server restart resumes once`, async () => {+ const f = await fixture(engine);+ await f.send("ses_work"); await f.send("ses_done"); await f.tick();+ // The watchdog replaced a dead engine: both runs stopped, one had finished.+ f.tasks.get("ses_work")!.active = false;+ f.tasks.get("ses_done")!.active = false;+ f.tasks.get("ses_done")!.completed = true;+ f.engineLost();+ await f.tick(); await f.tick();+ expect(f.resumes.map((resume) => resume.id)).toEqual(["ses_work"]);+ await f.tick(); await f.tick();+ expect(f.resumes).toHaveLength(1);+ });++ test(`${engine}: without an engine replacement an idle turn is never re-prompted`, async () => {+ const f = await fixture(engine);+ await f.send("ses_work"); await f.tick();+ f.tasks.get("ses_work")!.active = false;+ await f.tick(); await f.tick();+ expect(f.resumes).toEqual([]);+ });+test(`${engine}: shutdown checkpoints a just-admitted task before the first poll`, async () => {const f = await fixture(engine);await f.send("ses_work"); await f.restart(true); await f.tick();diff --git a/apps/server/src/task-recovery.ts b/apps/server/src/task-recovery.tsindex f8cba641..d37111dd 100644--- a/apps/server/src/task-recovery.ts+++ b/apps/server/src/task-recovery.ts@@ -204,6 +204,24 @@ export async function createTaskRecovery(return {tick,owns: (request: Request) => ownedRequests.has(request),+ /**+ * The managed engine died under running turns and is being replaced+ * without a server restart. Treat those turns exactly like the ones a+ * restart interrupted: the startup path re-observes each one against the+ * new engine and resumes it only if the same turn is still unfinished.+ */+ requeueAfterEngineLoss() {+ if (stopped) return;+ let changed = false;+ for (const record of records.values()) {+ if (record.phase !== "running" || record.userId === null) continue;+ record.shutdown = true;+ startup.add(key(record));+ recovered.delete(key(record));+ changed = true;+ }+ if (changed) void persist().catch(() => undefined);+ },async forward(workspace: WorkspaceInfo, engine: Engine, path: string, req: Request, send: () => Promise<Response>) {if (workspace.workspaceType !== "local" || !["POST", "PUT", "PATCH", "DELETE"].includes(req.method)) return send();const match = path.replace(/^\/opencode2\/api|^\/opencode/, "").match(/^\/session\/([^/]+)(.*)$/);@@ -276,3 +294,4 @@ type TaskRecovery = Awaited<ReturnType<typeof createTaskRecovery>>;const coordinators = new WeakMap<ServerConfig, TaskRecovery>();export function setTaskRecovery(config: ServerConfig, recovery: TaskRecovery) { coordinators.set(config, recovery); }export async function stopTaskRecovery(config: ServerConfig) { await coordinators.get(config)?.stop(); }+export function requeueTaskRecoveryAfterEngineLoss(config: ServerConfig) { coordinators.get(config)?.requeueAfterEngineLoss(); }diff --git a/packages/types/src/desktop-ipc.ts b/packages/types/src/desktop-ipc.tsindex b16914ae..0219b1fe 100644--- a/packages/types/src/desktop-ipc.ts+++ b/packages/types/src/desktop-ipc.ts@@ -85,6 +85,17 @@ export type DesktopIntegrationResult = {error?: string;};+/** One built-in server or engine restart, as recorded in runtime-restarts.jsonl. */+export type OmniRushRuntimeRestartRecord = {+ at: string;+ kind: "server" | "engine";+ /** restarting, started, deferred, skipped, dropped, stopping, or restarted (engine watchdog). */+ action: string;+ reason: string;+ source: string;+ [key: string]: unknown;+};+export type OmniRushServerInfo = {running: boolean;/**@@ -113,6 +124,13 @@ export type OmniRushServerInfo = {lastStdout: string | null;lastStderr: string | null;managedOpencodeExecution: OpencodeExecutionSnapshot | null;+ /** Recent restarts, newest first (omnirushServerInfo only). */+ restarts?: OmniRushRuntimeRestartRecord[];+ /** An automatic restart that waits until no session is running. */+ pendingRestart?: { action: string; reason: string; source: string; requestedAt: string } | null;+ /** Set on an omnirushServerRestart answer that kept the running server. */+ restartDeferred?: boolean;+ restartSkipped?: boolean;};/** Outcome of the remote device-session revocation attempted during sign-out. */