Skip to content

Beautiful, AI-native markdown IDE and LLM wiki

fix(desktop): recover a window whose renderer crashed (PRD-7759) (#3155)

Andrew Mikofalvyc7e2bd1e6e7a
diff --git a/.changeset/renderer-crash-recovery.md b/.changeset/renderer-crash-recovery.md
new file mode 100644
index 000000000..3f3f1ebea
--- /dev/null
+++ b/.changeset/renderer-crash-recovery.md
@@ -0,0 +1,7 @@
+---
+"@inkeep/open-knowledge": patch
+---
+
+A window whose renderer stops responding now recovers itself instead of going blank. Chromium occasionally tears down the process that draws a window; the frame stays and nothing paints in it. OpenKnowledge noticed this and filed it for a crash report, but did nothing about the window, so it stayed empty until you happened to try Cmd-R. There was no message and no visible way back — one report described coming back from a short break to an empty window and reloading on a guess, unsure whether that would take the running agent and a session of unsaved comments with it.
+
+The window is now reloaded automatically the moment its renderer dies. If it dies a second time within a minute, the reload stops rather than looping, and a dialog offers to reload it or leave it alone. A window that dies repeatedly but slowly enough to keep dodging that minute is caught as well, after a few recoveries. That dialog also says what the blank window could not: documents and running agents live in the OpenKnowledge server, not in the window, so reloading restores the view without interrupting them.
diff --git a/packages/desktop/src/main/index.ts b/packages/desktop/src/main/index.ts
index 7f9978adb..6fe264aa1 100644
--- a/packages/desktop/src/main/index.ts
+++ b/packages/desktop/src/main/index.ts
@@ -115,7 +115,7 @@ import {
writeBundleDecision,
writeTargetVersion,
} from '@inkeep/open-knowledge-server';
-import type { BrowserWindowConstructorOptions, MessageBoxOptions } from 'electron';
+import type { BrowserWindowConstructorOptions, MessageBoxOptions, WebContents } from 'electron';
import {
app,
BrowserWindow,
@@ -323,6 +323,7 @@ import {
} from './reduced-transparency-handler.ts';
import { removeGitFolder } from './remove-git-folder.ts';
import { attachRendererConsoleCapture } from './renderer-console-capture.ts';
+import { createRendererRecovery, type RendererRecovery } from './renderer-recovery.ts';
import { resolveDetachedSpawnArgs } from './resolve-detached-spawn-args.ts';
import { resolveShareTarget as resolveShareTargetMain } from './resolve-share-target.ts';
import {
@@ -1083,6 +1084,14 @@ let mcpWiringHandle: RunMcpWiringHandle | null = null;
* boot paths, which never prompt.
*/
let crashDetection: CrashDetection | null = null;
+/**
+ * Renderer crash recovery — the window-facing half of `render-process-gone`
+ * (crashDetection owns the report-invitation half and stays window-blind).
+ * Created alongside crashDetection at the top of `bootPrimaryInstance`; null
+ * only in the duplicate-instance and driver-smoke boot paths, which never
+ * own a window long enough to recover one.
+ */
+let rendererRecovery: RendererRecovery | null = null;
/** Sentinel liveness heartbeat; cleared on `will-quit` with the other teardowns. */
let crashSentinelHeartbeat: NodeJS.Timeout | null = null;
@@ -6126,6 +6135,60 @@ function bootPrimaryInstance(): void {
logger: getLogger('crash-detection'),
});
crashDetection.detectBootCrash();
+ rendererRecovery = createRendererRecovery({
+ now: () => Date.now(),
+ logger: getLogger('renderer-recovery'),
+ defer: (fn) => {
+ setImmediate(fn);
+ },
+ // Async `showMessageBox`, never `showErrorBox` — the latter blocks the main
+ // process on macOS, which would wedge the very window being recovered (the
+ // same reason the boot unhandled-rejection path avoids it). The returned
+ // promise is load-bearing: recovery uses it to keep one dialog open per
+ // window rather than stacking a sheet per crash.
+ promptManualRecovery: (contents, info) => {
+ const log = getLogger('renderer-recovery');
+ // Safe despite the structural-subset type: every value reaching here came
+ // from `web-contents-created` and IS an Electron WebContents; the narrower
+ // type is the module's Electron-free boundary, not a different runtime shape.
+ const target = BrowserWindow.fromWebContents(contents as unknown as WebContents);
+ const options: MessageBoxOptions = {
+ type: 'warning',
+ title: 'This window stopped responding',
+ message: 'This window stopped responding',
+ detail:
+ 'OpenKnowledge reloaded it once and it stopped again. Your documents and any running agents live in the OpenKnowledge server rather than in this window, so reloading restores the view without interrupting them.',
+ buttons: ['Reload', 'Not Now'],
+ defaultId: 0,
+ cancelId: 1,
+ };
+ // Every reload here is guarded: the window can be closed between the
+ // check and the native call, and this runs inside a promise chain whose
+ // rejection would otherwise be indistinguishable from a dialog failure.
+ const reloadGuarded = (event: string) => {
+ if (contents.isDestroyed()) return;
+ try {
+ contents.reload();
+ } catch (err: unknown) {
+ log.warn({ event, reason: info.reason, err }, 'renderer reload threw past the guard');
+ }
+ };
+ return (target ? dialog.showMessageBox(target, options) : dialog.showMessageBox(options))
+ .then(({ response }) => {
+ if (response !== 0) return;
+ reloadGuarded('renderer-recovery.reload-after-confirm-failed');
+ })
+ .catch((err: unknown) => {
+ log.warn(
+ { event: 'renderer-recovery.prompt-failed', reason: info.reason, err },
+ 'renderer recovery prompt failed — falling back to a direct reload',
+ );
+ // The dialog was the last remaining affordance. If it could not be
+ // shown, reload rather than leaving the user staring at a blank window.
+ reloadGuarded('renderer-recovery.fallback-reload-failed');
+ });
+ },
+ });
// Keep the sentinel's liveness fresh and mirror power transitions into it,
// so the next boot can tell "the machine went down under a running app"
// (suppress the report prompt) from "the app died on its own" (prompt).
@@ -6171,6 +6234,13 @@ function bootPrimaryInstance(): void {
attachRendererConsoleCapture(contents);
contents.on('render-process-gone', (_e, details) => {
crashDetection?.handleRenderProcessGone(details);
+ // Detection arms a report invitation but is deliberately window-blind, so
+ // without this the window stays blank until the user discovers Cmd-R.
+ // Runs second so the invitation is armed against the pre-reload state.
+ rendererRecovery?.handleRenderProcessGone(contents, details);
+ });
+ contents.once('destroyed', () => {
+ rendererRecovery?.dispose(contents);
});
// A freshly-loaded renderer can take a waiting crash invitation (boot
// events detect before any window exists; delivery must not race load).
@@ -7139,6 +7209,11 @@ function bootPrimaryInstance(): void {
clearInterval(crashSentinelHeartbeat);
crashSentinelHeartbeat = null;
}
+ // Stop recovering windows once quit is under way. A renderer that dies
+ // abnormally mid-teardown (an OOM during cleanup, say) still reports a
+ // recoverable reason, and reloading it would spawn a fresh renderer for a
+ // window Electron is actively closing.
+ rendererRecovery = null;
// Reap every window's PTY host first so no user shell / spawn-helper
// outlives the app. Idempotent (clears the map; a second pass no-ops).
terminalReaper?.killAll();
diff --git a/packages/desktop/src/main/renderer-recovery.test.ts b/packages/desktop/src/main/renderer-recovery.test.ts
new file mode 100644
index 000000000..c771ac52f
--- /dev/null
+++ b/packages/desktop/src/main/renderer-recovery.test.ts
@@ -0,0 +1,495 @@
+import { describe, expect, test, vi } from 'vitest';
+import {
+ createRendererRecovery,
+ type RecoverableWebContents,
+ type RendererRecovery,
+ type RenderProcessGoneDetails,
+} from './renderer-recovery.ts';
+
+/**
+ * Renderer crash-recovery unit tests.
+ *
+ * Pure DI'd module — no Electron, no real timers, no real clock. The contract:
+ * a window whose renderer dies must not be left as an unexplained blank
+ * surface. One silent auto-reload restores the common transient case; a repeat
+ * crash inside the loop window, or exhausting the lifetime budget, stops
+ * reloading and hands the user an explicit choice, so a renderer that dies on
+ * every load can never thrash.
+ *
+ * `defer` is injected as a captured queue rather than run inline, so the tests
+ * can prove the reload never happens on the caller's stack — that deferral is
+ * load-bearing (see the reload comment in the module).
+ */
+
+interface MockContents extends RecoverableWebContents {
+ reload: ReturnType<typeof vi.fn>;
+ markDestroyed: () => void;
+}
+
+let nextContentsId = 1;
+
+function makeContents(): MockContents {
+ let destroyed = false;
+ return {
+ id: nextContentsId++,
+ reload: vi.fn(() => {}),
+ isDestroyed: () => destroyed,
+ markDestroyed: () => {
+ destroyed = true;
+ },
+ };
+}
+
+interface LogLine {
+ level: 'info' | 'warn';
+ obj: Record<string, unknown>;
+ msg: string;
+}
+
+interface CapturedPrompt {
+ contents: RecoverableWebContents;
+ info: Record<string, unknown>;
+}
+
+interface Rig {
+ recovery: RendererRecovery;
+ prompts: CapturedPrompt[];
+ logs: LogLine[];
+ advance: (ms: number) => void;
+ /** Run everything queued via the injected `defer`. */
+ drainDeferred: () => void;
+ /** Resolve every prompt currently awaiting an answer. */
+ settlePrompts: () => Promise<void>;
+}
+
+function makeRig(opts?: {
+ loopWindowMs?: number;
+ maxAutoReloads?: number;
+ maxLifetimeAutoReloads?: number;
+}): Rig {
+ let clockMs = Date.parse('2026-08-03T17:54:19.000Z');
+ const prompts: CapturedPrompt[] = [];
+ const logs: LogLine[] = [];
+ const pendingPrompts: Array<() => void> = [];
+ const deferred: Array<() => void> = [];
+ const record = (level: 'info' | 'warn') => (obj: Record<string, unknown>, msg: string) => {
+ logs.push({ level, obj, msg });
+ };
+ const recovery = createRendererRecovery({
+ now: () => clockMs,
+ logger: { info: record('info'), warn: record('warn') },
+ defer: (fn) => {
+ deferred.push(fn);
+ },
+ promptManualRecovery: (contents, info) => {
+ prompts.push({ contents, info: info as unknown as Record<string, unknown> });
+ return new Promise<void>((resolve) => {
+ pendingPrompts.push(resolve);
+ });
+ },
+ ...opts,
+ });
+ return {
+ recovery,
+ prompts,
+ logs,
+ advance: (ms: number) => {
+ clockMs += ms;
+ },
+ drainDeferred: () => {
+ for (const fn of deferred.splice(0)) fn();
+ },
+ settlePrompts: async () => {
+ for (const resolve of pendingPrompts.splice(0)) resolve();
+ // Let the module's `.then` bookkeeping run before the caller asserts.
+ await Promise.resolve();
+ await Promise.resolve();
+ },
+ };
+}
+
+const crashed: RenderProcessGoneDetails = { reason: 'crashed', exitCode: 5 };
+
+describe('renderer crash recovery', () => {
+ test('reloads a crashed renderer exactly once', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ expect(rig.prompts).toHaveLength(0);
+ });
+
+ test('the reload never runs on the caller stack', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+
+ // Electron emits `render-process-gone` from inside Chromium's process-death
+ // observer loop; reloading there CHECK-crashes the browser process.
+ expect(contents.reload).not.toHaveBeenCalled();
+
+ rig.drainDeferred();
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ });
+
+ test('a window closed during the deferral gap is not reloaded, and says so', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ contents.markDestroyed();
+ rig.drainDeferred();
+
+ expect(contents.reload).not.toHaveBeenCalled();
+ // The `reloading` line already claimed the reload as fact, so silence here
+ // would leave a bundle unable to tell it from a reload that happened.
+ expect(rig.logs.some((l) => l.obj.event === 'renderer-recovery.reload-abandoned')).toBe(true);
+ });
+
+ test.each([
+ ['oom' as const],
+ ['launch-failed' as const],
+ ['integrity-failure' as const],
+ ])('reloads on abnormal reason %s', (reason) => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, { reason, exitCode: 1 });
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ });
+
+ test.each([
+ ['clean-exit' as const],
+ ['killed' as const],
+ ['abnormal-exit' as const],
+ // Chromium reclaiming a backgrounded renderer on purpose. Reloading would
+ // re-spawn what it just freed and race the window's own lifecycle.
+ ['memory-eviction' as const],
+ ])('ignores non-crash exit reason %s, but logs it', (reason) => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, { reason, exitCode: 0 });
+ rig.drainDeferred();
+
+ expect(contents.reload).not.toHaveBeenCalled();
+ expect(rig.prompts).toHaveLength(0);
+ // Silence here would make the exclusion unfalsifiable from a bundle.
+ const ignored = rig.logs.find((l) => l.obj.event === 'renderer-recovery.ignored');
+ expect(ignored?.obj).toMatchObject({ reason, contentsId: contents.id });
+ // Ordinary teardown is noise; a reason that strands the user on a blank
+ // window should stand out in triage.
+ const routine = reason === 'clean-exit' || reason === 'killed';
+ expect(ignored?.level).toBe(routine ? 'info' : 'warn');
+ });
+
+ test('an async prompt rejection clears the flag and leaves a log line', async () => {
+ const logs: LogLine[] = [];
+ const record = (level: 'info' | 'warn') => (obj: Record<string, unknown>, msg: string) => {
+ logs.push({ level, obj, msg });
+ };
+ let clockMs = 0;
+ let calls = 0;
+ const recovery = createRendererRecovery({
+ now: () => clockMs,
+ logger: { info: record('info'), warn: record('warn') },
+ defer: (fn) => {
+ fn();
+ },
+ promptManualRecovery: () => {
+ calls += 1;
+ return Promise.reject(new Error('dialog subsystem unavailable'));
+ },
+ });
+ const contents = makeContents();
+
+ recovery.handleRenderProcessGone(contents, crashed);
+ clockMs += 1_000;
+ recovery.handleRenderProcessGone(contents, crashed);
+ await Promise.resolve();
+ await Promise.resolve();
+
+ expect(logs.some((l) => l.obj.event === 'renderer-recovery.prompt-failed')).toBe(true);
+
+ // The flag was cleared, so a later crash still reaches the prompt.
+ clockMs += 1_000;
+ recovery.handleRenderProcessGone(contents, crashed);
+ expect(calls).toBe(2);
+ });
+
+ test('a repeat crash inside the loop window prompts instead of reloading again', () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+
+ rig.advance(5_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ expect(rig.prompts.map((p) => p.contents)).toEqual([contents]);
+ });
+
+ test('a crash exactly at the loop-window boundary is still inside the window', () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(60_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ expect(rig.prompts).toHaveLength(1);
+ });
+
+ test('a crash after the loop window elapses is a fresh incident and reloads again', () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(60_001);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(2);
+ expect(rig.prompts).toHaveLength(0);
+ });
+
+ test('the lifetime cap stops a renderer that dies just outside the window forever', () => {
+ const rig = makeRig({ loopWindowMs: 60_000, maxLifetimeAutoReloads: 3 });
+ const contents = makeContents();
+
+ // Each crash lands in a brand-new loop window, so the per-window budget
+ // refreshes every time. Only the lifetime cap can stop this.
+ for (let i = 0; i < 3; i++) {
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(60_001);
+ }
+ rig.drainDeferred();
+ expect(contents.reload).toHaveBeenCalledTimes(3);
+ expect(rig.prompts).toHaveLength(0);
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(3);
+ expect(rig.prompts).toHaveLength(1);
+ expect(rig.logs.some((l) => l.obj.exhausted === 'lifetime')).toBe(true);
+ });
+
+ test('only one recovery prompt is open per webContents at a time', async () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ // The third crash must not stack a second dialog on the same window.
+ expect(rig.prompts).toHaveLength(1);
+ expect(rig.logs.some((l) => l.obj.event === 'renderer-recovery.prompt-suppressed')).toBe(true);
+
+ // Once the user answers, a later crash may prompt again.
+ await rig.settlePrompts();
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+
+ expect(rig.prompts).toHaveLength(2);
+ });
+
+ test('an open prompt suppresses auto-reload across a loop-window rollover', async () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+ expect(rig.prompts).toHaveLength(1);
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+
+ // The user is still reading the dialog when the loop window rolls over.
+ // The refreshed per-window budget must NOT reload behind the open prompt.
+ rig.advance(60_001);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(1);
+ expect(rig.prompts).toHaveLength(1);
+
+ // Answering must clear the flag on the state that is now live, not the one
+ // the rollover orphaned, or the affordance is lost forever.
+ await rig.settlePrompts();
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(2);
+ });
+
+ test('maxAutoReloads above 1 allows that many silent reloads before prompting', () => {
+ const rig = makeRig({ loopWindowMs: 60_000, maxAutoReloads: 2 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).toHaveBeenCalledTimes(2);
+ expect(rig.prompts).toHaveLength(1);
+ });
+
+ test('passes the crash context to the prompt', () => {
+ const rig = makeRig({ loopWindowMs: 60_000 });
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+ rig.advance(1_000);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+
+ expect(rig.prompts[0]?.info).toMatchObject({
+ reason: 'crashed',
+ exitCode: 5,
+ crashesInWindow: 2,
+ lifetimeAutoReloads: 1,
+ contentsId: contents.id,
+ });
+ });
+
+ test('never reloads a destroyed webContents', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+ contents.markDestroyed();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ expect(contents.reload).not.toHaveBeenCalled();
+ expect(rig.prompts).toHaveLength(0);
+ });
+
+ test('a reload that throws is contained, not propagated', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+ contents.reload.mockImplementation(() => {
+ throw new Error('Object has been destroyed');
+ });
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ expect(() => rig.drainDeferred()).not.toThrow();
+
+ expect(rig.logs.some((l) => l.obj.event === 'renderer-recovery.reload-failed')).toBe(true);
+ expect(rig.prompts).toHaveLength(0);
+ });
+
+ test('a prompt that throws synchronously is contained and clears the pending flag', () => {
+ let calls = 0;
+ const logs: LogLine[] = [];
+ const record = (level: 'info' | 'warn') => (obj: Record<string, unknown>, msg: string) => {
+ logs.push({ level, obj, msg });
+ };
+ let clockMs = 0;
+ const recovery = createRendererRecovery({
+ now: () => clockMs,
+ logger: { info: record('info'), warn: record('warn') },
+ defer: (fn) => {
+ fn();
+ },
+ promptManualRecovery: () => {
+ calls += 1;
+ throw new Error('dialog subsystem unavailable');
+ },
+ });
+ const contents = makeContents();
+
+ recovery.handleRenderProcessGone(contents, crashed);
+ clockMs += 1_000;
+ expect(() => recovery.handleRenderProcessGone(contents, crashed)).not.toThrow();
+ expect(logs.some((l) => l.obj.event === 'renderer-recovery.prompt-failed')).toBe(true);
+
+ // The flag was cleared, so a later crash still reaches the prompt.
+ clockMs += 1_000;
+ recovery.handleRenderProcessGone(contents, crashed);
+ expect(calls).toBe(2);
+ });
+
+ test('tracks each webContents independently', () => {
+ const rig = makeRig();
+ const first = makeContents();
+ const second = makeContents();
+
+ rig.recovery.handleRenderProcessGone(first, crashed);
+ rig.recovery.handleRenderProcessGone(first, crashed);
+ rig.recovery.handleRenderProcessGone(second, crashed);
+ rig.drainDeferred();
+
+ // `second` is on its first crash — it must still get its own auto-reload
+ // even though `first` has already saturated.
+ expect(first.reload).toHaveBeenCalledTimes(1);
+ expect(second.reload).toHaveBeenCalledTimes(1);
+ expect(rig.prompts.map((p) => p.contents)).toEqual([first]);
+ });
+
+ test('dispose releases per-contents state', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+ expect(rig.prompts).toHaveLength(1);
+
+ rig.recovery.dispose(contents);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ // A fresh registration starts over, so the auto-reload is available again.
+ expect(contents.reload).toHaveBeenCalledTimes(2);
+ expect(rig.prompts).toHaveLength(1);
+ });
+
+ test('logs the reload and the loop detection with the crash reason', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ const events = rig.logs.map((l) => l.obj.event);
+ expect(events).toContain('renderer-recovery.reloading');
+ expect(events).toContain('renderer-recovery.loop-detected');
+ for (const line of rig.logs) {
+ expect(line.obj.reason).toBe('crashed');
+ expect(line.obj.contentsId).toBe(contents.id);
+ }
+ });
+
+ test('the reloading log reports the post-increment counters', () => {
+ const rig = makeRig();
+ const contents = makeContents();
+
+ rig.recovery.handleRenderProcessGone(contents, crashed);
+ rig.drainDeferred();
+
+ const reloading = rig.logs.find((l) => l.obj.event === 'renderer-recovery.reloading');
+ expect(reloading?.obj).toMatchObject({ autoReloads: 1, lifetimeAutoReloads: 1 });
+ });
+});
diff --git a/packages/desktop/src/main/renderer-recovery.ts b/packages/desktop/src/main/renderer-recovery.ts
new file mode 100644
index 000000000..9afa18051
--- /dev/null
+++ b/packages/desktop/src/main/renderer-recovery.ts
@@ -0,0 +1,375 @@
+/**
+ * Renderer crash recovery.
+ *
+ * When Chromium tears down a renderer process, the BrowserWindow it backed
+ * keeps its frame but paints nothing. Electron does not re-create the renderer
+ * on its own, so without this module a dead renderer leaves a window that is
+ * blank forever and offers the user no way to understand or undo it — the only
+ * escape is knowing that Cmd-R happens to work.
+ *
+ * The policy is one silent auto-reload per loop window, bounded by a lifetime
+ * cap. A one-off renderer death is overwhelmingly transient, and reloading
+ * restores the window before the user has to think about it. A renderer that
+ * dies again inside the same window is a crash loop: reloading it a second time
+ * would thrash, so recovery stops and hands the user an explicit choice. The
+ * lifetime cap closes the gap the per-window budget alone leaves open — a
+ * renderer that dies just outside the window every time would otherwise earn a
+ * fresh budget forever and reload silently without end.
+ *
+ * Deliberately Electron-free and window-blind in the same spirit as
+ * `crash-detection.ts` — that module decides whether to invite a bug report and
+ * must not grow a dependency on window lifecycles. This one owns the opposite
+ * half (what happens to the window) and takes its Electron surfaces as injected
+ * deps, so the policy is unit-testable against fakes.
+ */
+
+/**
+ * Every reason Electron can report for `render-process-gone`, declared locally
+ * rather than imported so this module stays Electron-free. Spelling the union
+ * out (instead of widening to `string`) makes an Electron upgrade that adds a
+ * reason fail to compile at the wiring site, which forces a deliberate
+ * recoverable-or-not decision rather than letting a new reason silently fall
+ * through to "do nothing".
+ */
+type RenderProcessGoneReason =
+ | 'clean-exit'
+ | 'abnormal-exit'
+ | 'killed'
+ | 'crashed'
+ | 'oom'
+ | 'launch-failed'
+ | 'integrity-failure'
+ | 'memory-eviction';
+
+/**
+ * Structural subset of Electron's `WebContents`. Only the members recovery
+ * needs, so tests can fake it without an Electron import and so this never has
+ * to widen `BrowserWindowLike` (whose `webContents` shape is shared with every
+ * window-manager test fake).
+ */
+export interface RecoverableWebContents {
+ /** Electron's `WebContents.id`. Stamped on every log line so concurrent
+ * crashes in a multi-window session can be told apart. */
+ readonly id: number;
+ reload(): void;
+ isDestroyed(): boolean;
+}
+
+/** Structural subset of Electron's `RenderProcessGoneDetails`. */
+export interface RenderProcessGoneDetails {
+ reason: RenderProcessGoneReason;
+ exitCode?: number;
+}
+
+interface RecoveryLogger {
+ info(payload: Record<string, unknown>, msg: string): void;
+ warn(payload: Record<string, unknown>, msg: string): void;
+}
+
+export interface RendererRecoveryDeps {
+ /** Monotonic-enough wall clock in ms. Tests inject a controllable one. */
+ now: () => number;
+ logger: RecoveryLogger;
+ /**
+ * Run a callback on a later task. Production wires `setImmediate`; tests
+ * inject a captured queue so they can prove the reload does not happen on the
+ * caller's stack. Load-bearing, not a convenience — see the comment above the
+ * reload call for the browser-process crash this avoids.
+ */
+ defer: (fn: () => void) => void;
+ /**
+ * Surface the "this window stopped responding" choice once auto-reload is
+ * exhausted, and resolve when the user has answered.
+ *
+ * MUST be non-blocking: wire `dialog.showMessageBox` (async), never
+ * `dialog.showErrorBox`, which blocks the main process on macOS and would
+ * freeze the very window being recovered.
+ *
+ * The returned promise is what lets recovery keep at most one dialog open per
+ * webContents; resolving early would let a crash loop stack sheets.
+ */
+ promptManualRecovery: (contents: RecoverableWebContents, info: PromptInfo) => Promise<void>;
+ /**
+ * Crashes closer together than this count as one incident, so the auto-reload
+ * budget does not refresh. A crash further out is a fresh incident and earns
+ * a new budget. Default 60_000.
+ */
+ loopWindowMs?: number;
+ /** Automatic reloads allowed per loop window. Default 1. */
+ maxAutoReloads?: number;
+ /**
+ * Automatic reloads allowed for one webContents over its whole life,
+ * regardless of how widely spaced the crashes are. Default 5.
+ */
+ maxLifetimeAutoReloads?: number;
+}
+
+interface PromptInfo {
+ reason: RenderProcessGoneReason;
+ exitCode?: number;
+ /** Crashes seen for this webContents inside the current loop window. */
+ crashesInWindow: number;
+ /** Automatic reloads issued for this webContents since it was first seen. */
+ lifetimeAutoReloads: number;
+ /** Electron's `WebContents.id` for the window this prompt is about. */
+ contentsId: number;
+}
+
+export interface RendererRecovery {
+ /**
+ * Handle one `render-process-gone`. Reloads, prompts, or ignores per the
+ * policy above. Safe to call for any webContents — recovery state is created
+ * lazily on first crash. Never throws: a reload that fails is logged, because
+ * this runs inside an Electron event handler where an escaping exception
+ * would take down the main process.
+ */
+ handleRenderProcessGone(
+ contents: RecoverableWebContents,
+ details: RenderProcessGoneDetails,
+ ): void;
+ /**
+ * Release state for a webContents. Callers wire this to `destroyed` so a
+ * closed window cannot pin its entry.
+ *
+ * Takes the contents explicitly rather than returning a disposer closure the
+ * way `show-gate.ts` does, because there is no registration step to return
+ * one from: crash signals arrive on an emitter that already exists, and the
+ * first thing recovery learns about a webContents is that it has died.
+ */
+ dispose(contents: RecoverableWebContents): void;
+}
+
+/**
+ * Reasons that mean the renderer died abnormally and a reload is the right
+ * response. Mirrors `CRASH_REASONS` in `crash-detection.ts`.
+ *
+ * `killed` and `clean-exit` are ordinary teardown (including app quit), and
+ * reloading on those would fight the shutdown path.
+ *
+ * `memory-eviction` looks like it belongs here — the window goes just as blank
+ * — but it is excluded, on the policy argument alone: an eviction is Chromium
+ * deliberately reclaiming a backgrounded renderer, so reloading re-spawns the
+ * process it just freed.
+ *
+ * Whether an evicted renderer comes back on its own when its window is next
+ * shown is assumed, not verified. If that assumption is wrong the blank-window
+ * bug persists for evictions, which is why the ignored branch logs rather than
+ * returning silently.
+ *
+ * That log establishes how often these reasons occur, and nothing more. Whether
+ * anyone was actually stranded needs the window's subsequent lifecycle — a
+ * close, a manual reload, a next paint — and none of that is recorded anywhere
+ * today, so the outcome half stays open until renderer observability lands.
+ */
+const RECOVERABLE_REASONS = new Set<RenderProcessGoneReason>([
+ 'crashed',
+ 'oom',
+ 'launch-failed',
+ 'integrity-failure',
+]);
+
+/**
+ * Ignored reasons that are ordinary lifecycle rather than a window left blank.
+ * The rest of the ignored set (`abnormal-exit`, `memory-eviction`) does strand
+ * the user, so it logs at warn to surface in bundle triage.
+ */
+const ROUTINE_TEARDOWN_REASONS = new Set<RenderProcessGoneReason>(['clean-exit', 'killed']);
+
+const DEFAULT_LOOP_WINDOW_MS = 60_000;
+const DEFAULT_MAX_AUTO_RELOADS = 1;
+const DEFAULT_MAX_LIFETIME_AUTO_RELOADS = 5;
+
+interface PerContentsState {
+ /** Start of the current loop window. */
+ windowStartedAt: number;
+ /** Crashes seen since `windowStartedAt`. */
+ crashes: number;
+ /** Automatic reloads issued since `windowStartedAt`. */
+ autoReloads: number;
+ /** Automatic reloads issued since this webContents was first seen. Never reset. */
+ lifetimeAutoReloads: number;
+ /** A recovery dialog is open and unanswered for this webContents. */
+ promptPending: boolean;
+}
+
+export function createRendererRecovery(deps: RendererRecoveryDeps): RendererRecovery {
+ const states = new Map<RecoverableWebContents, PerContentsState>();
+ const loopWindowMs = deps.loopWindowMs ?? DEFAULT_LOOP_WINDOW_MS;
+ const maxAutoReloads = deps.maxAutoReloads ?? DEFAULT_MAX_AUTO_RELOADS;
+ const maxLifetimeAutoReloads = deps.maxLifetimeAutoReloads ?? DEFAULT_MAX_LIFETIME_AUTO_RELOADS;
+
+ return {
+ handleRenderProcessGone(contents, details) {
+ if (!RECOVERABLE_REASONS.has(details.reason)) {
+ // Logged rather than dropped silently: the excluded reasons rest on
+ // assumptions about what Chromium does next, and without a trace in the
+ // bundle there is no way to find out from the field whether they hold.
+ // Warn for the ones that leave the user looking at a blank window we
+ // chose not to act on; ordinary teardown stays at info.
+ const leavesBlankWindow = !ROUTINE_TEARDOWN_REASONS.has(details.reason);
+ const line = {
+ event: 'renderer-recovery.ignored',
+ reason: details.reason,
+ ...(details.exitCode !== undefined ? { exitCode: details.exitCode } : {}),
+ contentsId: contents.id,
+ };
+ const msg = 'renderer process gone for a reason recovery does not act on';
+ if (leavesBlankWindow) deps.logger.warn(line, msg);
+ else deps.logger.info(line, msg);
+ return;
+ }
+
+ // A window the user already closed has nothing to recover, and reloading
+ // destroyed contents throws. Drop the entry so it cannot pin the Map.
+ if (contents.isDestroyed()) {
+ states.delete(contents);
+ return;
+ }
+
+ const nowMs = deps.now();
+ const prior = states.get(contents);
+ // A crash further out than the loop window is a fresh incident, not a
+ // continuation, so it earns a new per-window budget. Without this a
+ // window that crashed once in the morning would never auto-recover
+ // again. `lifetimeAutoReloads` and `promptPending` deliberately survive
+ // the reset — they are what stop a renderer that dies just outside the
+ // window every time from reloading silently forever.
+ const state: PerContentsState =
+ prior === undefined || nowMs - prior.windowStartedAt > loopWindowMs
+ ? {
+ windowStartedAt: nowMs,
+ crashes: 0,
+ autoReloads: 0,
+ lifetimeAutoReloads: prior?.lifetimeAutoReloads ?? 0,
+ promptPending: prior?.promptPending ?? false,
+ }
+ : prior;
+ state.crashes += 1;
+ states.set(contents, state);
+
+ const crashFields = {
+ reason: details.reason,
+ ...(details.exitCode !== undefined ? { exitCode: details.exitCode } : {}),
+ crashesInWindow: state.crashes,
+ lifetimeAutoReloads: state.lifetimeAutoReloads,
+ contentsId: contents.id,
+ };
+
+ // Checked ahead of the budget gate, not inside it. While a dialog is open
+ // the user already holds the decision, so recovery must take no action at
+ // all — reloading behind an open prompt is both confusing and the thing
+ // the prompt exists to avoid. A loop-window rollover resets the per-window
+ // budget, so a check inside the gate would be skipped exactly when a
+ // long-open dialog spans the rollover.
+ if (state.promptPending) {
+ deps.logger.info(
+ { event: 'renderer-recovery.prompt-suppressed', ...crashFields },
+ 'recovery prompt is already open for this window',
+ );
+ return;
+ }
+
+ const windowBudgetSpent = state.autoReloads >= maxAutoReloads;
+ const lifetimeBudgetSpent = state.lifetimeAutoReloads >= maxLifetimeAutoReloads;
+
+ if (windowBudgetSpent || lifetimeBudgetSpent) {
+ deps.logger.warn(
+ {
+ event: 'renderer-recovery.loop-detected',
+ ...crashFields,
+ exhausted: lifetimeBudgetSpent ? 'lifetime' : 'window',
+ },
+ 'renderer died again after an automatic reload — asking the user instead of reloading',
+ );
+ state.promptPending = true;
+ // Reads the live entry instead of closing over `state`: a loop-window
+ // rollover while the dialog is open replaces the object in the Map, and
+ // clearing the orphan would leave the current entry pending forever,
+ // permanently suppressing the affordance for this window.
+ const clear = () => {
+ const current = states.get(contents);
+ if (current !== undefined) current.promptPending = false;
+ };
+ // Rejection clears the flag AND logs. The wiring catches its own dialog
+ // failures today, so this is a last resort, but a silent async path
+ // would leave a future wiring's failures invisible while the sync-throw
+ // path below is loud.
+ const clearAndLog = (err: unknown) => {
+ clear();
+ deps.logger.warn(
+ { event: 'renderer-recovery.prompt-failed', ...crashFields, err },
+ 'recovery prompt rejected',
+ );
+ };
+ try {
+ deps.promptManualRecovery(contents, crashFields).then(clear, clearAndLog);
+ } catch (err: unknown) {
+ clear();
+ deps.logger.warn(
+ { event: 'renderer-recovery.prompt-failed', ...crashFields, err },
+ 'recovery prompt threw synchronously',
+ );
+ }
+ return;
+ }
+
+ state.autoReloads += 1;
+ state.lifetimeAutoReloads += 1;
+ deps.logger.info(
+ {
+ event: 'renderer-recovery.reloading',
+ ...crashFields,
+ autoReloads: state.autoReloads,
+ lifetimeAutoReloads: state.lifetimeAutoReloads,
+ },
+ 'renderer died — reloading the window automatically',
+ );
+ // The reload MUST NOT run on the caller's stack. Electron emits
+ // `render-process-gone` synchronously from inside Chromium's
+ // process-death observer loop, and reloading from there re-enters
+ // `RenderProcessHostImpl::Init()` mid-iteration; the relaunched renderer
+ // then fails a CHECK and takes the whole browser process down. A
+ // try/catch cannot contain it — the CHECK fires a turn later, long after
+ // `reload()` has returned normally. Deferring by one task lets the
+ // notification unwind first. Fixed upstream in electron/electron#51900
+ // (backported to 41-x-y as #51917, after the 41.2.1 this app pins), so
+ // this stays until the pin moves past it.
+ deps.defer(() => {
+ // The whole body is guarded, not just the reload. This runs on a later
+ // task, so nothing above is on the stack to catch it and an escaping
+ // exception — including one from a misbehaving injected logger — would
+ // surface as an unhandled rejection in the main process.
+ try {
+ // Re-checked inside the deferred task: the deferral is exactly a
+ // window in which the user can close the window. Logged rather than
+ // returning silently — the `reloading` line above already claimed the
+ // reload as fact, so without a counter-record a bundle cannot tell a
+ // reload that happened from one abandoned here.
+ if (contents.isDestroyed()) {
+ deps.logger.info(
+ { event: 'renderer-recovery.reload-abandoned', ...crashFields },
+ 'window closed during the reload deferral — nothing left to reload',
+ );
+ return;
+ }
+ contents.reload();
+ } catch (err: unknown) {
+ // Deliberately terminal: this window keeps its blank frame and gets no
+ // prompt. Reaching here means `reload()` threw on contents that report
+ // themselves alive, which no known path produces; routing it into the
+ // prompt would add a branch through the pending-prompt bookkeeping for
+ // a case that cannot be exercised, and the user still has Cmd-R and
+ // the crash-report invitation. The log is the signal that the
+ // assumption broke.
+ deps.logger.warn(
+ { event: 'renderer-recovery.reload-failed', ...crashFields, err },
+ 'renderer reload threw past the destroyed guard — window stays blank',
+ );
+ }
+ });
+ },
+ dispose(contents) {
+ states.delete(contents);
+ },
+ };
+}