Skip to content

Self-improving AI Employees for everyone

fix(desktop): outwait the backend's own shutdown deadline before tree-killing it

trohitg00b673fd1fb0
diff --git a/cli/tests/test_backend_shutdown.py b/cli/tests/test_backend_shutdown.py
index 88f38bc6..41fee1fd 100644
--- a/cli/tests/test_backend_shutdown.py
+++ b/cli/tests/test_backend_shutdown.py
@@ -28,6 +28,14 @@ VM_UNITS = (
# Room after the CLI's allowance for the supervisor's wait after its own
# tree-kill and for the processes to exit.
UNIT_STOP_MARGIN_SECONDS = 10
+# The desktop shell sets the backend's Temporal grace (env.ts) and waits
+# STOP_TIMEOUT_MS (shutdown.ts) before its own tree-kill; the backend's own
+# deadline is SHUTDOWN_DEADLINE_SECONDS (core/desktop.py).
+DESKTOP_ENV = "desktop/src/main/env.ts"
+DESKTOP_SHUTDOWN = "desktop/src/main/shutdown.ts"
+DESKTOP_HOST = "server/core/desktop.py"
+# Room after the backend's deadline for its own tree-kill and exit.
+DESKTOP_STOP_MARGIN_SECONDS = 10
@pytest.mark.parametrize("temporal_grace", [1, 30, 80])
@@ -137,3 +145,28 @@ def test_vm_unit_outwaits_the_backend_shutdown_allowance(rel: str, monkeypatch:
f"{rel}: TimeoutStopSec={timeouts[0]} does not outlast the CLI's backend shutdown "
f"allowance of {allowance:.0f}s plus {UNIT_STOP_MARGIN_SECONDS}s"
)
+
+
+def _one_number(rel: str, pattern: str) -> float:
+ found = re.findall(pattern, (project_root() / rel).read_text(encoding="utf-8"), re.MULTILINE)
+ assert len(found) == 1, f"{rel}: expected one match for {pattern!r}, found {found}"
+ return float(found[0].replace("_", ""))
+
+
+def test_desktop_shell_outwaits_the_backend_deadline(monkeypatch: pytest.MonkeyPatch):
+ """The desktop backend must finish, or kill its own tree, before the shell kills it.
+
+ Its deadline covers the CLI's allowance at the Temporal grace the shell
+ sets, and the shell waits longer than that deadline. A tree-kill before
+ the shutdown hooks run makes the next launch treat the quit as a crash
+ and pause running deployments.
+ """
+ grace = _one_number(DESKTOP_ENV, r'^\s*TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS: "(\d+)",\s*$')
+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", str(int(grace)))
+ allowance = backend_shutdown_grace_seconds(replace(load_config(), temporal_enabled=True))
+ deadline = _one_number(DESKTOP_HOST, r"^SHUTDOWN_DEADLINE_SECONDS = ([\d.]+)\s*$")
+ shell_wait = _one_number(DESKTOP_SHUTDOWN, r"^export const STOP_TIMEOUT_MS = ([\d_]+);\s*$") / 1000
+ assert deadline >= allowance, f"the backend's {deadline:.0f}s deadline is shorter than the {allowance:.0f}s allowance"
+ assert shell_wait >= deadline + DESKTOP_STOP_MARGIN_SECONDS, (
+ f"the shell waits {shell_wait:.0f}s, less than {DESKTOP_STOP_MARGIN_SECONDS}s past the backend's {deadline:.0f}s deadline"
+ )
diff --git a/desktop/src/main/backend.ts b/desktop/src/main/backend.ts
index a0f4c735..e2b771ee 100644
--- a/desktop/src/main/backend.ts
+++ b/desktop/src/main/backend.ts
@@ -28,7 +28,7 @@ export interface SpawnBackendOptions {
* before running the lifespan shutdown. Its default is "forever": when the
* renderer's WebSocket died with a reset rather than a clean close, the
* handler task lingered and uvicorn never reached the lifespan, so the
- * shell's 30 s budget expired and the backend had to be tree-killed. Bound
+ * shell's stop budget expired and the backend had to be tree-killed. Bound
* it; the lifespan teardown (which reaps Temporal / the bun sidecar / edgymeow) is what
* actually matters and still runs after this window.
*/
diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.ts
index d4b0eafd..56855a48 100644
--- a/desktop/src/main/index.ts
+++ b/desktop/src/main/index.ts
@@ -27,7 +27,7 @@ import { installMenu } from "./menu";
import { type Layout, layoutReport, resolveLayout, targetKey, venvPython } from "./paths";
import { choosePort } from "./ports";
import { expectedStamp, needsProvision, runProvision } from "./provision";
-import { stopBackend, treeKill } from "./shutdown";
+import { STOP_TIMEOUT_MS, stopBackend, treeKill } from "./shutdown";
import { loadState, saveState } from "./state";
import { checkForUpdates, initUpdater } from "./updater";
@@ -36,7 +36,6 @@ const DEFAULT_PORT = 5678;
const SCAN_FROM = 5679;
const SCAN_TO = 5699;
const READY_TIMEOUT_MS = 180_000;
-const STOP_TIMEOUT_MS = 30_000;
// ---------------------------------------------------------------------------
// Layout
diff --git a/desktop/src/main/shutdown.ts b/desktop/src/main/shutdown.ts
index 90579f80..6e7d546b 100644
--- a/desktop/src/main/shutdown.ts
+++ b/desktop/src/main/shutdown.ts
@@ -13,6 +13,15 @@
import { spawn } from "node:child_process";
+/**
+ * How long the shell waits for the backend to exit before tree-killing it.
+ * Longer than the backend's own deadline (`SHUTDOWN_DEADLINE_SECONDS`, 65 s,
+ * in server/core/desktop.py), after which it kills its own process tree and
+ * exits, so this fallback only fires for a backend that could not. Locked by
+ * cli/tests/test_backend_shutdown.py.
+ */
+export const STOP_TIMEOUT_MS = 75_000;
+
export interface ChildLike {
pid?: number;
exitCode: number | null;
@@ -82,7 +91,7 @@ export type StopOutcome = "already-exited" | "graceful" | "signal" | "tree-kille
export async function stopBackend(opts: StopOptions): Promise<StopOutcome> {
const { child, port, token } = opts;
- const timeoutMs = opts.timeoutMs ?? 30_000;
+ const timeoutMs = opts.timeoutMs ?? STOP_TIMEOUT_MS;
const platform = opts.platform ?? process.platform;
const post = opts.post ?? postShutdown;
const kill = opts.treeKill ?? treeKill;
diff --git a/docs-internal/desktop_app.md b/docs-internal/desktop_app.md
index 17c39530..adf1b0a2 100644
--- a/docs-internal/desktop_app.md
+++ b/docs-internal/desktop_app.md
@@ -96,7 +96,8 @@ shell reads it directly (`stage/runtime/<os>-<arch>/`), in a packaged app
8. `checkForUpdatesAndNotify`.
Quit: `POST /api/desktop/shutdown` with the per-launch token, close stdin,
-wait up to 30 s, then `taskkill /T` (Windows) or SIGKILL the process group.
+wait up to 75 s (longer than the backend's own 65 s deadline; host contract
+§5), then `taskkill /T` (Windows) or SIGKILL the process group.
On the next launch a still-alive pid in `backend.pid` is tree-killed first.
If the backend crashes it is restarted with 1 s / 3 s / 9 s backoff, at most
three times in five minutes, then the setup window shows the error.
diff --git a/docs-internal/desktop_host_contract.md b/docs-internal/desktop_host_contract.md
index 18a0250e..8fd267c4 100644
--- a/docs-internal/desktop_host_contract.md
+++ b/docs-internal/desktop_host_contract.md
@@ -156,15 +156,22 @@ Locked by `tests/test_desktop_shutdown_endpoint.py`.
Also acceptable on POSIX: SIGTERM to the process. Do not `TerminateProcess`
first on Windows; that skips the lifespan.
-Shell should wait up to 30 s, then tree-kill as a fallback (Windows:
-`taskkill /PID <pid> /T /F`; POSIX: SIGKILL to the process group, so spawn
-the backend detached in its own group).
-
-**Known gap:** nothing keeps the backend's teardown inside that 30 s. Plugin
-hooks can take up to 10 s each, one after another, and the Temporal workers
-get their own grace period (`TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10` under the
-shell), so a slow teardown can outlast the wait and end in the tree-kill
-instead of a clean exit.
+The shell waits longer than the backend's own deadline (§6), then
+tree-kills as a fallback (Windows: `taskkill /PID <pid> /T /F`; POSIX:
+SIGKILL to the process group, so spawn the backend detached in its own
+group). The Electron shell waits 75 s (`STOP_TIMEOUT_MS`,
+`desktop/src/main/shutdown.ts`).
+
+The backend's deadline, `SHUTDOWN_DEADLINE_SECONDS` (65 s), covers the CLI's
+shutdown allowance (`backend_shutdown_grace_seconds`, `cli/_common.py`) at
+the shell's `TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10`: 5 s of connection
+drain, 10 s for each of the three Temporal stops (worker pool, manager
+worker, dev server) and 30 s for the rest. A slow teardown therefore ends in
+the backend's own deadline, not the shell's tree-kill. The order matters
+because a kill before the plugin hooks run skips `mark_clean_shutdown`, and
+the next launch then treats the quit as a crash and pauses running
+deployments. `cli/tests/test_backend_shutdown.py` locks allowance <=
+deadline < shell wait.
## 6. Parent death
@@ -176,8 +183,9 @@ Armed by `start_desktop_mode()` at lifespan start:
stdin; EOF means the parent is gone, noticed in milliseconds. The shell
must keep the pipe open for the process lifetime and close it as part of
quitting.
-- **Deadline**: `request_shutdown` arms a 45 s daemon thread that tree-kills
- our children and `os._exit(1)`s if the graceful path wedges.
+- **Deadline**: `request_shutdown` arms a daemon thread that, after
+ `SHUTDOWN_DEADLINE_SECONDS` (65 s, see §5), tree-kills our children and
+ `os._exit(1)`s if the graceful path wedges.
- **Windows Job Object**: the backend enrolls itself in a kill-on-close job
(ctypes, no pywin32). Children inherit membership, so if the backend is
terminated by any means the kernel kills Temporal / the bun sidecar / edgymeow with
diff --git a/server/core/desktop.py b/server/core/desktop.py
index 0ee0f206..909bee66 100644
--- a/server/core/desktop.py
+++ b/server/core/desktop.py
@@ -62,11 +62,14 @@ ENV_STDIN_WATCHDOG = "OPENCOMPANY_DESKTOP_STDIN"
PARENT_POLL_SECONDS = 2.0
# Time the graceful lifespan teardown gets before the deadline thread
-# tree-kills our children and exits. Temporal's own graceful window is
-# ``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS`` (the shell sets 10 s), the Node
-# sidecar 5 s, so 45 s is generous without letting a wedged teardown pin
-# a zombie backend under a closed window forever.
-SHUTDOWN_DEADLINE_SECONDS = 45.0
+# tree-kills our children and exits. It covers the allowance the CLI gives
+# the backend (``backend_shutdown_grace_seconds`` in ``cli/_common.py``) at
+# the ``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10`` the shell sets: 5 s of
+# connection drain, 10 s for each of the three Temporal stops (worker pool,
+# manager worker, dev server) and 30 s for the rest. The shell waits longer
+# before its own tree-kill, so the backend finishes or kills its own tree
+# first; ``cli/tests/test_backend_shutdown.py`` locks that order.
+SHUTDOWN_DEADLINE_SECONDS = 65.0
_shutdown_requested = False
_job_handle: Optional[int] = None
diff --git a/server/services/plugin/shutdown_hooks.py b/server/services/plugin/shutdown_hooks.py
index 3f73f76d..49158d3e 100644
--- a/server/services/plugin/shutdown_hooks.py
+++ b/server/services/plugin/shutdown_hooks.py
@@ -46,11 +46,12 @@ ShutdownHook = Callable[[], Awaitable[None]]
# lifespan teardown: under ``company serve`` the CLI's tree-kill (then 5 s
# after SIGTERM) masked it, but a desktop shell waiting for a clean exit (so
# Temporal / the JS executor sidecar / the WhatsApp bridge are reaped, not
-# orphaned) saw the backend hang until the 45 s desktop deadline killed it.
+# orphaned) saw the backend hang until the desktop deadline killed it.
# Each hook now gets this long, then teardown moves on and names the offender
-# at WARNING. Hooks run one after another, so the worst case grows with the
-# number of hooks, while the desktop shell waits only 30 s before its own
-# tree-kill (docs-internal/desktop_host_contract.md).
+# at WARNING. Hooks run one after another, so several hooks that each hit
+# this cap can outlast what the host allows the whole teardown (the desktop
+# deadline in ``core/desktop.py``, the CLI's ``backend_shutdown_grace_seconds``),
+# which then kills the process tree (docs-internal/desktop_host_contract.md).
HOOK_TIMEOUT_SECONDS = 10.0
diff --git a/server/tests/test_desktop_watchdog.py b/server/tests/test_desktop_watchdog.py
index 850d230d..9457d810 100644
--- a/server/tests/test_desktop_watchdog.py
+++ b/server/tests/test_desktop_watchdog.py
@@ -43,7 +43,7 @@ def desktop(monkeypatch):
mod = _load_desktop()
for var in (mod.ENV_FLAG, mod.ENV_PARENT_PID, mod.ENV_TOKEN, mod.ENV_STDIN_WATCHDOG):
monkeypatch.delenv(var, raising=False)
- # Never let a test really signal the pytest process or arm the 45 s exit.
+ # Never let a test really signal the pytest process or arm the deadline exit.
raised: list[int] = []
monkeypatch.setattr(mod.signal, "raise_signal", lambda sig: raised.append(sig))
monkeypatch.setattr(mod, "_deadline_thread", lambda reason: None)