Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(desktop): outwait the backend's own shutdown deadline before tree-killing it
trohitg00b673fd1fb0
diff --git a/cli/tests/test_backend_shutdown.py b/cli/tests/test_backend_shutdown.pyindex 88f38bc6..41fee1fd 100644--- a/cli/tests/test_backend_shutdown.py+++ b/cli/tests/test_backend_shutdown.py@@ -28,6 +28,14 @@ VM_UNITS = (# Room after the CLI's allowance for the supervisor's wait after its own# tree-kill and for the processes to exit.UNIT_STOP_MARGIN_SECONDS = 10+# The desktop shell sets the backend's Temporal grace (env.ts) and waits+# STOP_TIMEOUT_MS (shutdown.ts) before its own tree-kill; the backend's own+# deadline is SHUTDOWN_DEADLINE_SECONDS (core/desktop.py).+DESKTOP_ENV = "desktop/src/main/env.ts"+DESKTOP_SHUTDOWN = "desktop/src/main/shutdown.ts"+DESKTOP_HOST = "server/core/desktop.py"+# Room after the backend's deadline for its own tree-kill and exit.+DESKTOP_STOP_MARGIN_SECONDS = 10@pytest.mark.parametrize("temporal_grace", [1, 30, 80])@@ -137,3 +145,28 @@ def test_vm_unit_outwaits_the_backend_shutdown_allowance(rel: str, monkeypatch:f"{rel}: TimeoutStopSec={timeouts[0]} does not outlast the CLI's backend shutdown "f"allowance of {allowance:.0f}s plus {UNIT_STOP_MARGIN_SECONDS}s")+++def _one_number(rel: str, pattern: str) -> float:+ found = re.findall(pattern, (project_root() / rel).read_text(encoding="utf-8"), re.MULTILINE)+ assert len(found) == 1, f"{rel}: expected one match for {pattern!r}, found {found}"+ return float(found[0].replace("_", ""))+++def test_desktop_shell_outwaits_the_backend_deadline(monkeypatch: pytest.MonkeyPatch):+ """The desktop backend must finish, or kill its own tree, before the shell kills it.++ Its deadline covers the CLI's allowance at the Temporal grace the shell+ sets, and the shell waits longer than that deadline. A tree-kill before+ the shutdown hooks run makes the next launch treat the quit as a crash+ and pause running deployments.+ """+ grace = _one_number(DESKTOP_ENV, r'^\s*TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS: "(\d+)",\s*$')+ monkeypatch.setenv("TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS", str(int(grace)))+ allowance = backend_shutdown_grace_seconds(replace(load_config(), temporal_enabled=True))+ deadline = _one_number(DESKTOP_HOST, r"^SHUTDOWN_DEADLINE_SECONDS = ([\d.]+)\s*$")+ shell_wait = _one_number(DESKTOP_SHUTDOWN, r"^export const STOP_TIMEOUT_MS = ([\d_]+);\s*$") / 1000+ assert deadline >= allowance, f"the backend's {deadline:.0f}s deadline is shorter than the {allowance:.0f}s allowance"+ assert shell_wait >= deadline + DESKTOP_STOP_MARGIN_SECONDS, (+ f"the shell waits {shell_wait:.0f}s, less than {DESKTOP_STOP_MARGIN_SECONDS}s past the backend's {deadline:.0f}s deadline"+ )diff --git a/desktop/src/main/backend.ts b/desktop/src/main/backend.tsindex a0f4c735..e2b771ee 100644--- a/desktop/src/main/backend.ts+++ b/desktop/src/main/backend.ts@@ -28,7 +28,7 @@ export interface SpawnBackendOptions {* before running the lifespan shutdown. Its default is "forever": when the* renderer's WebSocket died with a reset rather than a clean close, the* handler task lingered and uvicorn never reached the lifespan, so the- * shell's 30 s budget expired and the backend had to be tree-killed. Bound+ * shell's stop budget expired and the backend had to be tree-killed. Bound* it; the lifespan teardown (which reaps Temporal / the bun sidecar / edgymeow) is what* actually matters and still runs after this window.*/diff --git a/desktop/src/main/index.ts b/desktop/src/main/index.tsindex d4b0eafd..56855a48 100644--- a/desktop/src/main/index.ts+++ b/desktop/src/main/index.ts@@ -27,7 +27,7 @@ import { installMenu } from "./menu";import { type Layout, layoutReport, resolveLayout, targetKey, venvPython } from "./paths";import { choosePort } from "./ports";import { expectedStamp, needsProvision, runProvision } from "./provision";-import { stopBackend, treeKill } from "./shutdown";+import { STOP_TIMEOUT_MS, stopBackend, treeKill } from "./shutdown";import { loadState, saveState } from "./state";import { checkForUpdates, initUpdater } from "./updater";@@ -36,7 +36,6 @@ const DEFAULT_PORT = 5678;const SCAN_FROM = 5679;const SCAN_TO = 5699;const READY_TIMEOUT_MS = 180_000;-const STOP_TIMEOUT_MS = 30_000;// ---------------------------------------------------------------------------// Layoutdiff --git a/desktop/src/main/shutdown.ts b/desktop/src/main/shutdown.tsindex 90579f80..6e7d546b 100644--- a/desktop/src/main/shutdown.ts+++ b/desktop/src/main/shutdown.ts@@ -13,6 +13,15 @@import { spawn } from "node:child_process";+/**+ * How long the shell waits for the backend to exit before tree-killing it.+ * Longer than the backend's own deadline (`SHUTDOWN_DEADLINE_SECONDS`, 65 s,+ * in server/core/desktop.py), after which it kills its own process tree and+ * exits, so this fallback only fires for a backend that could not. Locked by+ * cli/tests/test_backend_shutdown.py.+ */+export const STOP_TIMEOUT_MS = 75_000;+export interface ChildLike {pid?: number;exitCode: number | null;@@ -82,7 +91,7 @@ export type StopOutcome = "already-exited" | "graceful" | "signal" | "tree-killeexport async function stopBackend(opts: StopOptions): Promise<StopOutcome> {const { child, port, token } = opts;- const timeoutMs = opts.timeoutMs ?? 30_000;+ const timeoutMs = opts.timeoutMs ?? STOP_TIMEOUT_MS;const platform = opts.platform ?? process.platform;const post = opts.post ?? postShutdown;const kill = opts.treeKill ?? treeKill;diff --git a/docs-internal/desktop_app.md b/docs-internal/desktop_app.mdindex 17c39530..adf1b0a2 100644--- a/docs-internal/desktop_app.md+++ b/docs-internal/desktop_app.md@@ -96,7 +96,8 @@ shell reads it directly (`stage/runtime/<os>-<arch>/`), in a packaged app8. `checkForUpdatesAndNotify`.Quit: `POST /api/desktop/shutdown` with the per-launch token, close stdin,-wait up to 30 s, then `taskkill /T` (Windows) or SIGKILL the process group.+wait up to 75 s (longer than the backend's own 65 s deadline; host contract+§5), then `taskkill /T` (Windows) or SIGKILL the process group.On the next launch a still-alive pid in `backend.pid` is tree-killed first.If the backend crashes it is restarted with 1 s / 3 s / 9 s backoff, at mostthree times in five minutes, then the setup window shows the error.diff --git a/docs-internal/desktop_host_contract.md b/docs-internal/desktop_host_contract.mdindex 18a0250e..8fd267c4 100644--- a/docs-internal/desktop_host_contract.md+++ b/docs-internal/desktop_host_contract.md@@ -156,15 +156,22 @@ Locked by `tests/test_desktop_shutdown_endpoint.py`.Also acceptable on POSIX: SIGTERM to the process. Do not `TerminateProcess`first on Windows; that skips the lifespan.-Shell should wait up to 30 s, then tree-kill as a fallback (Windows:-`taskkill /PID <pid> /T /F`; POSIX: SIGKILL to the process group, so spawn-the backend detached in its own group).--**Known gap:** nothing keeps the backend's teardown inside that 30 s. Plugin-hooks can take up to 10 s each, one after another, and the Temporal workers-get their own grace period (`TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10` under the-shell), so a slow teardown can outlast the wait and end in the tree-kill-instead of a clean exit.+The shell waits longer than the backend's own deadline (§6), then+tree-kills as a fallback (Windows: `taskkill /PID <pid> /T /F`; POSIX:+SIGKILL to the process group, so spawn the backend detached in its own+group). The Electron shell waits 75 s (`STOP_TIMEOUT_MS`,+`desktop/src/main/shutdown.ts`).++The backend's deadline, `SHUTDOWN_DEADLINE_SECONDS` (65 s), covers the CLI's+shutdown allowance (`backend_shutdown_grace_seconds`, `cli/_common.py`) at+the shell's `TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10`: 5 s of connection+drain, 10 s for each of the three Temporal stops (worker pool, manager+worker, dev server) and 30 s for the rest. A slow teardown therefore ends in+the backend's own deadline, not the shell's tree-kill. The order matters+because a kill before the plugin hooks run skips `mark_clean_shutdown`, and+the next launch then treats the quit as a crash and pauses running+deployments. `cli/tests/test_backend_shutdown.py` locks allowance <=+deadline < shell wait.## 6. Parent death@@ -176,8 +183,9 @@ Armed by `start_desktop_mode()` at lifespan start:stdin; EOF means the parent is gone, noticed in milliseconds. The shellmust keep the pipe open for the process lifetime and close it as part ofquitting.-- **Deadline**: `request_shutdown` arms a 45 s daemon thread that tree-kills- our children and `os._exit(1)`s if the graceful path wedges.+- **Deadline**: `request_shutdown` arms a daemon thread that, after+ `SHUTDOWN_DEADLINE_SECONDS` (65 s, see §5), tree-kills our children and+ `os._exit(1)`s if the graceful path wedges.- **Windows Job Object**: the backend enrolls itself in a kill-on-close job(ctypes, no pywin32). Children inherit membership, so if the backend isterminated by any means the kernel kills Temporal / the bun sidecar / edgymeow withdiff --git a/server/core/desktop.py b/server/core/desktop.pyindex 0ee0f206..909bee66 100644--- a/server/core/desktop.py+++ b/server/core/desktop.py@@ -62,11 +62,14 @@ ENV_STDIN_WATCHDOG = "OPENCOMPANY_DESKTOP_STDIN"PARENT_POLL_SECONDS = 2.0# Time the graceful lifespan teardown gets before the deadline thread-# tree-kills our children and exits. Temporal's own graceful window is-# ``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS`` (the shell sets 10 s), the Node-# sidecar 5 s, so 45 s is generous without letting a wedged teardown pin-# a zombie backend under a closed window forever.-SHUTDOWN_DEADLINE_SECONDS = 45.0+# tree-kills our children and exits. It covers the allowance the CLI gives+# the backend (``backend_shutdown_grace_seconds`` in ``cli/_common.py``) at+# the ``TEMPORAL_GRACEFUL_SHUTDOWN_SECONDS=10`` the shell sets: 5 s of+# connection drain, 10 s for each of the three Temporal stops (worker pool,+# manager worker, dev server) and 30 s for the rest. The shell waits longer+# before its own tree-kill, so the backend finishes or kills its own tree+# first; ``cli/tests/test_backend_shutdown.py`` locks that order.+SHUTDOWN_DEADLINE_SECONDS = 65.0_shutdown_requested = False_job_handle: Optional[int] = Nonediff --git a/server/services/plugin/shutdown_hooks.py b/server/services/plugin/shutdown_hooks.pyindex 3f73f76d..49158d3e 100644--- a/server/services/plugin/shutdown_hooks.py+++ b/server/services/plugin/shutdown_hooks.py@@ -46,11 +46,12 @@ ShutdownHook = Callable[[], Awaitable[None]]# lifespan teardown: under ``company serve`` the CLI's tree-kill (then 5 s# after SIGTERM) masked it, but a desktop shell waiting for a clean exit (so# Temporal / the JS executor sidecar / the WhatsApp bridge are reaped, not-# orphaned) saw the backend hang until the 45 s desktop deadline killed it.+# orphaned) saw the backend hang until the desktop deadline killed it.# Each hook now gets this long, then teardown moves on and names the offender-# at WARNING. Hooks run one after another, so the worst case grows with the-# number of hooks, while the desktop shell waits only 30 s before its own-# tree-kill (docs-internal/desktop_host_contract.md).+# at WARNING. Hooks run one after another, so several hooks that each hit+# this cap can outlast what the host allows the whole teardown (the desktop+# deadline in ``core/desktop.py``, the CLI's ``backend_shutdown_grace_seconds``),+# which then kills the process tree (docs-internal/desktop_host_contract.md).HOOK_TIMEOUT_SECONDS = 10.0diff --git a/server/tests/test_desktop_watchdog.py b/server/tests/test_desktop_watchdog.pyindex 850d230d..9457d810 100644--- a/server/tests/test_desktop_watchdog.py+++ b/server/tests/test_desktop_watchdog.py@@ -43,7 +43,7 @@ def desktop(monkeypatch):mod = _load_desktop()for var in (mod.ENV_FLAG, mod.ENV_PARENT_PID, mod.ENV_TOKEN, mod.ENV_STDIN_WATCHDOG):monkeypatch.delenv(var, raising=False)- # Never let a test really signal the pytest process or arm the 45 s exit.+ # Never let a test really signal the pytest process or arm the deadline exit.raised: list[int] = []monkeypatch.setattr(mod.signal, "raise_signal", lambda sig: raised.append(sig))monkeypatch.setattr(mod, "_deadline_thread", lambda reason: None)