Skip to content

Self-improving AI Employees for everyone

fix(cli): send CTRL_BREAK without os.kill, which crashed company stop

trohitga0e46f125d5b
diff --git a/cli/ports.py b/cli/ports.py
index 675182da..46c44b30 100644
--- a/cli/ports.py
+++ b/cli/ports.py
@@ -7,7 +7,6 @@ same battle-tested ``psutil`` paths that are already in production.
from __future__ import annotations
import os
-import signal
import sys
import subprocess
import time
@@ -16,6 +15,8 @@ from pathlib import Path
import psutil
+from cli.tree import send_ctrl_break
+
# Post-kill grace before re-checking the port. Windows can lag a few
# hundred ms releasing the listener socket after the bound process
@@ -107,37 +108,19 @@ def kill_pid(pid: int, *, graceful_timeout: float = 3.0) -> bool:
Windows: send ``CTRL_BREAK_EVENT`` first so daemons spawned with
``CREATE_NEW_PROCESS_GROUP`` (the supervisor's children — see
``cli/tree.py:new_session_kwargs``) get a real shutdown signal and
- can release listener sockets cleanly. ``proc.terminate()``
- (= ``TerminateProcess``) is the fallback for processes that weren't
- spawned with a process group — equivalent to SIGKILL, leaves the
- OS holding sockets briefly. Same pattern as
- ``cli/supervisor.py:_stop_proc``.
+ can release listener sockets cleanly. The event only reaches
+ processes attached to this console, so anything else (a service
+ started in another terminal, an orphan from a previous session) goes
+ straight to ``proc.terminate()`` (= ``TerminateProcess``) --
+ equivalent to SIGKILL, leaves the OS holding sockets briefly. The
+ event is sent by ``cli/tree.py:send_ctrl_break``, never ``os.kill``;
+ its docstring says why. Same pattern as ``cli/supervisor.py:_stop_proc``.
POSIX: plain ``proc.terminate()`` (SIGTERM).
"""
try:
proc = psutil.Process(pid)
- if sys.platform == "win32":
- try:
- os.kill(pid, signal.CTRL_BREAK_EVENT)
- except (OSError, ProcessLookupError, SystemError):
- # Fall back to TerminateProcess via psutil.terminate.
- # * ``OSError`` -- target wasn't in our process group.
- # * ``SystemError`` -- CPython issue #106148: on
- # Windows, ``os.kill`` returns success AND sets an
- # ``OSError`` when ``GenerateConsoleCtrlEvent``
- # fails with ERROR_INVALID_PARAMETER (e.g. the
- # target process isn't in the caller's console
- # group). CPython detects the inconsistency and
- # raises ``SystemError`` instead of propagating
- # the underlying OSError. ``kill_pid`` is used to
- # clean up stale state (port squatters, orphans
- # from a prior session) -- none of them are in
- # our console group, so this path fires every
- # time on Windows and used to crash
- # ``company stop``.
- proc.terminate()
- else:
+ if sys.platform != "win32" or not send_ctrl_break(pid):
proc.terminate()
try:
proc.wait(timeout=graceful_timeout)
diff --git a/cli/supervisor.py b/cli/supervisor.py
index 450599bd..cac6d43f 100644
--- a/cli/supervisor.py
+++ b/cli/supervisor.py
@@ -33,7 +33,7 @@ import anyio
from cli.colors import emit, next_color
from cli.run import which_argv
from cli.tcp import wait_for_tcp_port
-from cli.tree import add_to_job, kill_tree, new_session_kwargs
+from cli.tree import add_to_job, kill_tree, new_session_kwargs, send_ctrl_break
class RestartPolicy(enum.Enum):
@@ -198,7 +198,8 @@ class Manager:
POSIX: ``proc.terminate()`` sends ``SIGTERM`` to the child.
- Windows: ``os.kill(pid, CTRL_BREAK_EVENT)`` — the only Win32
+ Windows: ``CTRL_BREAK_EVENT`` via ``cli/tree.py:send_ctrl_break``
+ (never ``os.kill``; its docstring says why) — the only Win32
signal-equivalent that lets a child run cleanup handlers. We
spawn every child with ``CREATE_NEW_PROCESS_GROUP``
(``cli/tree.py:new_session_kwargs``) so ``CTRL_BREAK_EVENT`` is
@@ -223,7 +224,8 @@ class Manager:
with anyio.CancelScope(shield=True):
try:
if sys.platform == "win32":
- os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
+ if not send_ctrl_break(proc.pid):
+ return
else:
proc.terminate()
except (ProcessLookupError, OSError):
diff --git a/cli/tests/test_ports.py b/cli/tests/test_ports.py
index b381a9af..db0e7693 100644
--- a/cli/tests/test_ports.py
+++ b/cli/tests/test_ports.py
@@ -37,25 +37,31 @@ def test_windows_kill_pid_delivers_graceful_signal_before_waiting():
with (
patch.object(psutil, "Process", return_value=proc),
patch.object(ports.sys, "platform", "win32"),
- patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),
- patch.object(ports.os, "kill") as signal_process,
+ patch.object(ports, "send_ctrl_break", return_value=True) as send_break,
+ patch.object(ports.os, "kill") as os_kill,
):
assert ports.kill_pid(123, graceful_timeout=125) is True
- signal_process.assert_called_once_with(123, 1)
+ send_break.assert_called_once_with(123)
+ os_kill.assert_not_called()
proc.wait.assert_called_once_with(timeout=125)
proc.terminate.assert_not_called()
proc.kill.assert_not_called()
-def test_windows_kill_pid_falls_back_if_control_signal_unavailable():
+def test_windows_kill_pid_terminates_what_the_signal_cannot_reach():
+ """A service started in another terminal cannot get the event. Sending it
+ with ``os.kill`` crashed ``company stop``: on Python 3.12.8 the refused
+ event became a hard kill plus a pending ``OSError``, which surfaced as a
+ ``SystemError`` inside psutil's ``Process.wait``."""
proc = MagicMock()
with (
patch.object(psutil, "Process", return_value=proc),
patch.object(ports.sys, "platform", "win32"),
- patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),
- patch.object(ports.os, "kill", side_effect=OSError("no process group")),
+ patch.object(ports, "send_ctrl_break", return_value=False),
+ patch.object(ports.os, "kill") as os_kill,
):
assert ports.kill_pid(123) is True
+ os_kill.assert_not_called()
proc.terminate.assert_called_once()
proc.wait.assert_called_once_with(timeout=3.0)
diff --git a/cli/tests/test_supervisor.py b/cli/tests/test_supervisor.py
index 441d1fc4..1578376b 100644
--- a/cli/tests/test_supervisor.py
+++ b/cli/tests/test_supervisor.py
@@ -67,6 +67,47 @@ async def test_manager_runs_clean_exit_service():
assert rc == 0
+class _ExitsWhenWaited:
+ """Stands in for an ``anyio`` process that exits once it is waited on."""
+
+ pid = 4321
+
+ def __init__(self) -> None:
+ self.returncode = None
+ self.terminated = False
+
+ async def wait(self) -> int:
+ self.returncode = 0
+ return 0
+
+ def terminate(self) -> None:
+ self.terminated = True
+
+
[email protected]("sent", [True, False])
+async def test_windows_stop_sends_ctrl_break_without_os_kill(monkeypatch, sent):
+ """``os.kill(pid, CTRL_BREAK_EVENT)`` hard-kills the child and leaves an
+ ``OSError`` pending when Windows refuses the event (Python 3.12.8). When the
+ event does not go out, ``_stop_proc`` returns without waiting, as it did on
+ that ``OSError``."""
+ from cli import supervisor
+
+ def forbidden(*_args):
+ raise AssertionError("_stop_proc must not call os.kill")
+
+ sent_to = []
+ monkeypatch.setattr(supervisor.sys, "platform", "win32")
+ monkeypatch.setattr(supervisor.os, "kill", forbidden)
+ monkeypatch.setattr(supervisor, "send_ctrl_break", lambda pid: sent_to.append(pid) or sent)
+ monkeypatch.setattr(supervisor, "emit", lambda *_args, **_kwargs: None)
+ proc = _ExitsWhenWaited()
+ await Manager()._stop_proc(proc, ServiceSpec(name="svc", argv=["true"]))
+ assert sent_to == [proc.pid]
+ assert proc.returncode == (0 if sent else None)
+ assert not proc.terminated
+
+
@pytest.fixture
def anyio_backend():
return "asyncio"
diff --git a/cli/tests/test_tree.py b/cli/tests/test_tree.py
new file mode 100644
index 00000000..dabb1e33
--- /dev/null
+++ b/cli/tests/test_tree.py
@@ -0,0 +1,66 @@
+"""Tests for ``cli.tree``."""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+import time
+from unittest.mock import patch
+
+import pytest
+
+from cli import tree
+
+windows_only = pytest.mark.skipif(sys.platform != "win32", reason="console control events exist only on Windows")
+
+
+def _sleeper(creationflags: int) -> subprocess.Popen:
+ # The base interpreter: a venv launcher would add a process of its own.
+ return subprocess.Popen(
+ [sys._base_executable, "-c", "import time; time.sleep(30)"],
+ creationflags=creationflags,
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+
+
+def test_send_ctrl_break_is_windows_only():
+ with patch.object(tree.sys, "platform", "linux"):
+ assert tree.send_ctrl_break(123) is False
+
+
+@windows_only
+def test_send_ctrl_break_leaves_a_process_outside_this_console_alone():
+ """Windows cannot deliver the event outside this console. ``os.kill`` turned
+ that refusal into a hard kill and a pending ``OSError`` on Python 3.12.8."""
+ sleeper = _sleeper(subprocess.DETACHED_PROCESS)
+ try:
+ assert tree.send_ctrl_break(sleeper.pid) is False
+ with pytest.raises(subprocess.TimeoutExpired):
+ sleeper.wait(timeout=0.5)
+ finally:
+ sleeper.kill()
+ sleeper.wait(timeout=10)
+
+
+@windows_only
+def test_send_ctrl_break_stops_a_process_group_on_this_console():
+ pywintypes = pytest.importorskip("pywintypes")
+ win32console = pytest.importorskip("win32console")
+ try:
+ win32console.GetConsoleProcessList()
+ except pywintypes.error:
+ pytest.skip("this test process has no console")
+ sleeper = _sleeper(subprocess.CREATE_NEW_PROCESS_GROUP)
+ try:
+ deadline = time.monotonic() + 10
+ while sleeper.pid not in win32console.GetConsoleProcessList():
+ assert time.monotonic() < deadline, "the sleeper never attached to this console"
+ time.sleep(0.05)
+ assert tree.send_ctrl_break(sleeper.pid) is True
+ sleeper.wait(timeout=10) # long before its own 30 s sleep ends
+ finally:
+ if sleeper.poll() is None:
+ sleeper.kill()
+ sleeper.wait(timeout=10)
diff --git a/cli/tree.py b/cli/tree.py
index 762eadb6..04909471 100644
--- a/cli/tree.py
+++ b/cli/tree.py
@@ -70,6 +70,47 @@ def signal_group(pid: int, sig: signal.Signals = signal.SIGTERM) -> None:
pass
+def send_ctrl_break(pid: int) -> bool:
+ """Send ``CTRL_BREAK_EVENT`` to the process group ``pid`` leads (Windows only).
+
+ Returns whether the event went out: False on other platforms, when
+ ``pid`` is not attached to this process's console (Windows delivers
+ console events only within one console), and when Windows refuses it.
+
+ Goes through pywin32, never ``os.kill(pid, CTRL_BREAK_EVENT)``. Before
+ CPython 3.12.9 and 3.13.2 (gh-58689), an ``os.kill`` whose event
+ Windows refuses hard-kills the target with ``TerminateProcess`` and
+ returns with the ``OSError`` still set; the stale error surfaces later
+ as a ``SystemError`` from an unrelated call. ``company stop`` died that
+ way inside psutil's ``Process.wait``, because the services it stops run
+ in another terminal's console. Checking the console first also covers
+ the console hosts that accept an event for another console and deliver
+ nothing, which left callers waiting out their whole grace period.
+ https://learn.microsoft.com/en-us/windows/console/generateconsolectrlevent
+ """
+ if sys.platform != "win32":
+ return False
+ try:
+ import pywintypes # type: ignore[import-not-found]
+ import win32api # type: ignore[import-not-found]
+ import win32con # type: ignore[import-not-found]
+ import win32console # type: ignore[import-not-found]
+ except ImportError as e:
+ sys.stderr.write(
+ f"WARN: pywin32 import failed ({e}); stopping pid={pid} without "
+ f"CTRL_BREAK. Reinstall via: pip install --force-reinstall pywin32\n"
+ )
+ return False
+ try:
+ # Raises when this process has no console.
+ if pid not in win32console.GetConsoleProcessList():
+ return False
+ win32api.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, pid)
+ except pywintypes.error:
+ return False
+ return True
+
+
# ---------------------------------------------------------------- Windows Job Object
diff --git a/docs-internal/errors.md b/docs-internal/errors.md
index dc814f49..88740d9a 100644
--- a/docs-internal/errors.md
+++ b/docs-internal/errors.md
@@ -594,3 +594,23 @@ Seen 2026-09-05 with a `tikhubAction` tool whose results were about 400,000 char
Locked by `server/tests/services/test_tool_output.py`, `server/tests/temporal/test_agent_context_pressure.py`, `server/tests/temporal/test_agent_workflow_pressure.py` and `server/tests/temporal/test_prepare_payload_pressure.py`. Design record: [ARCHIVE/AGENT_COMPACTION_FIX_PLAN.md](./ARCHIVE/AGENT_COMPACTION_FIX_PLAN.md).
**Recovery for a row saved before the fix**: clear the conversation once from the Context panel (or Reset the deployment). New transcripts stay well under the cap.
+
+## 29. `company stop` crashes on Windows with `SystemError: <class 'int'> returned a result with an exception set`
+
+**Symptom**: `company stop` (or `bun run stop`) frees the ports and kills Temporal, then dies while reaping orphaned processes. The traceback ends inside psutil's `Process.wait`:
+
+```
+OSError: [WinError 87] The parameter is incorrect
+
+The above exception was the direct cause of the following exception:
+...
+SystemError: <class 'int'> returned a result with an exception set
+```
+
+The `OSError` has no traceback of its own. Orphans the reaper had not reached yet stay alive, and the command exits 1. The CLI supervisor's shutdown and the backend's `terminate_then_kill` sent the event the same way and could fail the same way.
+
+**Root cause**: `kill_pid` sent `CTRL_BREAK_EVENT` with `os.kill`. Windows delivers console events only to processes attached to the caller's console, and the services `company stop` reaps run in another terminal's console, so Windows refuses the event (error 87 in the reported case, error 6 when the caller has no console). Before CPython 3.12.9 and 3.13.2 ([gh-58689](https://github.com/python/cpython/issues/58689)), `os.kill` does not raise on that refusal: it records the error, falls through to `TerminateProcess` (hard-killing the target), and returns success with the error still set. CPython reports that inconsistency only on its generic call path. Once the `os.kill` call site has been specialized after its first run, release builds skip the check, and the stale error surfaces at the next call that does check: the `int()` inside psutil's `Process.wait`. So the `except SystemError` around `os.kill` covered only the first orphan. The CLI and server venvs here run 3.12.8.
+
+**Fix**: nothing sends console events through `os.kill` any more. `cli/tree.py::send_ctrl_break` (through pywin32) and its server copy in `services/_supervisor/util.py` (through `ctypes`, since the server has no pywin32) call `GenerateConsoleCtrlEvent` only for a process attached to the caller's console, and return whether the event went out. `kill_pid` terminates everything else at once. Before, some console hosts accepted an event for another console, delivered nothing, and `kill_pid` waited out its grace period. `_stop_proc` and `terminate_then_kill` treat an event that did not go out as they treated `os.kill`'s `OSError`. Locked by `cli/tests/test_tree.py`, `cli/tests/test_ports.py`, `cli/tests/test_supervisor.py` and `server/tests/services/test_supervisor.py`; their Windows-only tests send real events to processes they start.
+
+To reproduce the old failure on 3.12.8, run `kill_orphaned_opencompany_processes` from a process with no console against sleepers it started itself, with a root their command lines name: it raises the `SystemError`.
diff --git a/server/services/_supervisor/util.py b/server/services/_supervisor/util.py
index 8b245171..4d88d0f0 100644
--- a/server/services/_supervisor/util.py
+++ b/server/services/_supervisor/util.py
@@ -10,7 +10,6 @@ from __future__ import annotations
import asyncio
import logging
-import os
import signal
import sys
from typing import Callable, Optional
@@ -45,6 +44,48 @@ def kill_tree(pid: int) -> None:
pass
+def _console_process_ids() -> set[int]:
+ """PIDs attached to this process's console; empty when it has none (Windows only)."""
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel32.GetConsoleProcessList.restype = wintypes.DWORD
+ kernel32.GetConsoleProcessList.argtypes = [wintypes.LPDWORD, wintypes.DWORD]
+ ids = (wintypes.DWORD * 1)()
+ while True:
+ # 0 without a console; a count larger than the buffer is the size it needs.
+ count = kernel32.GetConsoleProcessList(ids, len(ids))
+ if count <= len(ids):
+ return set(ids[:count])
+ ids = (wintypes.DWORD * count)()
+
+
+def send_ctrl_break(pid: int) -> bool:
+ """Send ``CTRL_BREAK_EVENT`` to the process group ``pid`` leads (Windows only).
+
+ Returns whether the event went out: False on other platforms, when
+ ``pid`` is not attached to this process's console (Windows delivers
+ console events only within one console), and when Windows refuses it.
+ Calls ``GenerateConsoleCtrlEvent`` rather than ``os.kill``, which before
+ CPython 3.12.9 and 3.13.2 (gh-58689) turns a refused event into a
+ ``TerminateProcess`` hard kill and returns with the ``OSError`` still
+ set, so a ``SystemError`` surfaces later from an unrelated call. The CLI
+ has the same helper in ``cli/tree.py``; the server cannot import the CLI
+ package and has no pywin32.
+ https://learn.microsoft.com/en-us/windows/console/generateconsolectrlevent
+ """
+ if sys.platform != "win32" or pid not in _console_process_ids():
+ return False
+ import ctypes
+ from ctypes import wintypes
+
+ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
+ kernel32.GenerateConsoleCtrlEvent.restype = wintypes.BOOL
+ kernel32.GenerateConsoleCtrlEvent.argtypes = [wintypes.DWORD, wintypes.DWORD]
+ return bool(kernel32.GenerateConsoleCtrlEvent(signal.CTRL_BREAK_EVENT, pid))
+
+
async def terminate_then_kill(
proc: anyio.abc.Process,
*,
@@ -56,17 +97,15 @@ async def terminate_then_kill(
POSIX path: ``proc.terminate()`` (SIGTERM), wait, ``kill_tree()``.
Windows graceful path (``use_ctrl_break=True``, requires the process
to have been spawned with ``CREATE_NEW_PROCESS_GROUP``):
- ``os.kill(pid, CTRL_BREAK_EVENT)`` — the only way to send a real
- SIGINT-equivalent on Windows.
+ ``CTRL_BREAK_EVENT`` via :func:`send_ctrl_break` — the only way to send
+ a real SIGINT-equivalent on Windows.
"""
if proc.returncode is not None:
return
if use_ctrl_break and sys.platform == "win32":
- try:
- os.kill(proc.pid, signal.CTRL_BREAK_EVENT)
- except (ProcessLookupError, OSError):
- pass
+ # When the event does not go out, the grace wait ends in the tree-kill below.
+ send_ctrl_break(proc.pid)
else:
try:
proc.terminate()
diff --git a/server/tests/services/test_supervisor.py b/server/tests/services/test_supervisor.py
index 64cae745..867877f9 100644
--- a/server/tests/services/test_supervisor.py
+++ b/server/tests/services/test_supervisor.py
@@ -3,7 +3,10 @@
from __future__ import annotations
import asyncio
+import os
+import subprocess
import sys
+import time
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock, patch
@@ -25,7 +28,13 @@ from services._supervisor import ( # noqa: E402
shutdown_all_supervisors,
)
from services._supervisor.registry import _SUPERVISORS # noqa: E402
-from services._supervisor.util import drain_stream, kill_tree, terminate_then_kill # noqa: E402
+from services._supervisor.util import ( # noqa: E402
+ _console_process_ids,
+ drain_stream,
+ kill_tree,
+ send_ctrl_break,
+ terminate_then_kill,
+)
# --- helpers ---------------------------------------------------------------
@@ -237,6 +246,79 @@ async def test_terminate_then_kill_force_kills_after_grace():
kill_tree_mock.assert_called_once_with(42)
+async def test_terminate_then_kill_sends_ctrl_break_without_os_kill():
+ """``os.kill(pid, CTRL_BREAK_EVENT)`` hard-kills the child and leaves an
+ ``OSError`` pending when Windows refuses the event (Python 3.12.8, gh-58689)."""
+ proc = MagicMock()
+ proc.returncode = None
+ proc.pid = 42
+
+ async def exit_after_the_break():
+ proc.returncode = 0
+
+ proc.wait = AsyncMock(side_effect=exit_after_the_break)
+ with (
+ patch("services._supervisor.util.sys.platform", "win32"),
+ patch("services._supervisor.util.send_ctrl_break", return_value=True) as send_break,
+ patch("services._supervisor.util.kill_tree") as kill_tree_mock,
+ patch("os.kill", side_effect=AssertionError("terminate_then_kill must not call os.kill")),
+ ):
+ await terminate_then_kill(proc, grace=1.0, use_ctrl_break=True)
+
+ send_break.assert_called_once_with(42)
+ proc.terminate.assert_not_called()
+ kill_tree_mock.assert_not_called()
+
+
+# --- util.send_ctrl_break (real Win32 calls) --------------------------------
+
+windows_only = pytest.mark.skipif(sys.platform != "win32", reason="console control events exist only on Windows")
+
+
+def _sleeper(creationflags: int) -> subprocess.Popen:
+ # The base interpreter: a venv launcher would add a process of its own.
+ return subprocess.Popen(
+ [sys._base_executable, "-c", "import time; time.sleep(30)"],
+ creationflags=creationflags,
+ stdin=subprocess.DEVNULL,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+
+
+@windows_only
+def test_send_ctrl_break_leaves_a_process_outside_this_console_alone():
+ sleeper = _sleeper(subprocess.DETACHED_PROCESS)
+ try:
+ assert send_ctrl_break(sleeper.pid) is False
+ with pytest.raises(subprocess.TimeoutExpired):
+ sleeper.wait(timeout=0.5)
+ finally:
+ sleeper.kill()
+ sleeper.wait(timeout=10)
+
+
+@windows_only
+def test_send_ctrl_break_stops_a_process_group_on_this_console():
+ attached = _console_process_ids()
+ if not attached:
+ pytest.skip("this test process has no console")
+ assert os.getpid() in attached
+ sleeper = _sleeper(subprocess.CREATE_NEW_PROCESS_GROUP)
+ try:
+ deadline = time.monotonic() + 10
+ while sleeper.pid not in _console_process_ids():
+ assert time.monotonic() < deadline, "the sleeper never attached to this console"
+ time.sleep(0.05)
+ assert send_ctrl_break(sleeper.pid) is True
+ sleeper.wait(timeout=10) # long before its own 30 s sleep ends
+ finally:
+ if sleeper.poll() is None:
+ sleeper.kill()
+ sleeper.wait(timeout=10)
+
+
# --- util.drain_stream -----------------------------------------------------