Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(cli): send CTRL_BREAK without os.kill, which crashed company stop
trohitga0e46f125d5b
diff --git a/cli/ports.py b/cli/ports.pyindex 675182da..46c44b30 100644--- a/cli/ports.py+++ b/cli/ports.py@@ -7,7 +7,6 @@ same battle-tested ``psutil`` paths that are already in production.from __future__ import annotationsimport os-import signalimport sysimport subprocessimport time@@ -16,6 +15,8 @@ from pathlib import Pathimport psutil+from cli.tree import send_ctrl_break+# Post-kill grace before re-checking the port. Windows can lag a few# hundred ms releasing the listener socket after the bound process@@ -107,37 +108,19 @@ def kill_pid(pid: int, *, graceful_timeout: float = 3.0) -> bool:Windows: send ``CTRL_BREAK_EVENT`` first so daemons spawned with``CREATE_NEW_PROCESS_GROUP`` (the supervisor's children — see``cli/tree.py:new_session_kwargs``) get a real shutdown signal and- can release listener sockets cleanly. ``proc.terminate()``- (= ``TerminateProcess``) is the fallback for processes that weren't- spawned with a process group — equivalent to SIGKILL, leaves the- OS holding sockets briefly. Same pattern as- ``cli/supervisor.py:_stop_proc``.+ can release listener sockets cleanly. The event only reaches+ processes attached to this console, so anything else (a service+ started in another terminal, an orphan from a previous session) goes+ straight to ``proc.terminate()`` (= ``TerminateProcess``) --+ equivalent to SIGKILL, leaves the OS holding sockets briefly. The+ event is sent by ``cli/tree.py:send_ctrl_break``, never ``os.kill``;+ its docstring says why. Same pattern as ``cli/supervisor.py:_stop_proc``.POSIX: plain ``proc.terminate()`` (SIGTERM)."""try:proc = psutil.Process(pid)- if sys.platform == "win32":- try:- os.kill(pid, signal.CTRL_BREAK_EVENT)- except (OSError, ProcessLookupError, SystemError):- # Fall back to TerminateProcess via psutil.terminate.- # * ``OSError`` -- target wasn't in our process group.- # * ``SystemError`` -- CPython issue #106148: on- # Windows, ``os.kill`` returns success AND sets an- # ``OSError`` when ``GenerateConsoleCtrlEvent``- # fails with ERROR_INVALID_PARAMETER (e.g. the- # target process isn't in the caller's console- # group). CPython detects the inconsistency and- # raises ``SystemError`` instead of propagating- # the underlying OSError. ``kill_pid`` is used to- # clean up stale state (port squatters, orphans- # from a prior session) -- none of them are in- # our console group, so this path fires every- # time on Windows and used to crash- # ``company stop``.- proc.terminate()- else:+ if sys.platform != "win32" or not send_ctrl_break(pid):proc.terminate()try:proc.wait(timeout=graceful_timeout)diff --git a/cli/supervisor.py b/cli/supervisor.pyindex 450599bd..cac6d43f 100644--- a/cli/supervisor.py+++ b/cli/supervisor.py@@ -33,7 +33,7 @@ import anyiofrom cli.colors import emit, next_colorfrom cli.run import which_argvfrom cli.tcp import wait_for_tcp_port-from cli.tree import add_to_job, kill_tree, new_session_kwargs+from cli.tree import add_to_job, kill_tree, new_session_kwargs, send_ctrl_breakclass RestartPolicy(enum.Enum):@@ -198,7 +198,8 @@ class Manager:POSIX: ``proc.terminate()`` sends ``SIGTERM`` to the child.- Windows: ``os.kill(pid, CTRL_BREAK_EVENT)`` — the only Win32+ Windows: ``CTRL_BREAK_EVENT`` via ``cli/tree.py:send_ctrl_break``+ (never ``os.kill``; its docstring says why) — the only Win32signal-equivalent that lets a child run cleanup handlers. Wespawn every child with ``CREATE_NEW_PROCESS_GROUP``(``cli/tree.py:new_session_kwargs``) so ``CTRL_BREAK_EVENT`` is@@ -223,7 +224,8 @@ class Manager:with anyio.CancelScope(shield=True):try:if sys.platform == "win32":- os.kill(proc.pid, signal.CTRL_BREAK_EVENT)+ if not send_ctrl_break(proc.pid):+ returnelse:proc.terminate()except (ProcessLookupError, OSError):diff --git a/cli/tests/test_ports.py b/cli/tests/test_ports.pyindex b381a9af..db0e7693 100644--- a/cli/tests/test_ports.py+++ b/cli/tests/test_ports.py@@ -37,25 +37,31 @@ def test_windows_kill_pid_delivers_graceful_signal_before_waiting():with (patch.object(psutil, "Process", return_value=proc),patch.object(ports.sys, "platform", "win32"),- patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),- patch.object(ports.os, "kill") as signal_process,+ patch.object(ports, "send_ctrl_break", return_value=True) as send_break,+ patch.object(ports.os, "kill") as os_kill,):assert ports.kill_pid(123, graceful_timeout=125) is True- signal_process.assert_called_once_with(123, 1)+ send_break.assert_called_once_with(123)+ os_kill.assert_not_called()proc.wait.assert_called_once_with(timeout=125)proc.terminate.assert_not_called()proc.kill.assert_not_called()-def test_windows_kill_pid_falls_back_if_control_signal_unavailable():+def test_windows_kill_pid_terminates_what_the_signal_cannot_reach():+ """A service started in another terminal cannot get the event. Sending it+ with ``os.kill`` crashed ``company stop``: on Python 3.12.8 the refused+ event became a hard kill plus a pending ``OSError``, which surfaced as a+ ``SystemError`` inside psutil's ``Process.wait``."""proc = MagicMock()with (patch.object(psutil, "Process", return_value=proc),patch.object(ports.sys, "platform", "win32"),- patch.object(ports.signal, "CTRL_BREAK_EVENT", 1, create=True),- patch.object(ports.os, "kill", side_effect=OSError("no process group")),+ patch.object(ports, "send_ctrl_break", return_value=False),+ patch.object(ports.os, "kill") as os_kill,):assert ports.kill_pid(123) is True+ os_kill.assert_not_called()proc.terminate.assert_called_once()proc.wait.assert_called_once_with(timeout=3.0)diff --git a/cli/tests/test_supervisor.py b/cli/tests/test_supervisor.pyindex 441d1fc4..1578376b 100644--- a/cli/tests/test_supervisor.py+++ b/cli/tests/test_supervisor.py@@ -67,6 +67,47 @@ async def test_manager_runs_clean_exit_service():assert rc == 0+class _ExitsWhenWaited:+ """Stands in for an ``anyio`` process that exits once it is waited on."""++ pid = 4321++ def __init__(self) -> None:+ self.returncode = None+ self.terminated = False++ async def wait(self) -> int:+ self.returncode = 0+ return 0++ def terminate(self) -> None:+ self.terminated = True++[email protected]("sent", [True, False])+async def test_windows_stop_sends_ctrl_break_without_os_kill(monkeypatch, sent):+ """``os.kill(pid, CTRL_BREAK_EVENT)`` hard-kills the child and leaves an+ ``OSError`` pending when Windows refuses the event (Python 3.12.8). When the+ event does not go out, ``_stop_proc`` returns without waiting, as it did on+ that ``OSError``."""+ from cli import supervisor++ def forbidden(*_args):+ raise AssertionError("_stop_proc must not call os.kill")++ sent_to = []+ monkeypatch.setattr(supervisor.sys, "platform", "win32")+ monkeypatch.setattr(supervisor.os, "kill", forbidden)+ monkeypatch.setattr(supervisor, "send_ctrl_break", lambda pid: sent_to.append(pid) or sent)+ monkeypatch.setattr(supervisor, "emit", lambda *_args, **_kwargs: None)+ proc = _ExitsWhenWaited()+ await Manager()._stop_proc(proc, ServiceSpec(name="svc", argv=["true"]))+ assert sent_to == [proc.pid]+ assert proc.returncode == (0 if sent else None)+ assert not proc.terminated++@pytest.fixturedef anyio_backend():return "asyncio"diff --git a/cli/tests/test_tree.py b/cli/tests/test_tree.pynew file mode 100644index 00000000..dabb1e33--- /dev/null+++ b/cli/tests/test_tree.py@@ -0,0 +1,66 @@+"""Tests for ``cli.tree``."""++from __future__ import annotations++import subprocess+import sys+import time+from unittest.mock import patch++import pytest++from cli import tree++windows_only = pytest.mark.skipif(sys.platform != "win32", reason="console control events exist only on Windows")+++def _sleeper(creationflags: int) -> subprocess.Popen:+ # The base interpreter: a venv launcher would add a process of its own.+ return subprocess.Popen(+ [sys._base_executable, "-c", "import time; time.sleep(30)"],+ creationflags=creationflags,+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+++def test_send_ctrl_break_is_windows_only():+ with patch.object(tree.sys, "platform", "linux"):+ assert tree.send_ctrl_break(123) is False+++@windows_only+def test_send_ctrl_break_leaves_a_process_outside_this_console_alone():+ """Windows cannot deliver the event outside this console. ``os.kill`` turned+ that refusal into a hard kill and a pending ``OSError`` on Python 3.12.8."""+ sleeper = _sleeper(subprocess.DETACHED_PROCESS)+ try:+ assert tree.send_ctrl_break(sleeper.pid) is False+ with pytest.raises(subprocess.TimeoutExpired):+ sleeper.wait(timeout=0.5)+ finally:+ sleeper.kill()+ sleeper.wait(timeout=10)+++@windows_only+def test_send_ctrl_break_stops_a_process_group_on_this_console():+ pywintypes = pytest.importorskip("pywintypes")+ win32console = pytest.importorskip("win32console")+ try:+ win32console.GetConsoleProcessList()+ except pywintypes.error:+ pytest.skip("this test process has no console")+ sleeper = _sleeper(subprocess.CREATE_NEW_PROCESS_GROUP)+ try:+ deadline = time.monotonic() + 10+ while sleeper.pid not in win32console.GetConsoleProcessList():+ assert time.monotonic() < deadline, "the sleeper never attached to this console"+ time.sleep(0.05)+ assert tree.send_ctrl_break(sleeper.pid) is True+ sleeper.wait(timeout=10) # long before its own 30 s sleep ends+ finally:+ if sleeper.poll() is None:+ sleeper.kill()+ sleeper.wait(timeout=10)diff --git a/cli/tree.py b/cli/tree.pyindex 762eadb6..04909471 100644--- a/cli/tree.py+++ b/cli/tree.py@@ -70,6 +70,47 @@ def signal_group(pid: int, sig: signal.Signals = signal.SIGTERM) -> None:pass+def send_ctrl_break(pid: int) -> bool:+ """Send ``CTRL_BREAK_EVENT`` to the process group ``pid`` leads (Windows only).++ Returns whether the event went out: False on other platforms, when+ ``pid`` is not attached to this process's console (Windows delivers+ console events only within one console), and when Windows refuses it.++ Goes through pywin32, never ``os.kill(pid, CTRL_BREAK_EVENT)``. Before+ CPython 3.12.9 and 3.13.2 (gh-58689), an ``os.kill`` whose event+ Windows refuses hard-kills the target with ``TerminateProcess`` and+ returns with the ``OSError`` still set; the stale error surfaces later+ as a ``SystemError`` from an unrelated call. ``company stop`` died that+ way inside psutil's ``Process.wait``, because the services it stops run+ in another terminal's console. Checking the console first also covers+ the console hosts that accept an event for another console and deliver+ nothing, which left callers waiting out their whole grace period.+ https://learn.microsoft.com/en-us/windows/console/generateconsolectrlevent+ """+ if sys.platform != "win32":+ return False+ try:+ import pywintypes # type: ignore[import-not-found]+ import win32api # type: ignore[import-not-found]+ import win32con # type: ignore[import-not-found]+ import win32console # type: ignore[import-not-found]+ except ImportError as e:+ sys.stderr.write(+ f"WARN: pywin32 import failed ({e}); stopping pid={pid} without "+ f"CTRL_BREAK. Reinstall via: pip install --force-reinstall pywin32\n"+ )+ return False+ try:+ # Raises when this process has no console.+ if pid not in win32console.GetConsoleProcessList():+ return False+ win32api.GenerateConsoleCtrlEvent(win32con.CTRL_BREAK_EVENT, pid)+ except pywintypes.error:+ return False+ return True++# ---------------------------------------------------------------- Windows Job Objectdiff --git a/docs-internal/errors.md b/docs-internal/errors.mdindex dc814f49..88740d9a 100644--- a/docs-internal/errors.md+++ b/docs-internal/errors.md@@ -594,3 +594,23 @@ Seen 2026-09-05 with a `tikhubAction` tool whose results were about 400,000 charLocked by `server/tests/services/test_tool_output.py`, `server/tests/temporal/test_agent_context_pressure.py`, `server/tests/temporal/test_agent_workflow_pressure.py` and `server/tests/temporal/test_prepare_payload_pressure.py`. Design record: [ARCHIVE/AGENT_COMPACTION_FIX_PLAN.md](./ARCHIVE/AGENT_COMPACTION_FIX_PLAN.md).**Recovery for a row saved before the fix**: clear the conversation once from the Context panel (or Reset the deployment). New transcripts stay well under the cap.++## 29. `company stop` crashes on Windows with `SystemError: <class 'int'> returned a result with an exception set`++**Symptom**: `company stop` (or `bun run stop`) frees the ports and kills Temporal, then dies while reaping orphaned processes. The traceback ends inside psutil's `Process.wait`:++```+OSError: [WinError 87] The parameter is incorrect++The above exception was the direct cause of the following exception:+...+SystemError: <class 'int'> returned a result with an exception set+```++The `OSError` has no traceback of its own. Orphans the reaper had not reached yet stay alive, and the command exits 1. The CLI supervisor's shutdown and the backend's `terminate_then_kill` sent the event the same way and could fail the same way.++**Root cause**: `kill_pid` sent `CTRL_BREAK_EVENT` with `os.kill`. Windows delivers console events only to processes attached to the caller's console, and the services `company stop` reaps run in another terminal's console, so Windows refuses the event (error 87 in the reported case, error 6 when the caller has no console). Before CPython 3.12.9 and 3.13.2 ([gh-58689](https://github.com/python/cpython/issues/58689)), `os.kill` does not raise on that refusal: it records the error, falls through to `TerminateProcess` (hard-killing the target), and returns success with the error still set. CPython reports that inconsistency only on its generic call path. Once the `os.kill` call site has been specialized after its first run, release builds skip the check, and the stale error surfaces at the next call that does check: the `int()` inside psutil's `Process.wait`. So the `except SystemError` around `os.kill` covered only the first orphan. The CLI and server venvs here run 3.12.8.++**Fix**: nothing sends console events through `os.kill` any more. `cli/tree.py::send_ctrl_break` (through pywin32) and its server copy in `services/_supervisor/util.py` (through `ctypes`, since the server has no pywin32) call `GenerateConsoleCtrlEvent` only for a process attached to the caller's console, and return whether the event went out. `kill_pid` terminates everything else at once. Before, some console hosts accepted an event for another console, delivered nothing, and `kill_pid` waited out its grace period. `_stop_proc` and `terminate_then_kill` treat an event that did not go out as they treated `os.kill`'s `OSError`. Locked by `cli/tests/test_tree.py`, `cli/tests/test_ports.py`, `cli/tests/test_supervisor.py` and `server/tests/services/test_supervisor.py`; their Windows-only tests send real events to processes they start.++To reproduce the old failure on 3.12.8, run `kill_orphaned_opencompany_processes` from a process with no console against sleepers it started itself, with a root their command lines name: it raises the `SystemError`.diff --git a/server/services/_supervisor/util.py b/server/services/_supervisor/util.pyindex 8b245171..4d88d0f0 100644--- a/server/services/_supervisor/util.py+++ b/server/services/_supervisor/util.py@@ -10,7 +10,6 @@ from __future__ import annotationsimport asyncioimport logging-import osimport signalimport sysfrom typing import Callable, Optional@@ -45,6 +44,48 @@ def kill_tree(pid: int) -> None:pass+def _console_process_ids() -> set[int]:+ """PIDs attached to this process's console; empty when it has none (Windows only)."""+ import ctypes+ from ctypes import wintypes++ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)+ kernel32.GetConsoleProcessList.restype = wintypes.DWORD+ kernel32.GetConsoleProcessList.argtypes = [wintypes.LPDWORD, wintypes.DWORD]+ ids = (wintypes.DWORD * 1)()+ while True:+ # 0 without a console; a count larger than the buffer is the size it needs.+ count = kernel32.GetConsoleProcessList(ids, len(ids))+ if count <= len(ids):+ return set(ids[:count])+ ids = (wintypes.DWORD * count)()+++def send_ctrl_break(pid: int) -> bool:+ """Send ``CTRL_BREAK_EVENT`` to the process group ``pid`` leads (Windows only).++ Returns whether the event went out: False on other platforms, when+ ``pid`` is not attached to this process's console (Windows delivers+ console events only within one console), and when Windows refuses it.+ Calls ``GenerateConsoleCtrlEvent`` rather than ``os.kill``, which before+ CPython 3.12.9 and 3.13.2 (gh-58689) turns a refused event into a+ ``TerminateProcess`` hard kill and returns with the ``OSError`` still+ set, so a ``SystemError`` surfaces later from an unrelated call. The CLI+ has the same helper in ``cli/tree.py``; the server cannot import the CLI+ package and has no pywin32.+ https://learn.microsoft.com/en-us/windows/console/generateconsolectrlevent+ """+ if sys.platform != "win32" or pid not in _console_process_ids():+ return False+ import ctypes+ from ctypes import wintypes++ kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)+ kernel32.GenerateConsoleCtrlEvent.restype = wintypes.BOOL+ kernel32.GenerateConsoleCtrlEvent.argtypes = [wintypes.DWORD, wintypes.DWORD]+ return bool(kernel32.GenerateConsoleCtrlEvent(signal.CTRL_BREAK_EVENT, pid))++async def terminate_then_kill(proc: anyio.abc.Process,*,@@ -56,17 +97,15 @@ async def terminate_then_kill(POSIX path: ``proc.terminate()`` (SIGTERM), wait, ``kill_tree()``.Windows graceful path (``use_ctrl_break=True``, requires the processto have been spawned with ``CREATE_NEW_PROCESS_GROUP``):- ``os.kill(pid, CTRL_BREAK_EVENT)`` — the only way to send a real- SIGINT-equivalent on Windows.+ ``CTRL_BREAK_EVENT`` via :func:`send_ctrl_break` — the only way to send+ a real SIGINT-equivalent on Windows."""if proc.returncode is not None:returnif use_ctrl_break and sys.platform == "win32":- try:- os.kill(proc.pid, signal.CTRL_BREAK_EVENT)- except (ProcessLookupError, OSError):- pass+ # When the event does not go out, the grace wait ends in the tree-kill below.+ send_ctrl_break(proc.pid)else:try:proc.terminate()diff --git a/server/tests/services/test_supervisor.py b/server/tests/services/test_supervisor.pyindex 64cae745..867877f9 100644--- a/server/tests/services/test_supervisor.py+++ b/server/tests/services/test_supervisor.py@@ -3,7 +3,10 @@from __future__ import annotationsimport asyncio+import os+import subprocessimport sys+import timefrom pathlib import Pathfrom unittest.mock import AsyncMock, MagicMock, patch@@ -25,7 +28,13 @@ from services._supervisor import ( # noqa: E402shutdown_all_supervisors,)from services._supervisor.registry import _SUPERVISORS # noqa: E402-from services._supervisor.util import drain_stream, kill_tree, terminate_then_kill # noqa: E402+from services._supervisor.util import ( # noqa: E402+ _console_process_ids,+ drain_stream,+ kill_tree,+ send_ctrl_break,+ terminate_then_kill,+)# --- helpers ---------------------------------------------------------------@@ -237,6 +246,79 @@ async def test_terminate_then_kill_force_kills_after_grace():kill_tree_mock.assert_called_once_with(42)+async def test_terminate_then_kill_sends_ctrl_break_without_os_kill():+ """``os.kill(pid, CTRL_BREAK_EVENT)`` hard-kills the child and leaves an+ ``OSError`` pending when Windows refuses the event (Python 3.12.8, gh-58689)."""+ proc = MagicMock()+ proc.returncode = None+ proc.pid = 42++ async def exit_after_the_break():+ proc.returncode = 0++ proc.wait = AsyncMock(side_effect=exit_after_the_break)+ with (+ patch("services._supervisor.util.sys.platform", "win32"),+ patch("services._supervisor.util.send_ctrl_break", return_value=True) as send_break,+ patch("services._supervisor.util.kill_tree") as kill_tree_mock,+ patch("os.kill", side_effect=AssertionError("terminate_then_kill must not call os.kill")),+ ):+ await terminate_then_kill(proc, grace=1.0, use_ctrl_break=True)++ send_break.assert_called_once_with(42)+ proc.terminate.assert_not_called()+ kill_tree_mock.assert_not_called()+++# --- util.send_ctrl_break (real Win32 calls) --------------------------------++windows_only = pytest.mark.skipif(sys.platform != "win32", reason="console control events exist only on Windows")+++def _sleeper(creationflags: int) -> subprocess.Popen:+ # The base interpreter: a venv launcher would add a process of its own.+ return subprocess.Popen(+ [sys._base_executable, "-c", "import time; time.sleep(30)"],+ creationflags=creationflags,+ stdin=subprocess.DEVNULL,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+++@windows_only+def test_send_ctrl_break_leaves_a_process_outside_this_console_alone():+ sleeper = _sleeper(subprocess.DETACHED_PROCESS)+ try:+ assert send_ctrl_break(sleeper.pid) is False+ with pytest.raises(subprocess.TimeoutExpired):+ sleeper.wait(timeout=0.5)+ finally:+ sleeper.kill()+ sleeper.wait(timeout=10)+++@windows_only+def test_send_ctrl_break_stops_a_process_group_on_this_console():+ attached = _console_process_ids()+ if not attached:+ pytest.skip("this test process has no console")+ assert os.getpid() in attached+ sleeper = _sleeper(subprocess.CREATE_NEW_PROCESS_GROUP)+ try:+ deadline = time.monotonic() + 10+ while sleeper.pid not in _console_process_ids():+ assert time.monotonic() < deadline, "the sleeper never attached to this console"+ time.sleep(0.05)+ assert send_ctrl_break(sleeper.pid) is True+ sleeper.wait(timeout=10) # long before its own 30 s sleep ends+ finally:+ if sleeper.poll() is None:+ sleeper.kill()+ sleeper.wait(timeout=10)++# --- util.drain_stream -----------------------------------------------------