Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Stop once, in switch-core's own order, on SIGTERM

Petr Bauch6c98854ed14a
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index c36762e76..334a9fe86 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -6,8 +6,8 @@ import logging
import os
import signal
import time
-from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine
-from contextlib import asynccontextmanager
+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator
+from contextlib import asynccontextmanager, contextmanager
from datetime import timedelta
from pathlib import Path
@@ -243,6 +243,46 @@ _CONNECTION_SWEEP_INTERVAL = 2.0
# `observability.logs.SHUTDOWN_FLUSH_SECONDS`.
_FORCED_EXIT_GRACE_SECONDS = 3.0
+
+class _Server(uvicorn.Server):
+ """uvicorn's server without uvicorn's own signal handling.
+
+ `run` stops the process on SIGTERM and SIGINT itself (`_shutdown`): it
+ hands the owned work over and drains before it tells uvicorn to stop.
+ uvicorn's handlers would stop serving at once, skipping that, and re-raise
+ the signal once serving ended, starting a second shutdown. asyncio
+ delivers a signal to both, so uvicorn's are not installed at all.
+ """
+
+ @contextmanager
+ def capture_signals(self) -> Iterator[None]:
+ yield
+
+
+class _ShutdownOnSignal:
+ """The process's stop-signal handler: the first signal starts the one
+ shutdown, and `finish` waits for it, which ends in `os._exit`. A second
+ signal while it runs exits at once, as a second Ctrl-C is meant to."""
+
+ def __init__(self, start: Callable[[], Coroutine[None, None, None]]) -> None:
+ self._start = start
+ self._task: asyncio.Task[None] | None = None
+
+ def __call__(self) -> None:
+ if self._task is not None:
+ logger.warning(
+ "A second stop signal arrived while shutting down; exiting now "
+ "without finishing the shutdown."
+ )
+ os._exit(1)
+ self._task = asyncio.get_running_loop().create_task(self._start())
+
+ async def finish(self) -> None:
+ """Wait for the shutdown a signal started; return at once if none did."""
+ if self._task is not None:
+ await self._task
+
+
# With several replicas, how long a process keeps serving after it is told to
# stop and has taken itself out of readiness: long enough for the Service to
# stop sending it new requests, so they go to another replica rather than
@@ -1501,30 +1541,29 @@ async def run(config: SwitchConfig, process_id: str) -> None:
log_level="info",
log_config=None,
)
- server = uvicorn.Server(server_config)
+ server = _Server(server_config)
loop = asyncio.get_event_loop()
- def _begin_shutdown() -> None:
- asyncio.create_task(
- _shutdown(
- server,
- draining.set,
- lambda: _release_process(
- ownership, shared_connections, management, process_lease
- ),
- _DRAIN_SECONDS if config.core_replicas > 1 else 0.0,
- connections.controllers,
- client_lifecycle,
- collab_lifecycle,
- connector_lifecycle,
- provisioning,
- discord_gateway,
- teams_app,
- telegram_app,
- telegram_app_task,
- )
+ _begin_shutdown = _ShutdownOnSignal(
+ lambda: _shutdown(
+ server,
+ draining.set,
+ lambda: _release_process(
+ ownership, shared_connections, management, process_lease
+ ),
+ _DRAIN_SECONDS if config.core_replicas > 1 else 0.0,
+ connections.controllers,
+ client_lifecycle,
+ collab_lifecycle,
+ connector_lifecycle,
+ provisioning,
+ discord_gateway,
+ teams_app,
+ telegram_app,
+ telegram_app_task,
)
+ )
for sig in (signal.SIGINT, signal.SIGTERM):
try:
@@ -1540,6 +1579,9 @@ async def run(config: SwitchConfig, process_id: str) -> None:
)
await server.serve()
+ # uvicorn returns once its own part is done, while `_shutdown` is still
+ # stopping the rest; returning here would end the loop under it.
+ await _begin_shutdown.finish()
async def _seed_admin_and_bootstrap_key(
diff --git a/core/tests/switch_core/test_main_signals.py b/core/tests/switch_core/test_main_signals.py
new file mode 100644
index 000000000..c7d3d44b9
--- /dev/null
+++ b/core/tests/switch_core/test_main_signals.py
@@ -0,0 +1,140 @@
+"""switch-core stops once, in its own order, on SIGTERM.
+
+uvicorn handles SIGTERM and SIGINT itself while it serves, and asyncio
+delivers a signal to its handlers and to switch-core's alike. Left in place,
+uvicorn stopped serving at once, before switch-core had handed its work over
+and drained, and re-raised the signal once serving ended, which started a
+second shutdown that the process then exited under ("Task was destroyed but
+it is pending").
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import signal
+import subprocess
+import sys
+import textwrap
+import time
+
+import pytest
+import uvicorn
+
+from switch_core import main as main_module
+from switch_core.main import _Server, _ShutdownOnSignal
+
+
+async def _app(scope: dict, receive: object, send: object) -> None:
+ if scope["type"] == "lifespan":
+ return
+
+
+def test_the_server_leaves_signal_handling_to_switch_core() -> None:
+ def ours(signum: int, frame: object) -> None:
+ raise AssertionError("not called")
+
+ previous = signal.signal(signal.SIGTERM, ours)
+ try:
+ server = _Server(uvicorn.Config(_app))
+ with server.capture_signals():
+ assert signal.getsignal(signal.SIGTERM) is ours
+ assert signal.getsignal(signal.SIGTERM) is ours
+ finally:
+ signal.signal(signal.SIGTERM, previous)
+
+
+async def test_the_first_signal_starts_the_one_shutdown_and_finish_waits_for_it(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ started: list[str] = []
+ release = asyncio.Event()
+
+ async def shutdown() -> None:
+ started.append("shutdown")
+ await release.wait()
+
+ exits: list[int] = []
+ monkeypatch.setattr(main_module.os, "_exit", exits.append)
+ on_signal = _ShutdownOnSignal(shutdown)
+
+ await on_signal.finish()
+ on_signal()
+ finishing = asyncio.create_task(on_signal.finish())
+ await asyncio.sleep(0)
+ assert started == ["shutdown"]
+ assert not finishing.done()
+
+ on_signal()
+ assert exits == [1]
+ assert started == ["shutdown"]
+
+ release.set()
+ await finishing
+
+
+_PROCESS = textwrap.dedent(
+ """
+ import asyncio, os, signal, sys
+ import uvicorn
+ from switch_core.main import _Server, _ShutdownOnSignal
+
+ async def app(scope, receive, send):
+ if scope["type"] == "lifespan":
+ while True:
+ message = await receive()
+ if message["type"] == "lifespan.startup":
+ await send({"type": "lifespan.startup.complete"})
+ else:
+ await send({"type": "lifespan.shutdown.complete"})
+ return
+
+ async def run():
+ server = _Server(uvicorn.Config(app, host="127.0.0.1", port=0, log_level="error"))
+
+ async def shutdown():
+ print("shutdown started", flush=True)
+ await asyncio.sleep(0.3)
+ server.should_exit = True
+ await asyncio.sleep(0.5)
+ print("shutdown finished", flush=True)
+ os._exit(0)
+
+ on_signal = _ShutdownOnSignal(shutdown)
+ asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, on_signal)
+ serving = asyncio.create_task(server.serve())
+ while not server.started:
+ await asyncio.sleep(0.01)
+ print("serving", flush=True)
+ await serving
+ print("serve returned", flush=True)
+ await on_signal.finish()
+
+ asyncio.run(run())
+ """
+)
+
+
[email protected](sys.platform == "win32", reason="no SIGTERM handlers on Windows")
+def test_sigterm_runs_the_shutdown_once_and_to_its_end() -> None:
+ process = subprocess.Popen(
+ [sys.executable, "-c", _PROCESS],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ )
+ assert process.stdout is not None
+ deadline = time.monotonic() + 20
+ while process.stdout.readline().strip() != "serving":
+ assert time.monotonic() < deadline, "the server did not start"
+ assert process.poll() is None, process.stderr.read() if process.stderr else ""
+ os.kill(process.pid, signal.SIGTERM)
+ out, err = process.communicate(timeout=20)
+
+ assert process.returncode == 0, err
+ lines = out.splitlines()
+ assert lines.count("shutdown started") == 1
+ assert lines[-1] == "shutdown finished"
+ # Serving continued until the shutdown told uvicorn to stop.
+ assert lines.index("serve returned") > lines.index("shutdown started")
+ assert "destroyed but it is pending" not in err