Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Stop once, in switch-core's own order, on SIGTERM
Petr Bauch6c98854ed14a
diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex c36762e76..334a9fe86 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -6,8 +6,8 @@ import loggingimport osimport signalimport time-from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine-from contextlib import asynccontextmanager+from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator+from contextlib import asynccontextmanager, contextmanagerfrom datetime import timedeltafrom pathlib import Path@@ -243,6 +243,46 @@ _CONNECTION_SWEEP_INTERVAL = 2.0# `observability.logs.SHUTDOWN_FLUSH_SECONDS`._FORCED_EXIT_GRACE_SECONDS = 3.0++class _Server(uvicorn.Server):+ """uvicorn's server without uvicorn's own signal handling.++ `run` stops the process on SIGTERM and SIGINT itself (`_shutdown`): it+ hands the owned work over and drains before it tells uvicorn to stop.+ uvicorn's handlers would stop serving at once, skipping that, and re-raise+ the signal once serving ended, starting a second shutdown. asyncio+ delivers a signal to both, so uvicorn's are not installed at all.+ """++ @contextmanager+ def capture_signals(self) -> Iterator[None]:+ yield+++class _ShutdownOnSignal:+ """The process's stop-signal handler: the first signal starts the one+ shutdown, and `finish` waits for it, which ends in `os._exit`. A second+ signal while it runs exits at once, as a second Ctrl-C is meant to."""++ def __init__(self, start: Callable[[], Coroutine[None, None, None]]) -> None:+ self._start = start+ self._task: asyncio.Task[None] | None = None++ def __call__(self) -> None:+ if self._task is not None:+ logger.warning(+ "A second stop signal arrived while shutting down; exiting now "+ "without finishing the shutdown."+ )+ os._exit(1)+ self._task = asyncio.get_running_loop().create_task(self._start())++ async def finish(self) -> None:+ """Wait for the shutdown a signal started; return at once if none did."""+ if self._task is not None:+ await self._task++# With several replicas, how long a process keeps serving after it is told to# stop and has taken itself out of readiness: long enough for the Service to# stop sending it new requests, so they go to another replica rather than@@ -1501,30 +1541,29 @@ async def run(config: SwitchConfig, process_id: str) -> None:log_level="info",log_config=None,)- server = uvicorn.Server(server_config)+ server = _Server(server_config)loop = asyncio.get_event_loop()- def _begin_shutdown() -> None:- asyncio.create_task(- _shutdown(- server,- draining.set,- lambda: _release_process(- ownership, shared_connections, management, process_lease- ),- _DRAIN_SECONDS if config.core_replicas > 1 else 0.0,- connections.controllers,- client_lifecycle,- collab_lifecycle,- connector_lifecycle,- provisioning,- discord_gateway,- teams_app,- telegram_app,- telegram_app_task,- )+ _begin_shutdown = _ShutdownOnSignal(+ lambda: _shutdown(+ server,+ draining.set,+ lambda: _release_process(+ ownership, shared_connections, management, process_lease+ ),+ _DRAIN_SECONDS if config.core_replicas > 1 else 0.0,+ connections.controllers,+ client_lifecycle,+ collab_lifecycle,+ connector_lifecycle,+ provisioning,+ discord_gateway,+ teams_app,+ telegram_app,+ telegram_app_task,)+ )for sig in (signal.SIGINT, signal.SIGTERM):try:@@ -1540,6 +1579,9 @@ async def run(config: SwitchConfig, process_id: str) -> None:)await server.serve()+ # uvicorn returns once its own part is done, while `_shutdown` is still+ # stopping the rest; returning here would end the loop under it.+ await _begin_shutdown.finish()async def _seed_admin_and_bootstrap_key(diff --git a/core/tests/switch_core/test_main_signals.py b/core/tests/switch_core/test_main_signals.pynew file mode 100644index 000000000..c7d3d44b9--- /dev/null+++ b/core/tests/switch_core/test_main_signals.py@@ -0,0 +1,140 @@+"""switch-core stops once, in its own order, on SIGTERM.++uvicorn handles SIGTERM and SIGINT itself while it serves, and asyncio+delivers a signal to its handlers and to switch-core's alike. Left in place,+uvicorn stopped serving at once, before switch-core had handed its work over+and drained, and re-raised the signal once serving ended, which started a+second shutdown that the process then exited under ("Task was destroyed but+it is pending").+"""++from __future__ import annotations++import asyncio+import os+import signal+import subprocess+import sys+import textwrap+import time++import pytest+import uvicorn++from switch_core import main as main_module+from switch_core.main import _Server, _ShutdownOnSignal+++async def _app(scope: dict, receive: object, send: object) -> None:+ if scope["type"] == "lifespan":+ return+++def test_the_server_leaves_signal_handling_to_switch_core() -> None:+ def ours(signum: int, frame: object) -> None:+ raise AssertionError("not called")++ previous = signal.signal(signal.SIGTERM, ours)+ try:+ server = _Server(uvicorn.Config(_app))+ with server.capture_signals():+ assert signal.getsignal(signal.SIGTERM) is ours+ assert signal.getsignal(signal.SIGTERM) is ours+ finally:+ signal.signal(signal.SIGTERM, previous)+++async def test_the_first_signal_starts_the_one_shutdown_and_finish_waits_for_it(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ started: list[str] = []+ release = asyncio.Event()++ async def shutdown() -> None:+ started.append("shutdown")+ await release.wait()++ exits: list[int] = []+ monkeypatch.setattr(main_module.os, "_exit", exits.append)+ on_signal = _ShutdownOnSignal(shutdown)++ await on_signal.finish()+ on_signal()+ finishing = asyncio.create_task(on_signal.finish())+ await asyncio.sleep(0)+ assert started == ["shutdown"]+ assert not finishing.done()++ on_signal()+ assert exits == [1]+ assert started == ["shutdown"]++ release.set()+ await finishing+++_PROCESS = textwrap.dedent(+ """+ import asyncio, os, signal, sys+ import uvicorn+ from switch_core.main import _Server, _ShutdownOnSignal++ async def app(scope, receive, send):+ if scope["type"] == "lifespan":+ while True:+ message = await receive()+ if message["type"] == "lifespan.startup":+ await send({"type": "lifespan.startup.complete"})+ else:+ await send({"type": "lifespan.shutdown.complete"})+ return++ async def run():+ server = _Server(uvicorn.Config(app, host="127.0.0.1", port=0, log_level="error"))++ async def shutdown():+ print("shutdown started", flush=True)+ await asyncio.sleep(0.3)+ server.should_exit = True+ await asyncio.sleep(0.5)+ print("shutdown finished", flush=True)+ os._exit(0)++ on_signal = _ShutdownOnSignal(shutdown)+ asyncio.get_running_loop().add_signal_handler(signal.SIGTERM, on_signal)+ serving = asyncio.create_task(server.serve())+ while not server.started:+ await asyncio.sleep(0.01)+ print("serving", flush=True)+ await serving+ print("serve returned", flush=True)+ await on_signal.finish()++ asyncio.run(run())+ """+)++[email protected](sys.platform == "win32", reason="no SIGTERM handlers on Windows")+def test_sigterm_runs_the_shutdown_once_and_to_its_end() -> None:+ process = subprocess.Popen(+ [sys.executable, "-c", _PROCESS],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ text=True,+ )+ assert process.stdout is not None+ deadline = time.monotonic() + 20+ while process.stdout.readline().strip() != "serving":+ assert time.monotonic() < deadline, "the server did not start"+ assert process.poll() is None, process.stderr.read() if process.stderr else ""+ os.kill(process.pid, signal.SIGTERM)+ out, err = process.communicate(timeout=20)++ assert process.returncode == 0, err+ lines = out.splitlines()+ assert lines.count("shutdown started") == 1+ assert lines[-1] == "shutdown finished"+ # Serving continued until the shutdown told uvicorn to stop.+ assert lines.index("serve returned") > lines.index("shutdown started")+ assert "destroyed but it is pending" not in err