Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
feat(runtime): name whatever blocks the service's event loop while it blocks (#2502)
Call White4febbf33c7e2
diff --git a/core/controller.py b/core/controller.pyindex 9bb6f5e0f..5b48a9e16 100644--- a/core/controller.py+++ b/core/controller.py@@ -51,6 +51,7 @@ from core.web_ui_watchdog import watch_web_uifrom core.vibe_agents import VibeAgent, VibeAgentStorefrom core.blocking import run_blockingfrom vibe.i18n import get_supported_languages, t as i18n_t+from core.loop_watchdog import EventLoopWatchdogfrom vibe.runtime import current_process_owns_service_instance, mark_service_instance_startedif TYPE_CHECKING:@@ -2148,6 +2149,9 @@ class Controller:try:self._loop = asyncio.new_event_loop()asyncio.set_event_loop(self._loop)+ # Names whatever blocks this loop while it blocks; see core/loop_watchdog.py.+ self._loop_watchdog = EventLoopWatchdog(self._loop)+ self._loop_watchdog.start()self._schedule_model_hub_snapshot_reconcile()self.show_git_checkpoint_service.start()# Internal control-IPC ASGI server for the Web UI / future@@ -2199,6 +2203,10 @@ class Controller:except Exception as exc:logger.debug("Internal control IPC cleanup skipped: %s", exc)self._internal_server_task = None+ loop_watchdog = getattr(self, "_loop_watchdog", None)+ if loop_watchdog is not None:+ loop_watchdog.stop()+ self._loop_watchdog = Noneif self._loop is not None:try:self._loop.stop()diff --git a/core/loop_watchdog.py b/core/loop_watchdog.pynew file mode 100644index 000000000..601b549e1--- /dev/null+++ b/core/loop_watchdog.py@@ -0,0 +1,122 @@+"""Report an event loop that stops turning, with the stack that stopped it.++A synchronous call on the service's loop -- a lock wait, a subprocess, a large+file rewrite -- stops every coroutine with it, and nothing on that loop can say+so while it happens: its own timers fire late, afterwards, with no trace of what+held them. A thread outside the loop can. It asks the loop to answer once a+second and, when the loop has not answered for ``STALL_SECONDS``, logs one+WARNING with the stack of every thread, so the blocking call is named in the+log while it is still blocking. When the loop answers again it logs how long+the stall lasted.+"""++from __future__ import annotations++import asyncio+import logging+import sys+import threading+import time+import traceback+from collections.abc import Callable++logger = logging.getLogger(__name__)++STALL_SECONDS = 5.0+POLL_SECONDS = 1.0+# At most one stack report per this long; a loop that keeps stalling still gets+# its recovery lines, which carry each stall's length.+REPORT_INTERVAL_SECONDS = 300.0+# Innermost frames kept per thread, and the report's overall size.+_FRAMES_PER_THREAD = 30+_MAX_REPORT_CHARS = 64 * 1024+++class EventLoopWatchdog:+ """A daemon thread that logs the stacks of a loop that has not answered for ``stall_seconds``."""++ def __init__(+ self,+ loop: asyncio.AbstractEventLoop,+ *,+ stall_seconds: float = STALL_SECONDS,+ poll_seconds: float = POLL_SECONDS,+ report_interval_seconds: float = REPORT_INTERVAL_SECONDS,+ monotonic: Callable[[], float] = time.monotonic,+ ) -> None:+ self._loop = loop+ self._stall_seconds = stall_seconds+ self._poll_seconds = poll_seconds+ self._report_interval_seconds = report_interval_seconds+ self._monotonic = monotonic+ self._stopped = threading.Event()+ self._thread: threading.Thread | None = None+ self._answered_at = monotonic()+ self._asking = False+ self._loop_thread_id: int | None = None+ self._last_report_at: float | None = None+ self._stall_reported = False++ def start(self) -> None:+ if self._thread is not None:+ return+ self._answered_at = self._monotonic()+ self._thread = threading.Thread(target=self._run, name="event-loop-watchdog", daemon=True)+ self._thread.start()++ def stop(self) -> None:+ self._stopped.set()+ thread, self._thread = self._thread, None+ if thread is not None and thread is not threading.current_thread():+ thread.join(timeout=self._poll_seconds * 2)++ def _answer(self) -> None:+ # Runs on the loop: the only work it adds there is two assignments.+ self._loop_thread_id = threading.get_ident()+ self._answered_at = self._monotonic()+ self._asking = False++ def _run(self) -> None:+ silent_since = self._answered_at+ while not self._stopped.wait(self._poll_seconds):+ answered_at = self._answered_at+ if self._stall_reported and answered_at > silent_since:+ logger.info("Event loop answered again after a %.1fs stall", answered_at - silent_since)+ self._stall_reported = False+ silent_since = answered_at+ if not self._asking:+ self._asking = True+ try:+ self._loop.call_soon_threadsafe(self._answer)+ except RuntimeError:+ return # the loop is closed+ now = self._monotonic()+ if self._stall_reported or now - answered_at < self._stall_seconds:+ continue+ self._stall_reported = True+ if self._last_report_at is not None and now - self._last_report_at < self._report_interval_seconds:+ continue+ self._last_report_at = now+ logger.warning(+ "Event loop has not answered for %.1fs; thread stacks follow.\n%s",+ now - answered_at,+ self._thread_stacks(),+ )++ def _thread_stacks(self) -> str:+ names = {thread.ident: thread.name for thread in threading.enumerate()}+ frames = sys._current_frames()+ # A loop that blocks before its first answer is still named: asyncio records its running thread.+ loop_thread = self._loop_thread_id or getattr(self._loop, "_thread_id", None)+ order = sorted(frames, key=lambda ident: (ident != loop_thread, names.get(ident, "")))+ sections = []+ for ident in order:+ if ident == threading.get_ident():+ continue+ label = "event loop" if ident == loop_thread else names.get(ident, "unnamed")+ stack = "".join(traceback.format_stack(frames[ident])[-_FRAMES_PER_THREAD:])+ sections.append(f"--- thread {ident} ({label}) ---\n{stack}")+ report = "".join(sections)+ if len(report) > _MAX_REPORT_CHARS:+ report = report[:_MAX_REPORT_CHARS] + "\n... (truncated)"+ return reportdiff --git a/tests/test_loop_watchdog.py b/tests/test_loop_watchdog.pynew file mode 100644index 000000000..68ddc136f--- /dev/null+++ b/tests/test_loop_watchdog.py@@ -0,0 +1,76 @@+from __future__ import annotations++import asyncio+import logging+import threading+import time++from core.loop_watchdog import EventLoopWatchdog+++def _wait_for(condition, timeout: float = 5.0) -> None:+ deadline = time.monotonic() + timeout+ while not condition():+ if time.monotonic() > deadline:+ raise AssertionError("condition not reached")+ time.sleep(0.01)+++def _block_loop_until(released: threading.Event) -> None:+ # The name is what the report has to show.+ released.wait(10)+++def _messages(caplog, level: int) -> list[str]:+ return [record.getMessage() for record in caplog.records if record.levelno == level and record.name == "core.loop_watchdog"]+++def test_a_blocked_loop_is_reported_once_with_the_blocking_stack_and_its_length(caplog) -> None:+ caplog.set_level(logging.INFO, logger="core.loop_watchdog")+ loop = asyncio.new_event_loop()+ runner = threading.Thread(target=loop.run_forever, daemon=True)+ runner.start()+ watchdog = EventLoopWatchdog(loop, stall_seconds=0.3, poll_seconds=0.05, report_interval_seconds=60)+ try:+ watchdog.start()+ released = threading.Event()+ loop.call_soon_threadsafe(_block_loop_until, released)+ _wait_for(lambda: _messages(caplog, logging.WARNING))+ # A stall keeps being one report, however long it lasts.+ time.sleep(0.5)+ released.set()+ _wait_for(lambda: _messages(caplog, logging.INFO))++ # A second stall inside the report interval adds only its recovery line.+ second = threading.Event()+ loop.call_soon_threadsafe(_block_loop_until, second)+ time.sleep(0.6)+ second.set()+ _wait_for(lambda: len(_messages(caplog, logging.INFO)) == 2)+ finally:+ watchdog.stop()+ loop.call_soon_threadsafe(loop.stop)+ runner.join(5)+ loop.close()++ (warning,) = _messages(caplog, logging.WARNING)+ assert "Event loop has not answered for" in warning+ assert "(event loop)" in warning and "_block_loop_until" in warning+ assert all("Event loop answered again after a" in line for line in _messages(caplog, logging.INFO))+++def test_a_turning_loop_is_never_reported(caplog) -> None:+ caplog.set_level(logging.INFO, logger="core.loop_watchdog")+ loop = asyncio.new_event_loop()+ runner = threading.Thread(target=loop.run_forever, daemon=True)+ runner.start()+ watchdog = EventLoopWatchdog(loop, stall_seconds=0.3, poll_seconds=0.05)+ try:+ watchdog.start()+ time.sleep(1.0)+ finally:+ watchdog.stop()+ loop.call_soon_threadsafe(loop.stop)+ runner.join(5)+ loop.close()+ assert _messages(caplog, logging.WARNING) == [] and _messages(caplog, logging.INFO) == []