Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

feat(runtime): name whatever blocks the service's event loop while it blocks (#2502)

Call White4febbf33c7e2
diff --git a/core/controller.py b/core/controller.py
index 9bb6f5e0f..5b48a9e16 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -51,6 +51,7 @@ from core.web_ui_watchdog import watch_web_ui
from core.vibe_agents import VibeAgent, VibeAgentStore
from core.blocking import run_blocking
from vibe.i18n import get_supported_languages, t as i18n_t
+from core.loop_watchdog import EventLoopWatchdog
from vibe.runtime import current_process_owns_service_instance, mark_service_instance_started
if TYPE_CHECKING:
@@ -2148,6 +2149,9 @@ class Controller:
try:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
+ # Names whatever blocks this loop while it blocks; see core/loop_watchdog.py.
+ self._loop_watchdog = EventLoopWatchdog(self._loop)
+ self._loop_watchdog.start()
self._schedule_model_hub_snapshot_reconcile()
self.show_git_checkpoint_service.start()
# Internal control-IPC ASGI server for the Web UI / future
@@ -2199,6 +2203,10 @@ class Controller:
except Exception as exc:
logger.debug("Internal control IPC cleanup skipped: %s", exc)
self._internal_server_task = None
+ loop_watchdog = getattr(self, "_loop_watchdog", None)
+ if loop_watchdog is not None:
+ loop_watchdog.stop()
+ self._loop_watchdog = None
if self._loop is not None:
try:
self._loop.stop()
diff --git a/core/loop_watchdog.py b/core/loop_watchdog.py
new file mode 100644
index 000000000..601b549e1
--- /dev/null
+++ b/core/loop_watchdog.py
@@ -0,0 +1,122 @@
+"""Report an event loop that stops turning, with the stack that stopped it.
+
+A synchronous call on the service's loop -- a lock wait, a subprocess, a large
+file rewrite -- stops every coroutine with it, and nothing on that loop can say
+so while it happens: its own timers fire late, afterwards, with no trace of what
+held them. A thread outside the loop can. It asks the loop to answer once a
+second and, when the loop has not answered for ``STALL_SECONDS``, logs one
+WARNING with the stack of every thread, so the blocking call is named in the
+log while it is still blocking. When the loop answers again it logs how long
+the stall lasted.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import sys
+import threading
+import time
+import traceback
+from collections.abc import Callable
+
+logger = logging.getLogger(__name__)
+
+STALL_SECONDS = 5.0
+POLL_SECONDS = 1.0
+# At most one stack report per this long; a loop that keeps stalling still gets
+# its recovery lines, which carry each stall's length.
+REPORT_INTERVAL_SECONDS = 300.0
+# Innermost frames kept per thread, and the report's overall size.
+_FRAMES_PER_THREAD = 30
+_MAX_REPORT_CHARS = 64 * 1024
+
+
+class EventLoopWatchdog:
+ """A daemon thread that logs the stacks of a loop that has not answered for ``stall_seconds``."""
+
+ def __init__(
+ self,
+ loop: asyncio.AbstractEventLoop,
+ *,
+ stall_seconds: float = STALL_SECONDS,
+ poll_seconds: float = POLL_SECONDS,
+ report_interval_seconds: float = REPORT_INTERVAL_SECONDS,
+ monotonic: Callable[[], float] = time.monotonic,
+ ) -> None:
+ self._loop = loop
+ self._stall_seconds = stall_seconds
+ self._poll_seconds = poll_seconds
+ self._report_interval_seconds = report_interval_seconds
+ self._monotonic = monotonic
+ self._stopped = threading.Event()
+ self._thread: threading.Thread | None = None
+ self._answered_at = monotonic()
+ self._asking = False
+ self._loop_thread_id: int | None = None
+ self._last_report_at: float | None = None
+ self._stall_reported = False
+
+ def start(self) -> None:
+ if self._thread is not None:
+ return
+ self._answered_at = self._monotonic()
+ self._thread = threading.Thread(target=self._run, name="event-loop-watchdog", daemon=True)
+ self._thread.start()
+
+ def stop(self) -> None:
+ self._stopped.set()
+ thread, self._thread = self._thread, None
+ if thread is not None and thread is not threading.current_thread():
+ thread.join(timeout=self._poll_seconds * 2)
+
+ def _answer(self) -> None:
+ # Runs on the loop: the only work it adds there is two assignments.
+ self._loop_thread_id = threading.get_ident()
+ self._answered_at = self._monotonic()
+ self._asking = False
+
+ def _run(self) -> None:
+ silent_since = self._answered_at
+ while not self._stopped.wait(self._poll_seconds):
+ answered_at = self._answered_at
+ if self._stall_reported and answered_at > silent_since:
+ logger.info("Event loop answered again after a %.1fs stall", answered_at - silent_since)
+ self._stall_reported = False
+ silent_since = answered_at
+ if not self._asking:
+ self._asking = True
+ try:
+ self._loop.call_soon_threadsafe(self._answer)
+ except RuntimeError:
+ return # the loop is closed
+ now = self._monotonic()
+ if self._stall_reported or now - answered_at < self._stall_seconds:
+ continue
+ self._stall_reported = True
+ if self._last_report_at is not None and now - self._last_report_at < self._report_interval_seconds:
+ continue
+ self._last_report_at = now
+ logger.warning(
+ "Event loop has not answered for %.1fs; thread stacks follow.\n%s",
+ now - answered_at,
+ self._thread_stacks(),
+ )
+
+ def _thread_stacks(self) -> str:
+ names = {thread.ident: thread.name for thread in threading.enumerate()}
+ frames = sys._current_frames()
+ # A loop that blocks before its first answer is still named: asyncio records its running thread.
+ loop_thread = self._loop_thread_id or getattr(self._loop, "_thread_id", None)
+ order = sorted(frames, key=lambda ident: (ident != loop_thread, names.get(ident, "")))
+ sections = []
+ for ident in order:
+ if ident == threading.get_ident():
+ continue
+ label = "event loop" if ident == loop_thread else names.get(ident, "unnamed")
+ stack = "".join(traceback.format_stack(frames[ident])[-_FRAMES_PER_THREAD:])
+ sections.append(f"--- thread {ident} ({label}) ---\n{stack}")
+ report = "".join(sections)
+ if len(report) > _MAX_REPORT_CHARS:
+ report = report[:_MAX_REPORT_CHARS] + "\n... (truncated)"
+ return report
diff --git a/tests/test_loop_watchdog.py b/tests/test_loop_watchdog.py
new file mode 100644
index 000000000..68ddc136f
--- /dev/null
+++ b/tests/test_loop_watchdog.py
@@ -0,0 +1,76 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+import threading
+import time
+
+from core.loop_watchdog import EventLoopWatchdog
+
+
+def _wait_for(condition, timeout: float = 5.0) -> None:
+ deadline = time.monotonic() + timeout
+ while not condition():
+ if time.monotonic() > deadline:
+ raise AssertionError("condition not reached")
+ time.sleep(0.01)
+
+
+def _block_loop_until(released: threading.Event) -> None:
+ # The name is what the report has to show.
+ released.wait(10)
+
+
+def _messages(caplog, level: int) -> list[str]:
+ return [record.getMessage() for record in caplog.records if record.levelno == level and record.name == "core.loop_watchdog"]
+
+
+def test_a_blocked_loop_is_reported_once_with_the_blocking_stack_and_its_length(caplog) -> None:
+ caplog.set_level(logging.INFO, logger="core.loop_watchdog")
+ loop = asyncio.new_event_loop()
+ runner = threading.Thread(target=loop.run_forever, daemon=True)
+ runner.start()
+ watchdog = EventLoopWatchdog(loop, stall_seconds=0.3, poll_seconds=0.05, report_interval_seconds=60)
+ try:
+ watchdog.start()
+ released = threading.Event()
+ loop.call_soon_threadsafe(_block_loop_until, released)
+ _wait_for(lambda: _messages(caplog, logging.WARNING))
+ # A stall keeps being one report, however long it lasts.
+ time.sleep(0.5)
+ released.set()
+ _wait_for(lambda: _messages(caplog, logging.INFO))
+
+ # A second stall inside the report interval adds only its recovery line.
+ second = threading.Event()
+ loop.call_soon_threadsafe(_block_loop_until, second)
+ time.sleep(0.6)
+ second.set()
+ _wait_for(lambda: len(_messages(caplog, logging.INFO)) == 2)
+ finally:
+ watchdog.stop()
+ loop.call_soon_threadsafe(loop.stop)
+ runner.join(5)
+ loop.close()
+
+ (warning,) = _messages(caplog, logging.WARNING)
+ assert "Event loop has not answered for" in warning
+ assert "(event loop)" in warning and "_block_loop_until" in warning
+ assert all("Event loop answered again after a" in line for line in _messages(caplog, logging.INFO))
+
+
+def test_a_turning_loop_is_never_reported(caplog) -> None:
+ caplog.set_level(logging.INFO, logger="core.loop_watchdog")
+ loop = asyncio.new_event_loop()
+ runner = threading.Thread(target=loop.run_forever, daemon=True)
+ runner.start()
+ watchdog = EventLoopWatchdog(loop, stall_seconds=0.3, poll_seconds=0.05)
+ try:
+ watchdog.start()
+ time.sleep(1.0)
+ finally:
+ watchdog.stop()
+ loop.call_soon_threadsafe(loop.stop)
+ runner.join(5)
+ loop.close()
+ assert _messages(caplog, logging.WARNING) == [] and _messages(caplog, logging.INFO) == []