Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

test: bound the event and clock gates with one shared hang guard (#2405)

Call Whitef3980603792d
diff --git a/tests/agent_core/tools/test_bash.py b/tests/agent_core/tools/test_bash.py
index 0190ebdaa..b5f12cc7e 100644
--- a/tests/agent_core/tools/test_bash.py
+++ b/tests/agent_core/tools/test_bash.py
@@ -19,6 +19,7 @@ import core.agent_core.tools.bash as bash_module
from core.agent_core.tools.bash import BashTool, settle_bash_call
import core.agent_core.tools.jobs as jobs_module
from core.agent_core.tools.jobs import LocalJobHost
+from tests import hang_guard
from tests.agent_core.tools.conftest import instance, result_text
LABEL = "Output log (first 1.0MB, then the last 2.0MB in tail.log beside it)"
@@ -152,10 +153,17 @@ async def test_only_the_recorded_reason_makes_a_gone_job_a_timeout(tmp_path, mak
go = tmp_path / "go"
command = f"until [ -e {go} ]; do sleep 0.01; done; seq 1 1000; sleep 30"
running = asyncio.ensure_future(BashTool(host).execute({"command": command, "timeout": 1.0}, make_ctx()))
- job_id = None
- while job_id is None or not os.path.exists(os.path.join(host.job_dir(job_id), "pid")):
- await asyncio.sleep(0.01)
- job_id = job_id or host.find_job(instance())
+ found: list[str] = []
+
+ def wrapper_started() -> bool:
+ # A call that ended before its wrapper wrote a pid surfaces its own result here.
+ assert not running.done(), running.result()
+ if not found and (job_id := host.find_job(instance())) is not None:
+ found.append(job_id)
+ return bool(found) and os.path.exists(os.path.join(host.job_dir(found[0]), "pid"))
+
+ await hang_guard.until(wrapper_started, "the job wrapper to write its pid")
+ job_id = found[0]
# The wrapper cannot write its tail snapshot once the head is full, as on a full disk.
wrapper_pid = open(os.path.join(host.job_dir(job_id), "pid")).read().strip()
os.mkdir(os.path.join(host.job_dir(job_id), f"tail.log.{wrapper_pid}.tmp"))
@@ -433,11 +441,14 @@ async def test_watch_true_returns_at_once(tmp_path, make_ctx, monkeypatch):
async def next_read() -> None:
clock.read.clear()
reading = asyncio.ensure_future(clock.read.wait())
- await asyncio.wait({reading, running}, return_when=asyncio.FIRST_COMPLETED)
+ await hang_guard.within(
+ asyncio.wait({reading, running}, return_when=asyncio.FIRST_COMPLETED), "bash to read its clock or end"
+ )
reading.cancel()
try:
while clock.now != 100.0:
+ assert not running.done(), running.result() # a call that failed to start
await next_read()
clock.now = 100.49
# bash reads 100.49, then comes back for another reading: it decided to keep waiting.
diff --git a/tests/hang_guard.py b/tests/hang_guard.py
new file mode 100644
index 000000000..a35fc9138
--- /dev/null
+++ b/tests/hang_guard.py
@@ -0,0 +1,114 @@
+"""One generous bound for test gates that wait on events, not on time.
+
+A gate that waits for a worker, a thread, a process or a clock reading ends on a
+state change, never on a guess of how long the work takes. When that state never
+comes, because a regression hangs, the gate still fails loudly inside its test
+instead of leaving the CI per-file watchdog to kill the whole file.
+``HANG_GUARD_SECONDS`` is far longer than any loaded runner needs, so it never
+decides a passing outcome.
+
+Invariant: when a gate fails, no worker it owns keeps running. Every stuck
+process is killed before the failure. A stuck thread cannot be stopped, and one
+left running would act inside the tests that follow, so the run ends instead
+(``pytest.exit``): that process can no longer isolate tests.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import multiprocessing.connection
+import queue
+import threading
+import time
+from typing import Any, Awaitable, Callable, Iterable
+
+import pytest
+
+HANG_GUARD_SECONDS = 120.0
+
+
+def _hung(what: str) -> None:
+ __tracebackhide__ = True
+ pytest.fail(f"hung for {HANG_GUARD_SECONDS:.0f}s waiting for {what}")
+
+
+def wait(event: Any, what: str) -> None:
+ """``event.wait()`` for a threading or multiprocessing Event."""
+ __tracebackhide__ = True
+ if not event.wait(HANG_GUARD_SECONDS):
+ _hung(what)
+
+
+def join(worker: Any, what: str) -> None:
+ """``worker.join()`` for a daemon Thread or a Process; see ``join_all``."""
+ __tracebackhide__ = True
+ join_all([worker], what)
+
+
+def join_all(workers: Iterable[Any], what: str) -> None:
+ """Join every worker within one shared bound before failing for any of them.
+
+ Every stuck process is killed first, so none outlives its test. A stuck thread
+ ends the run (see the module invariant); it must be a daemon, so that it cannot
+ hold the interpreter open at exit either.
+ """
+ __tracebackhide__ = True
+ workers = list(workers)
+ for worker in workers:
+ if isinstance(worker, threading.Thread) and not worker.daemon:
+ raise ValueError(f"hang_guard needs a daemon thread, so a hang cannot block exit: {worker!r}")
+ give_up = time.monotonic() + HANG_GUARD_SECONDS
+ for worker in workers:
+ worker.join(max(0.0, give_up - time.monotonic()))
+ stuck = [worker for worker in workers if worker.is_alive()]
+ for worker in stuck:
+ if not isinstance(worker, threading.Thread):
+ worker.kill()
+ worker.join()
+ stuck_threads = [worker for worker in stuck if isinstance(worker, threading.Thread)]
+ if stuck_threads:
+ pytest.exit(
+ f"hung for {HANG_GUARD_SECONDS:.0f}s waiting for {what}: {len(stuck_threads)} thread(s) "
+ "still running, and a thread cannot be stopped, so this process can no longer "
+ "isolate tests",
+ returncode=pytest.ExitCode.TESTS_FAILED,
+ )
+ if stuck:
+ _hung(f"{what} ({len(stuck)} of {len(workers)} still running)")
+
+
+def get(items: queue.SimpleQueue | queue.Queue, what: str) -> Any:
+ """``items.get()``."""
+ __tracebackhide__ = True
+ try:
+ return items.get(timeout=HANG_GUARD_SECONDS)
+ except queue.Empty:
+ _hung(what)
+
+
+def ready(waitables: Iterable[Any], what: str) -> list[Any]:
+ """``multiprocessing.connection.wait(waitables)``: the ones that became ready."""
+ __tracebackhide__ = True
+ woke = multiprocessing.connection.wait(list(waitables), timeout=HANG_GUARD_SECONDS)
+ if not woke:
+ _hung(what)
+ return woke
+
+
+async def within(awaitable: Awaitable[Any], what: str) -> Any:
+ """``await awaitable``."""
+ __tracebackhide__ = True
+ try:
+ return await asyncio.wait_for(awaitable, HANG_GUARD_SECONDS)
+ except asyncio.TimeoutError:
+ _hung(what)
+
+
+async def until(predicate: Callable[[], Any], what: str, *, interval: float = 0.01) -> None:
+ """Poll ``predicate`` until it holds; for state with no event to wait on."""
+ __tracebackhide__ = True
+ give_up = time.monotonic() + HANG_GUARD_SECONDS
+ while not predicate():
+ if time.monotonic() >= give_up:
+ _hung(what)
+ await asyncio.sleep(interval)
diff --git a/tests/test_pairing_recovery.py b/tests/test_pairing_recovery.py
index e8bd655df..eb781e139 100644
--- a/tests/test_pairing_recovery.py
+++ b/tests/test_pairing_recovery.py
@@ -6,7 +6,6 @@ import ipaddress
import inspect
import json
import multiprocessing
-import multiprocessing.connection
import os
from pathlib import Path
import queue
@@ -19,6 +18,7 @@ import pytest
from config import paths
from config.v2_config import RemoteAccessConfig, V2Config
from storage import remote_access_authorization_service
+from tests import hang_guard
from tests.test_remote_access_vibe_cloud import _config
from tests.ui_server_test_helpers import csrf_headers
from vibe import api, cli, remote_access, ui_server
@@ -116,16 +116,17 @@ class _ThreadRedeemGate:
finally:
self._events.put("finished")
- self._worker = threading.Thread(target=run)
+ self._worker = threading.Thread(target=run, daemon=True)
self._worker.start()
return self
def wait_entered(self):
- assert self._events.get() == "entered", "pair() finished before it reached redeem"
+ event = hang_guard.get(self._events, "the pairing worker to reach redeem")
+ assert event == "entered", "pair() finished before it reached redeem"
def finish(self):
self._release.set()
- self._worker.join()
+ hang_guard.join(self._worker, "the released pairing worker to finish")
return self.result
@@ -148,14 +149,16 @@ class _ProcessRedeemGate:
return self._worker
def wait_entered(self):
- ready = multiprocessing.connection.wait([self._entered_reader, self._worker.sentinel])
+ ready = hang_guard.ready(
+ [self._entered_reader, self._worker.sentinel], "the pairing child to reach redeem or exit"
+ )
assert self._entered_reader in ready, (
f"worker exited with {self._worker.exitcode} before it reached redeem"
)
def finish(self):
self._release.set()
- self._worker.join()
+ hang_guard.join(self._worker, "the released pairing child to exit")
return self._worker
@@ -1512,12 +1515,11 @@ def test_concurrent_resume_has_one_owner(pairing_host, monkeypatch):
barrier.wait()
results.append(remote_access.pair("", ""))
- workers = [threading.Thread(target=resume) for _ in range(2)]
+ workers = [threading.Thread(target=resume, daemon=True) for _ in range(2)]
for worker in workers:
worker.start()
- barrier.wait()
- for worker in workers:
- worker.join()
+ barrier.wait(hang_guard.HANG_GUARD_SECONDS)
+ hang_guard.join_all(workers, "the concurrent resumes to finish")
assert sum(bool(result.get("ok")) for result in results) == 1
assert sum(result.get("error") == "missing_pairing_key" for result in results) == 1
assert V2Config.load().remote_access.vibe_cloud.instance_id == "inst_A"
@@ -1545,12 +1547,11 @@ def test_concurrent_resume_has_one_owner_across_processes(pairing_host, monkeypa
worker.start()
try:
for (ready, _), worker in zip(readies, workers):
- woke = multiprocessing.connection.wait([ready, worker.sentinel])
+ woke = hang_guard.ready([ready, worker.sentinel], "a resume child to be ready or exit")
assert ready in woke, f"worker exited with {worker.exitcode} before it was ready"
finally:
start.set()
- for worker in workers:
- worker.join()
+ hang_guard.join_all(workers, "the resume children to exit")
for worker in workers:
assert worker.exitcode == 0
results = [