Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(live): keep the runner driver alive for the scheduler's lifetime

Romzhoua66511312016
diff --git a/agent/src/api/live_routes.py b/agent/src/api/live_routes.py
index 0b7e066ce..39735c020 100644
--- a/agent/src/api/live_routes.py
+++ b/agent/src/api/live_routes.py
@@ -722,14 +722,79 @@ def _build_live_runner(broker: str) -> Any:
return runner
+def _runner_scheduler_task(runner: Any) -> Optional["asyncio.Task[Any]"]:
+ """Return the live task driving the runner's scheduler, or ``None``.
+
+ Neither ``LiveRunner`` nor ``Scheduler`` exposes a public accessor for the
+ scheduler loop task (``LiveRunner`` only publishes ``runner_id``;
+ ``Scheduler``'s public surface is ``start``/``stop``/``add_job``/
+ ``remove_job``/``jobs``), so both hops are read defensively with
+ ``getattr``. A runner whose ``run_loop`` declined to start — no mandate, or
+ an expired one (``runner.py`` returns before calling ``Scheduler.start``) —
+ leaves ``_task`` unset and has nothing to await.
+ """
+ task = getattr(getattr(runner, "_scheduler", None), "_task", None)
+ if task is None or task.done():
+ return None
+ return task
+
+
+async def _stop_scheduler(runner: Any) -> None:
+ """Tear the runner's scheduler down, tolerating a sync or async ``stop``.
+
+ ``Scheduler.stop`` is a coroutine function, but the ``_Scheduler`` protocol
+ in ``runner.py`` types ``stop`` as returning ``Any``, so injected doubles may
+ be plain callables. Teardown is best-effort: a scheduler that refuses to
+ stop must not turn into an unhandled error inside the driver task.
+ """
+ stop = getattr(getattr(runner, "_scheduler", None), "stop", None)
+ if stop is None:
+ return
+ try:
+ result = stop()
+ if asyncio.iscoroutine(result):
+ await result
+ except Exception: # noqa: BLE001 - teardown is best-effort by contract
+ logger.warning(
+ "live scheduler teardown failed for %s",
+ getattr(runner, "broker", "?"),
+ exc_info=True,
+ )
+
+
async def _drive_runner(runner: Any) -> None:
- """Run a runner's ``run_loop`` to completion, sync or async."""
+ """Run a runner's ``run_loop`` and stay alive for as long as its scheduler.
+
+ ``run_loop`` is fire-and-forget: it resolves the jobs, calls
+ ``Scheduler.start()`` (which only spawns the loop task) and returns. The
+ task wrapping this coroutine is the *only* handle
+ ``POST /live/runner/stop`` holds, and its ``add_done_callback`` unregisters
+ it the moment this coroutine returns — so a driver that returned alongside
+ ``run_loop`` would leave the scheduler firing ``run_once()`` on its cadence
+ while ``stop`` answered ``was_running: false``.
+
+ Keeping this coroutine parked on the scheduler task makes the driver's
+ lifetime the scheduler's lifetime, so cancelling it actually stops trading.
+ """
result = runner.run_loop()
if asyncio.iscoroutine(result):
await result
else:
await asyncio.get_running_loop().run_in_executor(None, lambda: result)
+ scheduler_task = _runner_scheduler_task(runner)
+ if scheduler_task is None:
+ # ``run_loop`` declined to start (missing or expired mandate): there is
+ # no scheduler to own, so the driver returns immediately as before.
+ return
+
+ try:
+ # Shielded: cancelling this task is precisely what ``stop`` does, and the
+ # teardown below is what has to reach the scheduler loop task.
+ await asyncio.shield(scheduler_task)
+ finally:
+ await _stop_scheduler(runner)
+
# ============================================================================
# Route registration
diff --git a/agent/tests/test_api_live_runtime.py b/agent/tests/test_api_live_runtime.py
index 11798d2fd..74f8f4290 100644
--- a/agent/tests/test_api_live_runtime.py
+++ b/agent/tests/test_api_live_runtime.py
@@ -13,10 +13,12 @@ All tests run against stubbed runner/liveness state — no real agent or broker.
from __future__ import annotations
+import asyncio
import json
from pathlib import Path
from types import SimpleNamespace
+import httpx
from fastapi.testclient import TestClient
import pytest
@@ -242,6 +244,270 @@ def test_runner_stop_cancels_running_task(tmp_path: Path, monkeypatch) -> None:
assert cancelled["value"] is True
+# --------------------------------------------------------------------------- #
+# Runner stop must actually stop the scheduler (R-INT lifecycle)
+# --------------------------------------------------------------------------- #
+
+
+def _live_env(tmp_path: Path, monkeypatch) -> None:
+ """Sandbox the runtime root and the module-level runner registries."""
+ monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path), raising=False)
+ monkeypatch.setattr(api_server, "_runner_tasks", {}, raising=False)
+ monkeypatch.setattr(api_server, "_runner_factory", None, raising=False)
+
+
+def _live_client() -> httpx.AsyncClient:
+ """An ASGI client bound to the *running* loop (no lifespan, no portal thread).
+
+ ``TestClient`` opens a throwaway event loop per request unless it is used as a
+ context manager, and the context-manager form additionally pays the app
+ lifespan (~10s here). The driver task started by ``/live/runner/start``
+ outlives the request that created it, so these tests drive the app on a
+ single long-lived loop — which is what uvicorn does in production.
+
+ The base URL is loopback so the DNS-rebinding Host check and the loopback
+ auth trust behave exactly as they do for ``_client`` above.
+ """
+ return httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=api_server.app, client=("127.0.0.1", 50000)),
+ base_url="http://127.0.0.1:50000",
+ )
+
+
+class _StubScheduler:
+ """Scheduler stub mirroring ``Scheduler``'s ``_task`` / async ``stop`` contract."""
+
+ def __init__(self) -> None:
+ self._task: asyncio.Task | None = None
+ self.stop_calls = 0
+ self.ticks = 0
+
+ def start(self) -> None:
+ async def _loop() -> None:
+ while True: # a real scheduler never returns on its own
+ self.ticks += 1
+ await asyncio.sleep(0)
+
+ self._task = asyncio.get_running_loop().create_task(_loop(), name="live-scheduler")
+
+ async def stop(self) -> None:
+ self.stop_calls += 1
+ task, self._task = self._task, None
+ if task is None:
+ return
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ @property
+ def running(self) -> bool:
+ return self._task is not None and not self._task.done()
+
+
+def _install_runner(monkeypatch, broker: str = "robinhood", *, start: bool = True):
+ """Stub the runner factory; return the ``_StubScheduler`` the runner starts."""
+ monkeypatch.setattr(
+ api_server, "_active_mandate_state", lambda b: _valid_mandate_state(b)
+ )
+ scheduler = _StubScheduler()
+
+ class _StubRunner:
+ def __init__(self) -> None:
+ self.broker = broker
+ self._scheduler = scheduler
+
+ def run_loop(self, jobs=None) -> None: # sync, non-blocking, returns None
+ if start:
+ self._scheduler.start()
+
+ monkeypatch.setattr(api_server, "_runner_factory", lambda b: _StubRunner())
+ return scheduler
+
+
+async def _await_ticks(scheduler: _StubScheduler, ticks: int = 5) -> None:
+ """Let the scheduler loop actually run, so "it is still alive" is meaningful."""
+ for _ in range(200):
+ if scheduler.ticks >= ticks:
+ return
+ await asyncio.sleep(0.001)
+ raise AssertionError("scheduler loop never ticked")
+
+
+async def _await_stopped(scheduler: _StubScheduler) -> None:
+ """Let the cancelled driver run its teardown."""
+ for _ in range(200):
+ if not scheduler.running:
+ return
+ await asyncio.sleep(0.001)
+ raise AssertionError("scheduler loop never stopped")
+
+
+def test_runner_start_keeps_task_registered_while_scheduler_runs(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """The driver task must stay registered for as long as the scheduler ticks.
+
+ Regression: ``_drive_runner`` returned as soon as ``run_loop`` did, so the
+ ``add_done_callback`` popped ``_runner_tasks[broker]`` while the scheduler
+ was still firing — leaving the runner unstoppable from the API.
+ """
+ _live_env(tmp_path, monkeypatch)
+ scheduler = _install_runner(monkeypatch)
+
+ async def scenario() -> None:
+ async with _live_client() as client:
+ response = await client.post("/live/runner/start", json={"broker": "robinhood"})
+ assert response.status_code == 200, response.text
+ assert response.json() == {
+ "broker": "robinhood",
+ "started": True,
+ "already_running": False,
+ }
+
+ await _await_ticks(scheduler)
+ # Give a wrongly-finished driver every chance to finish and unregister.
+ for _ in range(5):
+ await asyncio.sleep(0)
+
+ task = api_server._runner_tasks.get("robinhood")
+ assert task is not None, (
+ "driver task was unregistered while the scheduler was still running"
+ )
+ assert not task.done(), "driver task completed while the scheduler was running"
+ assert scheduler.running is True
+
+ await client.post("/live/runner/stop", json={"broker": "robinhood"})
+ await _await_stopped(scheduler)
+
+ asyncio.run(scenario())
+
+
+def test_runner_stop_actually_stops_the_scheduler(tmp_path: Path, monkeypatch) -> None:
+ """``POST /live/runner/stop`` must stop the scheduler, not just a wrapper task.
+
+ Regression: the stop endpoint reported ``was_running: false`` because the
+ wrapper task had already been popped, and the scheduler kept ticking.
+ """
+ _live_env(tmp_path, monkeypatch)
+ scheduler = _install_runner(monkeypatch)
+
+ async def scenario() -> None:
+ async with _live_client() as client:
+ started = await client.post("/live/runner/start", json={"broker": "robinhood"})
+ assert started.status_code == 200, started.text
+ await _await_ticks(scheduler)
+
+ response = await client.post("/live/runner/stop", json={"broker": "robinhood"})
+
+ assert response.status_code == 200
+ assert response.json() == {"broker": "robinhood", "stopped": True, "was_running": True}
+
+ await _await_stopped(scheduler)
+ assert scheduler.running is False, "scheduler loop still running after stop"
+ assert scheduler.stop_calls == 1, "scheduler.stop() was not awaited"
+ assert "robinhood" not in api_server._runner_tasks
+
+ asyncio.run(scenario())
+
+
+def test_runner_stop_is_idempotent_after_a_real_stop(tmp_path: Path, monkeypatch) -> None:
+ """A second stop on an already-stopped runner stays a no-op."""
+ _live_env(tmp_path, monkeypatch)
+ scheduler = _install_runner(monkeypatch)
+
+ async def scenario() -> None:
+ async with _live_client() as client:
+ await client.post("/live/runner/start", json={"broker": "robinhood"})
+ await _await_ticks(scheduler)
+
+ first = await client.post("/live/runner/stop", json={"broker": "robinhood"})
+ assert first.json() == {"broker": "robinhood", "stopped": True, "was_running": True}
+ await _await_stopped(scheduler)
+
+ second = await client.post("/live/runner/stop", json={"broker": "robinhood"})
+ assert second.status_code == 200
+ assert second.json() == {
+ "broker": "robinhood",
+ "stopped": False,
+ "was_running": False,
+ }
+ assert scheduler.stop_calls == 1
+
+ asyncio.run(scenario())
+
+
+def test_runner_start_without_a_scheduler_task_still_returns(
+ tmp_path: Path, monkeypatch
+) -> None:
+ """``run_loop`` declining to start (no mandate) must not pin the driver forever.
+
+ ``LiveRunner.run_loop`` returns early when the mandate is missing or expired
+ and never calls ``Scheduler.start()``, so there is no task to await and the
+ driver must settle immediately — preserving the legacy behaviour.
+ """
+ _live_env(tmp_path, monkeypatch)
+ scheduler = _install_runner(monkeypatch, start=False)
+
+ async def scenario() -> None:
+ async with _live_client() as client:
+ response = await client.post("/live/runner/start", json={"broker": "robinhood"})
+ assert response.status_code == 200, response.text
+
+ for _ in range(200):
+ if "robinhood" not in api_server._runner_tasks:
+ break
+ await asyncio.sleep(0.001)
+ assert "robinhood" not in api_server._runner_tasks
+ assert scheduler.stop_calls == 0
+
+ stopped = await client.post("/live/runner/stop", json={"broker": "robinhood"})
+ assert stopped.json() == {
+ "broker": "robinhood",
+ "stopped": False,
+ "was_running": False,
+ }
+
+ asyncio.run(scenario())
+
+
+def test_drive_runner_awaits_an_async_run_loop_and_stops_the_scheduler() -> None:
+ """An ``async def run_loop`` is awaited, and its scheduler is still stopped."""
+ scheduler = _StubScheduler()
+
+ class _AsyncRunner:
+ broker = "robinhood"
+
+ def __init__(self) -> None:
+ self._scheduler = scheduler
+
+ async def run_loop(self, jobs=None) -> None:
+ await asyncio.sleep(0)
+ self._scheduler.start()
+
+ runner = _AsyncRunner()
+
+ async def scenario() -> None:
+ task = asyncio.ensure_future(api_server._drive_runner(runner))
+ for _ in range(200):
+ if scheduler.ticks >= 5:
+ break
+ await asyncio.sleep(0.001)
+ assert scheduler.ticks >= 5, "scheduler loop never started"
+
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+
+ asyncio.run(scenario())
+
+ assert scheduler.running is False
+ assert scheduler.stop_calls == 1
+
+
# --------------------------------------------------------------------------- #
# C1 — propose_mandate_profiles tool_result -> mandate.proposal SSE frame
# --------------------------------------------------------------------------- #