"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(live): latch the halt sweep on attempted side effects, not on reads succeeding
cgycorey5558f387e488
diff --git a/agent/src/api/live_routes.py b/agent/src/api/live_routes.pyindex 40f355563..2353f3122 100644--- a/agent/src/api/live_routes.py+++ b/agent/src/api/live_routes.py@@ -565,8 +565,42 @@ def _build_live_runner(broker: str) -> Any:# _live_broker_adapter is monkeypatched on host by testsadapter = h._live_broker_adapter(broker)- def _read(remote_tool: str):- return lambda: adapter.call_tool(remote_tool, {})+ def _read(remote_tool: str, record_key: str | None = None):+ """Wrap adapter.call_tool into a READ callable with a normalized payload.++ The MCP adapter returns ``{"status": "error", ...}`` envelopes instead+ of raising, and wraps successful values in ``{"status": "ok",+ "data": ...}``. Normalize at the boundary so consumers (reconcile and+ the halt sweep) receive broker records — or the balance dict — not+ envelopes: error envelopes raise, success payloads unwrap to ``data``+ (with the pinned ``record_key`` extracted for positions/orders record+ reads, e.g. ``data.positions`` / ``data.orders``). Bare values pass+ through unchanged.+ """+ def read() -> Any:+ payload = adapter.call_tool(remote_tool, {})+ if isinstance(payload, list):+ return payload # injected/legacy bare response+ if not isinstance(payload, dict) or payload.get("status") != "ok":+ if isinstance(payload, dict) and payload.get("status") == "error":+ raise RuntimeError(+ str(payload.get("error") or "broker read failed")+ )+ return payload+ data = payload.get("data")+ if record_key is None:+ return data+ if isinstance(data, list):+ return data+ if isinstance(data, dict):+ records = data.get(record_key)+ if isinstance(records, list):+ return records+ raise RuntimeError(+ f"broker read response missing {record_key!r} records"+ )+ raise RuntimeError("broker read response has no data payload")+ return readdef _submit(order: Dict[str, Any]) -> Dict[str, Any]:if order.get("action") == "cancel":@@ -599,9 +633,9 @@ def _build_live_runner(broker: str) -> Any:broker,agent_caller=_agent_caller,reconcile_fn=reconcile,- read_positions=_read(positions_tool),+ read_positions=_read(positions_tool, "positions"),read_balance=_read(balance_tool),- read_open_orders=_read(open_orders_tool),+ read_open_orders=_read(open_orders_tool, "orders"),submit_fn=_submit,write_audit_fn=_audit_with_bus,scheduler=scheduler,diff --git a/agent/src/live/runtime/runner.py b/agent/src/live/runtime/runner.pyindex b41cb04d5..aa0afddb5 100644--- a/agent/src/live/runtime/runner.py+++ b/agent/src/live/runtime/runner.py@@ -48,8 +48,8 @@ from src.live.runtime.flatten import flatten_and_cancelfrom src.live.runtime.jobstore import JobStorefrom src.live.runtime.sweep_latch import mark_sweep_fired, sweep_already_firedfrom src.live.runtime.liveness import write_heartbeat-from src.live.runtime.scheduler import Job, Scheduler-from src.live.runtime.triggers import Trigger, due_now+from src.live.runtime.scheduler import Job+from src.live.runtime.triggers import Triggerlogger = logging.getLogger(__name__)@@ -572,25 +572,38 @@ class LiveRunner:Invoked the moment the runner observes a tripped HALT. Idempotent acrossticks via the ``_flatten_fired`` latch so the no-retry rule (SPEC §8.5)- holds even if the halted runner keeps waking. The latch is also persisted- on disk (``sweep_latch``) so a restart with flatten orders still working- does not replay the sweep. A sweep failure is audited but not retried —- flatten/cancel side effects are not idempotent.+ holds even if the halted runner keeps waking. The latch is persisted on+ disk (``sweep_latch``) so a restart with flatten orders still working+ does not replay the sweep.++ The persisted latch is written only when the sweep actually attempted+ a broker write. ``flatten_and_cancel`` reports+ ``side_effects_attempted`` — set the moment any cancel/close submit is+ tried. When that flag is false, no cancel or close was attempted+ (reads failed, or there was nothing to do), so nothing happened that a+ later tick (or a restart) must not repeat: latching would silently+ skip the kill switch's action for that episode. Any attempted broker+ write — even one that errors, and even when a later read phase fails —+ latches: those are the non-idempotent cases SPEC §8.5 protects. A+ sweep that raises outright latches too (side effects may have begun)."""if self._flatten_fired or self._submit_fn is None:return- self._flatten_fired = Trueif sweep_already_fired(self.broker):+ self._flatten_fired = Truereturn- mark_sweep_fired(self.broker)try:- self._flatten_fn(+ report = self._flatten_fn(self.broker,self._submit_fn,self._read_positions,self._read_open_orders,)except Exception as exc: # noqa: BLE001 — surfaced via audit, never retried+ # Unknown failure: cancel/flatten side effects may have begun and+ # are not retryable — latch so a restart does not replay closes.+ self._flatten_fired = True+ mark_sweep_fired(self.broker)logger.exception("preemptive flatten failed for %s", self.broker)self._audit(kind="breach",@@ -598,6 +611,40 @@ class LiveRunner:intent="preemptive halt sweep failed — not retried (no-retry §8.5)",error=str(exc),)+ return+ report = report or {}+ read_errors = [+ entry+ for entry in report.get("errors", [])+ if isinstance(entry, dict)+ and entry.get("phase") in ("read_open_orders", "read_positions")+ ]+ if read_errors:+ detail = "; ".join(str(entry.get("error")) for entry in read_errors)+ logger.warning(+ "preemptive sweep read failure for %s during execution: %s",+ self.broker,+ detail,+ )+ if not report.get("side_effects_attempted"):+ # No cancel or close was attempted → nothing happened that a+ # later tick (or a restart) must not repeat. Do NOT latch: the+ # sweep retries for this episode.+ if read_errors:+ detail = "; ".join(str(e.get("error")) for e in read_errors)+ intent = "preemptive halt sweep read failed — retrying on next tick"+ else:+ detail = "no open orders or positions to act on"+ intent = "preemptive halt sweep found nothing to act on — re-checking on next tick"+ self._audit(+ kind="breach",+ outcome="error",+ intent=intent,+ error=detail,+ )+ return+ self._flatten_fired = True+ mark_sweep_fired(self.broker)def _no_mandate_result(self) -> dict[str, Any]:"""Audit + return the result when no valid mandate is on file."""diff --git a/agent/src/live/runtime/sweep_latch.py b/agent/src/live/runtime/sweep_latch.pyindex 5aaaadd18..12d3a760f 100644--- a/agent/src/live/runtime/sweep_latch.py+++ b/agent/src/live/runtime/sweep_latch.py@@ -8,12 +8,15 @@ from long to net short. This module records the sweep's firing on disk, boundto the halt *episode* that caused it.The latch lives next to the per-broker HALT sentinel at-``<runtime_root>/live/<broker>/FLATTEN_FIRED`` and carries the episode identity-of the halt that was tripped when the sweep fired: the sentinel's ``tripped_at``-when it has one, otherwise the sentinel file's mtime. A later trip writes a new-sentinel (fresh ``tripped_at`` / fresh mtime), so a stale latch from a previous-episode never suppresses a new one; clearing HALT and re-tripping therefore-re-arms the sweep without anyone deleting the latch.+``<runtime_root>/live/<broker>/FLATTEN_FIRED`` and carries the identity of+every halt *episode* the sweep has fired for: the sentinel's ``tripped_at``+when it has one, otherwise the sentinel file's mtime. A later trip writes a+new sentinel (fresh ``tripped_at`` / fresh mtime), so a stale latch from a+previous episode never suppresses a new one; clearing HALT and re-tripping+therefore re-arms the sweep without anyone deleting the latch. When the+per-broker and the global HALTs are both tripped, the *newest* sentinel wins+the current-episode lookup — the global HALT is authoritative+(``halt_flag_set`` halts every channel), but only for its own, newer episode."""from __future__ import annotations@@ -37,28 +40,49 @@ def latch_path(broker: str) -> Path:def _halt_episode(broker: str) -> str | None:- """Return the identity of the currently tripped halt episode, if any.-- The per-broker sentinel wins when both exist, mirroring how a targeted- halt is the one this broker's runner reacts to. ``tripped_at`` is the- identity when the sentinel carries it; a hand-touched sentinel with no- readable payload falls back to the file mtime, which still changes on- every fresh ``touch``.+ """Return the identity of the *newest* tripped halt episode, if any.++ Both the per-broker and the global sentinel can be tripped, at different+ times. The global HALT is authoritative (``halt_flag_set`` halts every+ channel), but a stale per-broker episode must not suppress the sweep for+ a newer global trip — nor an older global trip override a newer+ per-broker one. The newest sentinel therefore wins, ranked by its+ explicit ``tripped_at`` payload when it carries one; the file mtime is+ used only as the fallback for hand-touched/malformed sentinels without+ a parseable ``tripped_at``. Ranking by mtime alone is not safe: two+ sentinels written within one filesystem timestamp quantum share an+ mtime, and the order would silently follow directory iteration instead+ of trip time."""+ candidates: list[tuple[int, str]] = []for path, payload in ((broker_halt_path(broker), read_halt(broker)),(halt_path(), read_halt()),):if payload is None:continue- tripped_at = payload.get("tripped_at")- if tripped_at:- return str(tripped_at)try:- return f"mtime:{path.stat().st_mtime_ns}"+ mtime_ns = path.stat().st_mtime_nsexcept OSError:continue- return None+ tripped_at = payload.get("tripped_at")+ if tripped_at:+ try:+ trip = datetime.fromisoformat(str(tripped_at))+ if trip.tzinfo is None:+ trip = trip.replace(tzinfo=timezone.utc)+ instant_ns = int(trip.timestamp() * 1_000_000_000)+ identity = str(tripped_at)+ except ValueError:+ instant_ns = mtime_ns+ identity = f"mtime:{mtime_ns}"+ else:+ instant_ns = mtime_ns+ identity = f"mtime:{mtime_ns}"+ candidates.append((instant_ns, identity))+ if not candidates:+ return None+ return max(candidates, key=lambda item: item[0])[1]def sweep_already_fired(broker: str) -> bool:@@ -70,7 +94,12 @@ def sweep_already_fired(broker: str) -> bool:record = json.loads(latch_path(broker).read_text(encoding="utf-8"))except (OSError, json.JSONDecodeError):return False- return isinstance(record, dict) and record.get("episode") == episode+ if not isinstance(record, dict):+ return False+ episodes = record.get("episodes")+ if isinstance(episodes, list):+ return episode in episodes+ return record.get("episode") == episode # legacy single-episode recorddef mark_sweep_fired(broker: str) -> None:@@ -78,24 +107,36 @@ def mark_sweep_fired(broker: str) -> None:Atomic write (same-directory temp file + ``os.replace``), same contractas the HALT sentinel. A no-op when no halt is tripped, since there is no- episode to bind the record to.+ episode to bind the record to. The record accumulates every episode the+ sweep has fired for on this broker (``episodes``), so clearing one halt+ never re-fires the sweep for an older episode that was already swept."""episode = _halt_episode(broker)if episode is None:return- record: dict[str, Any] = {- "episode": episode,- "fired_at": datetime.now(timezone.utc).isoformat(),- }path = latch_path(broker)path.parent.mkdir(parents=True, exist_ok=True)- fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")+ record: dict[str, Any] = {}try:- with os.fdopen(fd, "w", encoding="utf-8") as f:- json.dump(record, f)- os.replace(tmp, path)- finally:+ existing = json.loads(path.read_text(encoding="utf-8"))+ if isinstance(existing, dict) and isinstance(existing.get("episodes"), list):+ record = existing+ elif isinstance(existing, dict) and existing.get("episode"):+ record = {"episodes": [existing["episode"]]}+ except (OSError, json.JSONDecodeError):+ pass+ episodes = record.setdefault("episodes", [])+ if episode not in episodes:+ episodes.append(episode)+ record["episode"] = episode # legacy field kept pointing at the latest+ record["fired_at"] = datetime.now(timezone.utc).isoformat()+ fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")try:- os.unlink(tmp)- except OSError:- pass+ with os.fdopen(fd, "w", encoding="utf-8") as f:+ json.dump(record, f)+ os.replace(tmp, path)+ finally:+ try:+ os.unlink(tmp)+ except OSError:+ passdiff --git a/agent/tests/test_api_live_runtime.py b/agent/tests/test_api_live_runtime.pyindex f000c7d76..9a9c2b683 100644--- a/agent/tests/test_api_live_runtime.py+++ b/agent/tests/test_api_live_runtime.py@@ -19,6 +19,8 @@ from types import SimpleNamespacefrom fastapi.testclient import TestClient+import pytest+import api_server@@ -446,3 +448,80 @@ def test_fetch_broker_ceilings_falls_back_to_none(tmp_path, monkeypatch) -> Nonemonkeypatch.setattr(api_server, "_live_broker_adapter", _unavail)assert api_server._fetch_broker_ceilings("robinhood") is None+++def test_build_live_runner_reads_normalize_adapter_payloads(tmp_path, monkeypatch) -> None:+ # The halt-sweep + reconcile reads must receive broker RECORDS, not the+ # adapter's {status: ok, data: ...} envelope — a successful positions+ # read must decode into the records list, not be rejected as a dict.+ monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path), raising=False)+ monkeypatch.setattr(api_server, "_runner_factory", None, raising=False)++ class _StubAdapter:+ def __init__(self):+ self.calls: list[tuple[str, dict]] = []++ def call_tool(self, name, args):+ self.calls.append((name, args))+ if name.endswith("positions"):+ return {"status": "ok", "data": {"positions": [{"symbol": "NVDA", "qty": 3}]}}+ if name.endswith("orders"):+ return {"status": "ok", "data": {"orders": [{"order_id": "o1"}]}}+ if name.endswith("portfolio"):+ return {"status": "ok", "data": {"buying_power": 4200.0}}+ raise AssertionError(f"unexpected tool {name}")++ class _StubSvc:+ session_id = "live-sess-2"+ event_bus = SimpleNamespace(emit=lambda *a, **k: None)++ def create_session(self, title=""):+ return self++ async def send_message(self, sid, content, **kw):+ return {"message_id": "m1", "attempt_id": "a1"}++ adapter = _StubAdapter()+ monkeypatch.setattr(api_server, "_live_broker_adapter", lambda broker: adapter)+ monkeypatch.setattr(api_server, "_get_session_service", lambda: _StubSvc())++ runner = api_server._build_live_runner("robinhood")+ assert runner._read_positions() == [{"symbol": "NVDA", "qty": 3}]+ assert runner._read_open_orders() == [{"order_id": "o1"}]+ assert runner._read_balance() == {"buying_power": 4200.0}+++def test_build_live_runner_read_error_envelope_raises(tmp_path, monkeypatch) -> None:+ # An error envelope at the boundary must raise (fail-closed: reconcile+ # ticks error, the sweep records a structured read error) — never be+ # consumed as an empty/mangled record list.+ monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path), raising=False)+ monkeypatch.setattr(api_server, "_runner_factory", None, raising=False)++ class _StubAdapter:+ def call_tool(self, name, args):+ return {+ "status": "error",+ "server": "robinhood",+ "remote_tool": name,+ "tool": name,+ "error": "connection reset while reading positions",+ "error_type": "ConnectionError",+ }++ class _StubSvc:+ session_id = "live-sess-3"+ event_bus = SimpleNamespace(emit=lambda *a, **k: None)++ def create_session(self, title=""):+ return self++ async def send_message(self, sid, content, **kw):+ return {"message_id": "m1", "attempt_id": "a1"}++ monkeypatch.setattr(api_server, "_live_broker_adapter", lambda broker: _StubAdapter())+ monkeypatch.setattr(api_server, "_get_session_service", lambda: _StubSvc())++ runner = api_server._build_live_runner("robinhood")+ with pytest.raises(RuntimeError, match="connection reset while reading positions"):+ runner._read_positions()diff --git a/agent/tests/test_sweep_latch.py b/agent/tests/test_sweep_latch.pyindex dfac25b34..0853d08ad 100644--- a/agent/tests/test_sweep_latch.py+++ b/agent/tests/test_sweep_latch.py@@ -11,13 +11,14 @@ from __future__ import annotationsimport asyncioimport json+import osfrom pathlib import Pathfrom typing import Any, Mappingimport pytestfrom src.live import paths-from src.live.halt import broker_halt_path, clear_halt, halt_path, trip_halt+from src.live.halt import broker_halt_path, clear_halt, halt_pathfrom src.live.runtime import sweep_latchfrom src.live.runtime.runner import LiveRunner@@ -40,6 +41,26 @@ def _trip_with_timestamp(broker: str | None, tripped_at: str) -> None:)+def test_newer_global_halt_rearms_with_tied_mtimes(live_root: Path) -> None:+ # Regression for the combined-suite flake: broker + global sentinels+ # written within one filesystem timestamp quantum share an mtime.+ # Episode resolution must still rank by tripped_at (global wins), not+ # fall back to mtime order — otherwise the sweep for the newer global+ # halt is wrongly suppressed.+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True++ _trip_with_timestamp(None, "2026-08-27T10:00:00+00:00")+ stat = broker_halt_path(BROKER).stat()+ for path in (broker_halt_path(BROKER), halt_path()):+ os.utime(path, ns=(stat.st_atime_ns, stat.st_mtime_ns))++ assert sweep_latch.sweep_already_fired(BROKER) is False # global wins+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True++def test_mark_then_fired_for_same_episode(live_root: Path) -> None:_trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")assert sweep_latch.sweep_already_fired(BROKER) is False@@ -88,17 +109,19 @@ def test_corrupt_latch_falls_back_to_not_fired(live_root: Path) -> None:assert sweep_latch.sweep_already_fired(BROKER) is False-def _build_runner(live_root: Path, fired: list[str]) -> LiveRunner:+def _build_runner(+ live_root: Path, fired: list[str], flatten_fn: Any = None+) -> LiveRunner:"""A runner whose only observable behavior is recording sweep invocations."""-async def _agent_caller(session_id: str, prompt: str) -> Mapping[str, Any]:return {"status": "success"}def _submit(request: dict[str, Any]) -> dict[str, Any]:return {"status": "ok"}- def _flatten(broker, submit, read_positions, read_open_orders):+ def _default_flatten(broker, submit, read_positions, read_open_orders):fired.append(broker)+ return {"side_effects_attempted": True}def _audit(event) -> Mapping[str, Any]:return {"audit_id": "a1"}@@ -113,7 +136,7 @@ def _build_runner(live_root: Path, fired: list[str]) -> LiveRunner:write_audit_fn=_audit,halt_flag_fn=lambda broker: True,submit_fn=_submit,- flatten_fn=_flatten,+ flatten_fn=flatten_fn or _default_flatten,session_id="latch-test",)@@ -137,3 +160,88 @@ def test_new_episode_after_restart_fires_again(live_root: Path) -> None:_trip_with_timestamp(BROKER, "2026-08-27T09:30:00+00:00")asyncio.run(_build_runner(live_root, fired).run_once())assert fired == [BROKER, BROKER]+++def test_newer_global_halt_rearms_stale_broker_latch(live_root: Path) -> None:+ # A broker episode latched first; a NEWER global HALT must not be+ # suppressed — the global halt is authoritative (halt_flag_set), and a+ # stale per-broker latch must not skip the kill action for the new+ # episode. Clearing the global halt must not re-fire the sweep for the+ # already-swept broker episode either.+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True++ _trip_with_timestamp(None, "2026-08-27T10:00:00+00:00")+ assert sweep_latch.sweep_already_fired(BROKER) is False # re-armed++ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True+ clear_halt() # global gone; only the old broker episode remains+ assert sweep_latch.sweep_already_fired(BROKER) is True # still latched+++def test_read_failure_does_not_latch_and_restart_replays(live_root: Path) -> None:+ # The reviewer-reported safety hole: a sweep whose broker-state read+ # failed must NOT persist the latch, or a restart suppresses the kill+ # action for the episode forever. The read failure is the flattened+ # report shape flatten_and_cancel now produces for an adapter error+ # envelope / non-list read.+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []++ def _flatten(broker, submit, read_positions, read_open_orders):+ fired.append(broker)+ return {"errors": [{"phase": "read_positions", "error": "broker read failed"}]}++ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER]+ assert not sweep_latch.latch_path(BROKER).exists() # nothing persisted+ # Restart: the failed sweep must re-fire, not be suppressed.+ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER, BROKER]+++def test_nothing_to_do_does_not_latch_and_rechecks(live_root: Path) -> None:+ # Reads succeed but there are no open orders/positions: no broker write+ # was attempted, so per policy nothing must be latched — the sweep+ # re-checks on the next tick/restart (and would cancel a late-appearing+ # order), never suppressing the kill action for the episode.+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []++ def _flatten(broker, submit, read_positions, read_open_orders):+ fired.append(broker)+ return {"errors": []} # reads ok, nothing to act on++ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER]+ assert not sweep_latch.latch_path(BROKER).exists()+ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER, BROKER]+++def test_side_effect_attempted_latches_even_with_read_error(+ live_root: Path,+) -> None:+ # Reviewer sequence: order read fails, position read succeeds, a close+ # order is submitted. The report carries a read error AND a side effect.+ # Latching must key on the side effect: replaying that close after a+ # restart would be a duplicate market close (the failure the latch+ # exists to prevent).+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []++ def _flatten(broker, submit, read_positions, read_open_orders):+ fired.append(broker)+ return {+ "side_effects_attempted": True,+ "errors": [{"phase": "read_open_orders", "error": "orders read failed"}],+ }++ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER]+ assert sweep_latch.latch_path(BROKER).exists() # latched despite the error+ # Restart: the on-disk latch suppresses the duplicate close.+ asyncio.run(_build_runner(live_root, fired, _flatten).run_once())+ assert fired == [BROKER]