"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(live): persist the flatten latch across runner restarts
Yufeng He513bc5b26b2b
diff --git a/agent/src/live/runtime/runner.py b/agent/src/live/runtime/runner.pyindex b7611c45b..b41cb04d5 100644--- a/agent/src/live/runtime/runner.py+++ b/agent/src/live/runtime/runner.py@@ -46,6 +46,7 @@ from src.live.mandate.model import Mandatefrom src.live.mandate.store import load_mandatefrom src.live.runtime.flatten import flatten_and_cancelfrom src.live.runtime.jobstore import JobStore+from src.live.runtime.sweep_latch import mark_sweep_fired, sweep_already_firedfrom src.live.runtime.liveness import write_heartbeatfrom src.live.runtime.scheduler import Job, Schedulerfrom src.live.runtime.triggers import Trigger, due_now@@ -571,12 +572,17 @@ class LiveRunner:Invoked the moment the runner observes a tripped HALT. Idempotent acrossticks via the ``_flatten_fired`` latch so the no-retry rule (SPEC §8.5)- holds even if the halted runner keeps waking. A sweep failure is audited- but not retried — flatten/cancel side effects are not idempotent.+ holds even if the halted runner keeps waking. The latch is also persisted+ on disk (``sweep_latch``) so a restart with flatten orders still working+ does not replay the sweep. A sweep failure is audited but not retried —+ flatten/cancel side effects are not idempotent."""if self._flatten_fired or self._submit_fn is None:returnself._flatten_fired = True+ if sweep_already_fired(self.broker):+ return+ mark_sweep_fired(self.broker)try:self._flatten_fn(self.broker,diff --git a/agent/src/live/runtime/sweep_latch.py b/agent/src/live/runtime/sweep_latch.pynew file mode 100644index 000000000..5aaaadd18--- /dev/null+++ b/agent/src/live/runtime/sweep_latch.py@@ -0,0 +1,101 @@+"""Persist the preemptive-sweep latch across runner restarts.++The HALT sentinel is a file, so it survives a process restart; the runner's+``_flatten_fired`` flag does not. Without a persisted latch, a restart with+flatten orders still working replays the whole sweep on the next tick: a fresh+cancel pass plus a new market order per position, which can flip the account+from long to net short. This module records the sweep's firing on disk, bound+to the halt *episode* that caused it.++The latch lives next to the per-broker HALT sentinel at+``<runtime_root>/live/<broker>/FLATTEN_FIRED`` and carries the episode identity+of the halt that was tripped when the sweep fired: the sentinel's ``tripped_at``+when it has one, otherwise the sentinel file's mtime. A later trip writes a new+sentinel (fresh ``tripped_at`` / fresh mtime), so a stale latch from a previous+episode never suppresses a new one; clearing HALT and re-tripping therefore+re-arms the sweep without anyone deleting the latch.+"""++from __future__ import annotations++import json+import os+import tempfile+from datetime import datetime, timezone+from pathlib import Path+from typing import Any++from src.live.halt import broker_halt_path, halt_path, read_halt+from src.live.paths import broker_dir++_LATCH_FILENAME = "FLATTEN_FIRED"+++def latch_path(broker: str) -> Path:+ """Return the per-broker sweep latch path (not created here)."""+ return broker_dir(broker) / _LATCH_FILENAME+++def _halt_episode(broker: str) -> str | None:+ """Return the identity of the currently tripped halt episode, if any.++ The per-broker sentinel wins when both exist, mirroring how a targeted+ halt is the one this broker's runner reacts to. ``tripped_at`` is the+ identity when the sentinel carries it; a hand-touched sentinel with no+ readable payload falls back to the file mtime, which still changes on+ every fresh ``touch``.+ """+ for path, payload in (+ (broker_halt_path(broker), read_halt(broker)),+ (halt_path(), read_halt()),+ ):+ if payload is None:+ continue+ tripped_at = payload.get("tripped_at")+ if tripped_at:+ return str(tripped_at)+ try:+ return f"mtime:{path.stat().st_mtime_ns}"+ except OSError:+ continue+ return None+++def sweep_already_fired(broker: str) -> bool:+ """Return True when the sweep already fired for the current halt episode."""+ episode = _halt_episode(broker)+ if episode is None:+ return False+ try:+ record = json.loads(latch_path(broker).read_text(encoding="utf-8"))+ except (OSError, json.JSONDecodeError):+ return False+ return isinstance(record, dict) and record.get("episode") == episode+++def mark_sweep_fired(broker: str) -> None:+ """Record that the sweep fired for the current halt episode.++ Atomic write (same-directory temp file + ``os.replace``), same contract+ as the HALT sentinel. A no-op when no halt is tripped, since there is no+ episode to bind the record to.+ """+ episode = _halt_episode(broker)+ if episode is None:+ return+ record: dict[str, Any] = {+ "episode": episode,+ "fired_at": datetime.now(timezone.utc).isoformat(),+ }+ path = latch_path(broker)+ path.parent.mkdir(parents=True, exist_ok=True)+ fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as f:+ json.dump(record, f)+ os.replace(tmp, path)+ finally:+ try:+ os.unlink(tmp)+ except OSError:+ passdiff --git a/agent/tests/test_sweep_latch.py b/agent/tests/test_sweep_latch.pynew file mode 100644index 000000000..dfac25b34--- /dev/null+++ b/agent/tests/test_sweep_latch.py@@ -0,0 +1,139 @@+"""Tests for the restart-persistent preemptive-sweep latch.++The HALT sentinel is a file and survives restarts; the runner's in-memory+``_flatten_fired`` does not. The latch (src/live/runtime/sweep_latch.py) binds+the sweep's firing to the halt episode that caused it, so a restarted runner+with flatten orders still working does not replay the sweep, while a fresh+halt episode re-arms it.+"""++from __future__ import annotations++import asyncio+import json+from pathlib import Path+from typing import Any, Mapping++import pytest++from src.live import paths+from src.live.halt import broker_halt_path, clear_halt, halt_path, trip_halt+from src.live.runtime import sweep_latch+from src.live.runtime.runner import LiveRunner++BROKER = "robinhood"+++def live_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ """Point the live runtime root at an isolated tmp dir."""+ monkeypatch.setattr(paths, "get_runtime_root", lambda: tmp_path)+ return tmp_path+++def _trip_with_timestamp(broker: str | None, tripped_at: str) -> None:+ path = broker_halt_path(broker) if broker else halt_path()+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(+ json.dumps({"tripped_at": tripped_at, "by": "cli", "reason": "test"}),+ encoding="utf-8",+ )+++def test_mark_then_fired_for_same_episode(live_root: Path) -> None:+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ assert sweep_latch.sweep_already_fired(BROKER) is False+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True+++def test_fresh_halt_episode_rearms(live_root: Path) -> None:+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True+ # The operator clears the halt and a later incident trips it again: the+ # latch from the first episode must not suppress the second sweep.+ clear_halt(BROKER)+ _trip_with_timestamp(BROKER, "2026-08-27T09:30:00+00:00")+ assert sweep_latch.sweep_already_fired(BROKER) is False+++def test_mark_without_halt_is_noop(live_root: Path) -> None:+ sweep_latch.mark_sweep_fired(BROKER)+ assert not sweep_latch.latch_path(BROKER).exists()+++def test_hand_touched_halt_binds_via_mtime(live_root: Path) -> None:+ # A bare `touch` produces a sentinel with no JSON payload; the latch must+ # still bind to the episode (via the file mtime) rather than never firing.+ path = broker_halt_path(BROKER)+ path.parent.mkdir(parents=True, exist_ok=True)+ path.touch()+ assert sweep_latch.sweep_already_fired(BROKER) is False+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True+++def test_global_halt_episode_visible_to_broker(live_root: Path) -> None:+ _trip_with_timestamp(None, "2026-08-27T02:00:00+00:00")+ sweep_latch.mark_sweep_fired(BROKER)+ assert sweep_latch.sweep_already_fired(BROKER) is True+++def test_corrupt_latch_falls_back_to_not_fired(live_root: Path) -> None:+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ latch = sweep_latch.latch_path(BROKER)+ latch.parent.mkdir(parents=True, exist_ok=True)+ latch.write_text("{not json", encoding="utf-8")+ assert sweep_latch.sweep_already_fired(BROKER) is False+++def _build_runner(live_root: Path, fired: list[str]) -> LiveRunner:+ """A runner whose only observable behavior is recording sweep invocations."""++ async def _agent_caller(session_id: str, prompt: str) -> Mapping[str, Any]:+ return {"status": "success"}++ def _submit(request: dict[str, Any]) -> dict[str, Any]:+ return {"status": "ok"}++ def _flatten(broker, submit, read_positions, read_open_orders):+ fired.append(broker)++ def _audit(event) -> Mapping[str, Any]:+ return {"audit_id": "a1"}++ return LiveRunner(+ BROKER,+ agent_caller=_agent_caller,+ reconcile_fn=lambda *a, **k: None,+ read_positions=list,+ read_balance=list,+ read_open_orders=list,+ write_audit_fn=_audit,+ halt_flag_fn=lambda broker: True,+ submit_fn=_submit,+ flatten_fn=_flatten,+ session_id="latch-test",+ )+++def test_restart_does_not_replay_the_sweep(live_root: Path) -> None:+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []+ asyncio.run(_build_runner(live_root, fired).run_once())+ assert fired == [BROKER]+ # A new runner instance over the same runtime root (the restart): the+ # in-memory latch is gone, but the on-disk one suppresses the replay.+ asyncio.run(_build_runner(live_root, fired).run_once())+ assert fired == [BROKER]+++def test_new_episode_after_restart_fires_again(live_root: Path) -> None:+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []+ asyncio.run(_build_runner(live_root, fired).run_once())+ clear_halt(BROKER)+ _trip_with_timestamp(BROKER, "2026-08-27T09:30:00+00:00")+ asyncio.run(_build_runner(live_root, fired).run_once())+ assert fired == [BROKER, BROKER]