Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(live): persist the flatten latch across runner restarts

Yufeng He513bc5b26b2b
diff --git a/agent/src/live/runtime/runner.py b/agent/src/live/runtime/runner.py
index b7611c45b..b41cb04d5 100644
--- a/agent/src/live/runtime/runner.py
+++ b/agent/src/live/runtime/runner.py
@@ -46,6 +46,7 @@ from src.live.mandate.model import Mandate
from src.live.mandate.store import load_mandate
from src.live.runtime.flatten import flatten_and_cancel
from src.live.runtime.jobstore import JobStore
+from src.live.runtime.sweep_latch import mark_sweep_fired, sweep_already_fired
from src.live.runtime.liveness import write_heartbeat
from src.live.runtime.scheduler import Job, Scheduler
from src.live.runtime.triggers import Trigger, due_now
@@ -571,12 +572,17 @@ class LiveRunner:
Invoked the moment the runner observes a tripped HALT. Idempotent across
ticks via the ``_flatten_fired`` latch so the no-retry rule (SPEC §8.5)
- holds even if the halted runner keeps waking. A sweep failure is audited
- but not retried — flatten/cancel side effects are not idempotent.
+ holds even if the halted runner keeps waking. The latch is also persisted
+ on disk (``sweep_latch``) so a restart with flatten orders still working
+ does not replay the sweep. A sweep failure is audited but not retried —
+ flatten/cancel side effects are not idempotent.
"""
if self._flatten_fired or self._submit_fn is None:
return
self._flatten_fired = True
+ if sweep_already_fired(self.broker):
+ return
+ mark_sweep_fired(self.broker)
try:
self._flatten_fn(
self.broker,
diff --git a/agent/src/live/runtime/sweep_latch.py b/agent/src/live/runtime/sweep_latch.py
new file mode 100644
index 000000000..5aaaadd18
--- /dev/null
+++ b/agent/src/live/runtime/sweep_latch.py
@@ -0,0 +1,101 @@
+"""Persist the preemptive-sweep latch across runner restarts.
+
+The HALT sentinel is a file, so it survives a process restart; the runner's
+``_flatten_fired`` flag does not. Without a persisted latch, a restart with
+flatten orders still working replays the whole sweep on the next tick: a fresh
+cancel pass plus a new market order per position, which can flip the account
+from long to net short. This module records the sweep's firing on disk, bound
+to the halt *episode* that caused it.
+
+The latch lives next to the per-broker HALT sentinel at
+``<runtime_root>/live/<broker>/FLATTEN_FIRED`` and carries the episode identity
+of the halt that was tripped when the sweep fired: the sentinel's ``tripped_at``
+when it has one, otherwise the sentinel file's mtime. A later trip writes a new
+sentinel (fresh ``tripped_at`` / fresh mtime), so a stale latch from a previous
+episode never suppresses a new one; clearing HALT and re-tripping therefore
+re-arms the sweep without anyone deleting the latch.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from src.live.halt import broker_halt_path, halt_path, read_halt
+from src.live.paths import broker_dir
+
+_LATCH_FILENAME = "FLATTEN_FIRED"
+
+
+def latch_path(broker: str) -> Path:
+ """Return the per-broker sweep latch path (not created here)."""
+ return broker_dir(broker) / _LATCH_FILENAME
+
+
+def _halt_episode(broker: str) -> str | None:
+ """Return the identity of the currently tripped halt episode, if any.
+
+ The per-broker sentinel wins when both exist, mirroring how a targeted
+ halt is the one this broker's runner reacts to. ``tripped_at`` is the
+ identity when the sentinel carries it; a hand-touched sentinel with no
+ readable payload falls back to the file mtime, which still changes on
+ every fresh ``touch``.
+ """
+ for path, payload in (
+ (broker_halt_path(broker), read_halt(broker)),
+ (halt_path(), read_halt()),
+ ):
+ if payload is None:
+ continue
+ tripped_at = payload.get("tripped_at")
+ if tripped_at:
+ return str(tripped_at)
+ try:
+ return f"mtime:{path.stat().st_mtime_ns}"
+ except OSError:
+ continue
+ return None
+
+
+def sweep_already_fired(broker: str) -> bool:
+ """Return True when the sweep already fired for the current halt episode."""
+ episode = _halt_episode(broker)
+ if episode is None:
+ return False
+ try:
+ record = json.loads(latch_path(broker).read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError):
+ return False
+ return isinstance(record, dict) and record.get("episode") == episode
+
+
+def mark_sweep_fired(broker: str) -> None:
+ """Record that the sweep fired for the current halt episode.
+
+ Atomic write (same-directory temp file + ``os.replace``), same contract
+ as the HALT sentinel. A no-op when no halt is tripped, since there is no
+ episode to bind the record to.
+ """
+ episode = _halt_episode(broker)
+ if episode is None:
+ return
+ record: dict[str, Any] = {
+ "episode": episode,
+ "fired_at": datetime.now(timezone.utc).isoformat(),
+ }
+ path = latch_path(broker)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as f:
+ json.dump(record, f)
+ os.replace(tmp, path)
+ finally:
+ try:
+ os.unlink(tmp)
+ except OSError:
+ pass
diff --git a/agent/tests/test_sweep_latch.py b/agent/tests/test_sweep_latch.py
new file mode 100644
index 000000000..dfac25b34
--- /dev/null
+++ b/agent/tests/test_sweep_latch.py
@@ -0,0 +1,139 @@
+"""Tests for the restart-persistent preemptive-sweep latch.
+
+The HALT sentinel is a file and survives restarts; the runner's in-memory
+``_flatten_fired`` does not. The latch (src/live/runtime/sweep_latch.py) binds
+the sweep's firing to the halt episode that caused it, so a restarted runner
+with flatten orders still working does not replay the sweep, while a fresh
+halt episode re-arms it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+from pathlib import Path
+from typing import Any, Mapping
+
+import pytest
+
+from src.live import paths
+from src.live.halt import broker_halt_path, clear_halt, halt_path, trip_halt
+from src.live.runtime import sweep_latch
+from src.live.runtime.runner import LiveRunner
+
+BROKER = "robinhood"
+
+
+def live_root(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ """Point the live runtime root at an isolated tmp dir."""
+ monkeypatch.setattr(paths, "get_runtime_root", lambda: tmp_path)
+ return tmp_path
+
+
+def _trip_with_timestamp(broker: str | None, tripped_at: str) -> None:
+ path = broker_halt_path(broker) if broker else halt_path()
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(
+ json.dumps({"tripped_at": tripped_at, "by": "cli", "reason": "test"}),
+ encoding="utf-8",
+ )
+
+
+def test_mark_then_fired_for_same_episode(live_root: Path) -> None:
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ assert sweep_latch.sweep_already_fired(BROKER) is False
+ sweep_latch.mark_sweep_fired(BROKER)
+ assert sweep_latch.sweep_already_fired(BROKER) is True
+
+
+def test_fresh_halt_episode_rearms(live_root: Path) -> None:
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ sweep_latch.mark_sweep_fired(BROKER)
+ assert sweep_latch.sweep_already_fired(BROKER) is True
+ # The operator clears the halt and a later incident trips it again: the
+ # latch from the first episode must not suppress the second sweep.
+ clear_halt(BROKER)
+ _trip_with_timestamp(BROKER, "2026-08-27T09:30:00+00:00")
+ assert sweep_latch.sweep_already_fired(BROKER) is False
+
+
+def test_mark_without_halt_is_noop(live_root: Path) -> None:
+ sweep_latch.mark_sweep_fired(BROKER)
+ assert not sweep_latch.latch_path(BROKER).exists()
+
+
+def test_hand_touched_halt_binds_via_mtime(live_root: Path) -> None:
+ # A bare `touch` produces a sentinel with no JSON payload; the latch must
+ # still bind to the episode (via the file mtime) rather than never firing.
+ path = broker_halt_path(BROKER)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.touch()
+ assert sweep_latch.sweep_already_fired(BROKER) is False
+ sweep_latch.mark_sweep_fired(BROKER)
+ assert sweep_latch.sweep_already_fired(BROKER) is True
+
+
+def test_global_halt_episode_visible_to_broker(live_root: Path) -> None:
+ _trip_with_timestamp(None, "2026-08-27T02:00:00+00:00")
+ sweep_latch.mark_sweep_fired(BROKER)
+ assert sweep_latch.sweep_already_fired(BROKER) is True
+
+
+def test_corrupt_latch_falls_back_to_not_fired(live_root: Path) -> None:
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ latch = sweep_latch.latch_path(BROKER)
+ latch.parent.mkdir(parents=True, exist_ok=True)
+ latch.write_text("{not json", encoding="utf-8")
+ assert sweep_latch.sweep_already_fired(BROKER) is False
+
+
+def _build_runner(live_root: Path, fired: list[str]) -> LiveRunner:
+ """A runner whose only observable behavior is recording sweep invocations."""
+
+ async def _agent_caller(session_id: str, prompt: str) -> Mapping[str, Any]:
+ return {"status": "success"}
+
+ def _submit(request: dict[str, Any]) -> dict[str, Any]:
+ return {"status": "ok"}
+
+ def _flatten(broker, submit, read_positions, read_open_orders):
+ fired.append(broker)
+
+ def _audit(event) -> Mapping[str, Any]:
+ return {"audit_id": "a1"}
+
+ return LiveRunner(
+ BROKER,
+ agent_caller=_agent_caller,
+ reconcile_fn=lambda *a, **k: None,
+ read_positions=list,
+ read_balance=list,
+ read_open_orders=list,
+ write_audit_fn=_audit,
+ halt_flag_fn=lambda broker: True,
+ submit_fn=_submit,
+ flatten_fn=_flatten,
+ session_id="latch-test",
+ )
+
+
+def test_restart_does_not_replay_the_sweep(live_root: Path) -> None:
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ fired: list[str] = []
+ asyncio.run(_build_runner(live_root, fired).run_once())
+ assert fired == [BROKER]
+ # A new runner instance over the same runtime root (the restart): the
+ # in-memory latch is gone, but the on-disk one suppresses the replay.
+ asyncio.run(_build_runner(live_root, fired).run_once())
+ assert fired == [BROKER]
+
+
+def test_new_episode_after_restart_fires_again(live_root: Path) -> None:
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ fired: list[str] = []
+ asyncio.run(_build_runner(live_root, fired).run_once())
+ clear_halt(BROKER)
+ _trip_with_timestamp(BROKER, "2026-08-27T09:30:00+00:00")
+ asyncio.run(_build_runner(live_root, fired).run_once())
+ assert fired == [BROKER, BROKER]