Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(live): latch only the swept episodes under per-episode latch files

cgycoreybc37d4c7f11b
diff --git a/agent/src/live/runtime/runner.py b/agent/src/live/runtime/runner.py
index e09f6f2f8..58602397f 100644
--- a/agent/src/live/runtime/runner.py
+++ b/agent/src/live/runtime/runner.py
@@ -48,7 +48,7 @@ from src.live.runtime.flatten import flatten_and_cancel
from src.live.runtime.jobstore import JobStore
from src.live.runtime.sweep_latch import (
claim_sweep,
- halt_episode,
+ halt_snapshot,
mark_sweep_fired,
release_claim,
sweep_already_fired,
@@ -640,15 +640,21 @@ class LiveRunner:
"""
if self._flatten_fired or self._submit_fn is None:
return
- if sweep_already_fired(self.broker):
+ # Capture ONE coherent halt snapshot before claiming. The claim, the
+ # already-fired check, the latch write and the release must all bind
+ # to the SAME view of the halt state: re-reading the sentinels later
+ # would let a mid-sweep clear/re-trip record an episode whose sweep
+ # never ran (next runner would skip it — positions stay open).
+ active_episodes, episode = halt_snapshot(self.broker)
+ episode = episode or "unknown"
+ if sweep_already_fired(self.broker, episode):
self._flatten_fired = True
return
- # Resolve the episode ONCE: the claim and its release in ``finally``
- # must refer to the same episode. Re-resolving at release time can
- # bind to a NEWER episode (halt cleared + re-tripped mid-sweep) and
- # delete that episode's claim — even a different process's — which
- # would unprotect a concurrent sweep of the new episode.
- episode = halt_episode(self.broker) or "unknown"
+ # The episode is resolved exactly once: the claim and its release in
+ # ``finally`` must refer to the same one. Re-resolving at release time
+ # can bind to a NEWER episode (halt cleared + re-tripped mid-sweep)
+ # and delete that episode's claim — even a different process's —
+ # which would unprotect a concurrent sweep of the new episode.
if not claim_sweep(self.broker, episode):
# Another process holds this episode's claim (or a crash left it
# behind): the outcome is unknowable — re-sweeping could duplicate
@@ -681,7 +687,7 @@ class LiveRunner:
# and are not retryable — latch so a restart does not replay.
self._flatten_fired = True
try:
- mark_sweep_fired(self.broker)
+ mark_sweep_fired(self.broker, active_episodes)
except Exception as persist_exc: # noqa: BLE001 — double failure
# Both the sweep AND the durable latch write failed: the
# in-memory flag still shields this process, and the
@@ -750,7 +756,7 @@ class LiveRunner:
return
self._flatten_fired = True
try:
- mark_sweep_fired(self.broker)
+ mark_sweep_fired(self.broker, active_episodes)
except Exception as exc: # noqa: BLE001 — persistence must not be silent
# Durability failure AFTER a broker write: the in-memory flag
# protects this process, but a restart sees no latch and could
diff --git a/agent/src/live/runtime/sweep_latch.py b/agent/src/live/runtime/sweep_latch.py
index 3b16df5c1..efc1beb53 100644
--- a/agent/src/live/runtime/sweep_latch.py
+++ b/agent/src/live/runtime/sweep_latch.py
@@ -24,7 +24,6 @@ from __future__ import annotations
import hashlib
import json
import os
-import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
@@ -36,9 +35,20 @@ _LATCH_FILENAME = "FLATTEN_FIRED"
_CLAIM_FILENAME = "FLATTEN_CLAIM"
-def latch_path(broker: str) -> Path:
- """Return the per-broker sweep latch path (not created here)."""
- return broker_dir(broker) / _LATCH_FILENAME
+def latch_path(broker: str, episode: str | None = None) -> Path:
+ """Return the per-broker, per-episode sweep latch path (not created here).
+
+ One file per halt episode (``FLATTEN_FIRED-<sha16>(episode)``): the sweep
+ record is never a shared read-merge-write object, so concurrent
+ completions for different episodes cannot lose one another's entry —
+ each atomic ``O_CREAT|O_EXCL`` create is independent. ``episode=None``
+ returns the legacy single-record path (pre-per-episode format, read
+ for compatibility only).
+ """
+ if episode is None:
+ return broker_dir(broker) / _LATCH_FILENAME
+ digest = hashlib.sha256(episode.encode("utf-8")).hexdigest()[:16]
+ return broker_dir(broker) / f"{_LATCH_FILENAME}-{digest}"
def halt_episode(broker: str) -> str | None:
@@ -146,7 +156,20 @@ def _active_episodes(broker: str) -> list[str]:
other halt is still tripped. Both identities must therefore be latched
together.
"""
- identities: list[str] = []
+ return halt_snapshot(broker)[0]
+
+
+def halt_snapshot(broker: str) -> tuple[list[str], str | None]:
+ """One coherent read of every tripped sentinel.
+
+ Returns ``(active_episodes, newest_episode)`` from a single pass over
+ both sentinels. The runner MUST capture this snapshot once, before
+ claiming, and pass the captured episode list into
+ :func:`mark_sweep_fired` — re-reading the sentinels later would let a
+ mid-sweep clear/re-trip change what the latch records (recording an
+ episode whose sweep never ran, which would silently skip it).
+ """
+ candidates: list[tuple[int, str]] = []
for path, payload in (
(broker_halt_path(broker), read_halt(broker)),
(halt_path(), read_halt()),
@@ -155,8 +178,11 @@ def _active_episodes(broker: str) -> list[str]:
continue
resolved = _sentinel_identity(path, payload)
if resolved is not None:
- identities.append(resolved[1])
- return identities
+ candidates.append(resolved)
+ if not candidates:
+ return [], None
+ newest = max(candidates, key=lambda item: item[0])[1]
+ return [identity for _, identity in candidates], newest
def _halt_episode(broker: str) -> str | None:
@@ -168,26 +194,23 @@ def _halt_episode(broker: str) -> str | None:
a newer global trip — nor an older global trip override a newer
per-broker one. The newest sentinel therefore wins.
"""
- candidates: list[tuple[int, str]] = []
- for path, payload in (
- (broker_halt_path(broker), read_halt(broker)),
- (halt_path(), read_halt()),
- ):
- if payload is None:
- continue
- resolved = _sentinel_identity(path, payload)
- if resolved is not None:
- candidates.append(resolved)
- if not candidates:
- return None
- return max(candidates, key=lambda item: item[0])[1]
+ return halt_snapshot(broker)[1]
+
+def sweep_already_fired(broker: str, episode: str | None = None) -> bool:
+ """Return True when the sweep already fired for ``broker``'s ``episode``.
-def sweep_already_fired(broker: str) -> bool:
- """Return True when the sweep already fired for the current halt episode."""
- episode = _halt_episode(broker)
+ ``episode=None`` resolves the newest currently tripped episode (legacy
+ signature). The per-episode latch file is authoritative; the legacy
+ single-record file is consulted only as a compatibility fallback for
+ latches written by pre-per-episode versions.
+ """
if episode is None:
- return False
+ episode = _halt_episode(broker)
+ if episode is None:
+ return False
+ if latch_path(broker, episode).exists():
+ return True
try:
record = json.loads(latch_path(broker).read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
@@ -200,52 +223,43 @@ def sweep_already_fired(broker: str) -> bool:
return record.get("episode") == episode # legacy single-episode record
-def mark_sweep_fired(broker: str) -> None:
- """Record that the sweep fired for the currently tripped halt episodes.
+def mark_sweep_fired(broker: str, episodes: list[str] | None = None) -> None:
+ """Atomically record that the sweep fired for every episode in ``episodes``.
- Atomic write (same-directory temp file + ``os.replace``), same contract
- as the HALT sentinel. A no-op when no halt is tripped, since there is no
- episode to bind the record to. The record accumulates every episode the
- sweep has fired for on this broker (``episodes``), so clearing one halt
- never re-fires the sweep for an older episode that was already swept.
+ One ``O_CREAT|O_EXCL`` latch file per episode identity — no shared
+ read-merge-write record exists to race on: each create is independent and
+ idempotent (an existing file is a prior completion, left untouched). This
+ is what makes concurrent completions for different episodes safe without
+ a lock, and a re-trip never loses a completed episode's record.
- Both the per-broker and the global sentinel may be tripped at the same
- time: the sweep covers the halt state as a whole, so EVERY currently
- active episode identity is recorded together. Recording only the newest
- would leave the older still-active halt unrecorded — clearing the newer
- would then re-fire the sweep while the other halt remains tripped
- (redundant no-op re-sweep and audit noise).
+ The caller MUST pass the episode snapshot it captured BEFORE claiming
+ (see :func:`halt_snapshot`) — never re-read the sentinels here: a
+ mid-sweep clear/re-trip would record an episode whose sweep never ran,
+ causing the next runner to skip it. ``episodes=None`` resolves the
+ currently active episodes (legacy signature, for standalone use).
"""
- active = _active_episodes(broker)
- if not active:
- return
- path = latch_path(broker)
- path.parent.mkdir(parents=True, exist_ok=True)
- record: dict[str, Any] = {}
- try:
- existing = json.loads(path.read_text(encoding="utf-8"))
- if isinstance(existing, dict) and isinstance(existing.get("episodes"), list):
- record = existing
- elif isinstance(existing, dict) and existing.get("episode"):
- record = {"episodes": [existing["episode"]]}
- except (OSError, json.JSONDecodeError):
- pass
- episodes = record.setdefault("episodes", [])
- to_write = bool(episodes)
- for episode in active:
- if episode not in episodes:
- episodes.append(episode)
- to_write = True
- if to_write:
- record["episode"] = _halt_episode(broker) # legacy field = newest
- record["fired_at"] = datetime.now(timezone.utc).isoformat()
- fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")
+ if episodes is None:
+ episodes = _active_episodes(broker)
+ for episode in episodes:
+ path = latch_path(broker, episode)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
+ except FileExistsError:
+ continue # already latched by a prior completion
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
- json.dump(record, f)
- os.replace(tmp, path)
- finally:
+ json.dump(
+ {
+ "fired_at": datetime.now(timezone.utc).isoformat(),
+ "episode": episode,
+ },
+ f,
+ )
+ except Exception:
try:
- os.unlink(tmp)
+ os.unlink(path)
except OSError:
pass
+ raise # durability failure; caller decides (audit + flag)
+
diff --git a/agent/tests/test_sweep_latch.py b/agent/tests/test_sweep_latch.py
index 19cc0325b..b346e5414 100644
--- a/agent/tests/test_sweep_latch.py
+++ b/agent/tests/test_sweep_latch.py
@@ -41,6 +41,12 @@ def _trip_with_timestamp(broker: str | None, tripped_at: str) -> None:
)
+def _any_latch_files() -> bool:
+ """True when any per-episode/legacy latch file exists for BROKER."""
+ d = sweep_latch.latch_path(BROKER).parent
+ return any(p.name.startswith("FLATTEN_FIRED") for p in d.glob("FLATTEN_FIRED*"))
+
+
def test_newer_global_halt_rearms_with_tied_mtimes(live_root: Path) -> None:
# Regression for the combined-suite flake: broker + global sentinels
# written within one filesystem timestamp quantum share an mtime.
@@ -81,7 +87,7 @@ def test_fresh_halt_episode_rearms(live_root: Path) -> None:
def test_mark_without_halt_is_noop(live_root: Path) -> None:
sweep_latch.mark_sweep_fired(BROKER)
- assert not sweep_latch.latch_path(BROKER).exists()
+ assert not _any_latch_files()
def test_hand_touched_halt_binds_via_mtime(live_root: Path) -> None:
@@ -196,7 +202,7 @@ def test_read_failure_does_not_latch_and_restart_replays(live_root: Path) -> Non
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER]
- assert not sweep_latch.latch_path(BROKER).exists() # nothing persisted
+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists() # not latched
# Restart: the failed sweep must re-fire, not be suppressed.
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER, BROKER]
@@ -216,7 +222,7 @@ def test_nothing_to_do_does_not_latch_and_rechecks(live_root: Path) -> None:
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER]
- assert not sweep_latch.latch_path(BROKER).exists()
+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER, BROKER]
@@ -241,7 +247,7 @@ def test_side_effect_attempted_latches_even_with_read_error(
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER]
- assert sweep_latch.latch_path(BROKER).exists() # latched despite the error
+ assert sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()
# Restart: the on-disk latch suppresses the duplicate close.
asyncio.run(_build_runner(live_root, fired, _flatten).run_once())
assert fired == [BROKER]
@@ -316,7 +322,7 @@ def test_runner_claim_released_after_read_failure(live_root: Path) -> None:
assert fired == [BROKER]
ep = sweep_latch.halt_episode(BROKER)
assert not sweep_latch.claim_path(BROKER, ep).exists()
- assert not sweep_latch.latch_path(BROKER).exists()
+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()
def test_mark_records_both_active_episodes_older_global_newer_broker(
@@ -445,3 +451,60 @@ def test_persistent_read_failure_audits_the_breach_once_per_episode(
total, kinds = _count_audits(live_root, _flatten, ticks=5)
assert total == 6, kinds
assert kinds.count("breach") == 1, kinds
+
+
+def test_mid_sweep_retrip_records_only_swept_episode(live_root: Path) -> None:
+ # Reviewer merge-blocker: the latch must record the episodes the sweep
+ # COVERED (captured before claiming), never what the sentinels say AFTER
+ # the sweep. With a mid-sweep clear + re-trip (ep1 -> ep2), recording ep2
+ # would make the next runner skip it — positions stay open.
+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")
+ fired: list[str] = []
+
+ def _retripping_flatten(broker, submit, read_positions, read_open_orders):
+ fired.append(broker)
+ clear_halt(BROKER)
+ _trip_with_timestamp(BROKER, "2026-08-27T10:00:00+00:00")
+ return {"side_effects_attempted": True}
+
+ asyncio.run(_build_runner(live_root, fired, _retripping_flatten).run_once())
+ # ep1 (the swept episode) is latched; ep2 (never swept) is not.
+ assert sweep_latch.sweep_already_fired(BROKER, "2026-08-27T01:00:00+00:00") is True
+ ep2 = sweep_latch.halt_episode(BROKER)
+ assert ep2 == "2026-08-27T10:00:00+00:00"
+ assert sweep_latch.sweep_already_fired(BROKER, ep2) is False
+ # A subsequent runner must therefore sweep the NEW episode (not skip it).
+ asyncio.run(_build_runner(live_root, fired).run_once())
+ assert fired == [BROKER, BROKER]
+
+
+def test_concurrent_latch_updates_both_episodes_recorded(
+ live_root: Path,
+) -> None:
+ # Reviewer finding 2: two concurrent completions for DIFFERENT episodes
+ # must not lose one another's record (the old shared read-merge-write
+ # could drop an entry). Per-episode latch files make each create
+ # independent — even with true thread interleaving, both must survive.
+ import threading
+ from concurrent.futures import ThreadPoolExecutor
+
+ barriers = [threading.Barrier(2)]
+ results: list[bool] = []
+
+ def _marker(episode: str, barrier: threading.Barrier) -> None:
+ barrier.wait()
+ sweep_latch.mark_sweep_fired(BROKER, [episode])
+ results.append(True)
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ pool.submit(_marker, "2026-08-27T01:00:00+00:00", barriers[0])
+ pool.submit(_marker, "2026-08-27T10:00:00+00:00", barriers[0])
+ assert len(results) == 2
+ assert (
+ sweep_latch.sweep_already_fired(BROKER, "2026-08-27T01:00:00+00:00")
+ is True
+ )
+ assert (
+ sweep_latch.sweep_already_fired(BROKER, "2026-08-27T10:00:00+00:00")
+ is True
+ )