"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(live): latch only the swept episodes under per-episode latch files
cgycoreybc37d4c7f11b
diff --git a/agent/src/live/runtime/runner.py b/agent/src/live/runtime/runner.pyindex e09f6f2f8..58602397f 100644--- a/agent/src/live/runtime/runner.py+++ b/agent/src/live/runtime/runner.py@@ -48,7 +48,7 @@ from src.live.runtime.flatten import flatten_and_cancelfrom src.live.runtime.jobstore import JobStorefrom src.live.runtime.sweep_latch import (claim_sweep,- halt_episode,+ halt_snapshot,mark_sweep_fired,release_claim,sweep_already_fired,@@ -640,15 +640,21 @@ class LiveRunner:"""if self._flatten_fired or self._submit_fn is None:return- if sweep_already_fired(self.broker):+ # Capture ONE coherent halt snapshot before claiming. The claim, the+ # already-fired check, the latch write and the release must all bind+ # to the SAME view of the halt state: re-reading the sentinels later+ # would let a mid-sweep clear/re-trip record an episode whose sweep+ # never ran (next runner would skip it — positions stay open).+ active_episodes, episode = halt_snapshot(self.broker)+ episode = episode or "unknown"+ if sweep_already_fired(self.broker, episode):self._flatten_fired = Truereturn- # Resolve the episode ONCE: the claim and its release in ``finally``- # must refer to the same episode. Re-resolving at release time can- # bind to a NEWER episode (halt cleared + re-tripped mid-sweep) and- # delete that episode's claim — even a different process's — which- # would unprotect a concurrent sweep of the new episode.- episode = halt_episode(self.broker) or "unknown"+ # The episode is resolved exactly once: the claim and its release in+ # ``finally`` must refer to the same one. Re-resolving at release time+ # can bind to a NEWER episode (halt cleared + re-tripped mid-sweep)+ # and delete that episode's claim — even a different process's —+ # which would unprotect a concurrent sweep of the new episode.if not claim_sweep(self.broker, episode):# Another process holds this episode's claim (or a crash left it# behind): the outcome is unknowable — re-sweeping could duplicate@@ -681,7 +687,7 @@ class LiveRunner:# and are not retryable — latch so a restart does not replay.self._flatten_fired = Truetry:- mark_sweep_fired(self.broker)+ mark_sweep_fired(self.broker, active_episodes)except Exception as persist_exc: # noqa: BLE001 — double failure# Both the sweep AND the durable latch write failed: the# in-memory flag still shields this process, and the@@ -750,7 +756,7 @@ class LiveRunner:returnself._flatten_fired = Truetry:- mark_sweep_fired(self.broker)+ mark_sweep_fired(self.broker, active_episodes)except Exception as exc: # noqa: BLE001 — persistence must not be silent# Durability failure AFTER a broker write: the in-memory flag# protects this process, but a restart sees no latch and coulddiff --git a/agent/src/live/runtime/sweep_latch.py b/agent/src/live/runtime/sweep_latch.pyindex 3b16df5c1..efc1beb53 100644--- a/agent/src/live/runtime/sweep_latch.py+++ b/agent/src/live/runtime/sweep_latch.py@@ -24,7 +24,6 @@ from __future__ import annotationsimport hashlibimport jsonimport os-import tempfilefrom datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any@@ -36,9 +35,20 @@ _LATCH_FILENAME = "FLATTEN_FIRED"_CLAIM_FILENAME = "FLATTEN_CLAIM"-def latch_path(broker: str) -> Path:- """Return the per-broker sweep latch path (not created here)."""- return broker_dir(broker) / _LATCH_FILENAME+def latch_path(broker: str, episode: str | None = None) -> Path:+ """Return the per-broker, per-episode sweep latch path (not created here).++ One file per halt episode (``FLATTEN_FIRED-<sha16>(episode)``): the sweep+ record is never a shared read-merge-write object, so concurrent+ completions for different episodes cannot lose one another's entry —+ each atomic ``O_CREAT|O_EXCL`` create is independent. ``episode=None``+ returns the legacy single-record path (pre-per-episode format, read+ for compatibility only).+ """+ if episode is None:+ return broker_dir(broker) / _LATCH_FILENAME+ digest = hashlib.sha256(episode.encode("utf-8")).hexdigest()[:16]+ return broker_dir(broker) / f"{_LATCH_FILENAME}-{digest}"def halt_episode(broker: str) -> str | None:@@ -146,7 +156,20 @@ def _active_episodes(broker: str) -> list[str]:other halt is still tripped. Both identities must therefore be latchedtogether."""- identities: list[str] = []+ return halt_snapshot(broker)[0]+++def halt_snapshot(broker: str) -> tuple[list[str], str | None]:+ """One coherent read of every tripped sentinel.++ Returns ``(active_episodes, newest_episode)`` from a single pass over+ both sentinels. The runner MUST capture this snapshot once, before+ claiming, and pass the captured episode list into+ :func:`mark_sweep_fired` — re-reading the sentinels later would let a+ mid-sweep clear/re-trip change what the latch records (recording an+ episode whose sweep never ran, which would silently skip it).+ """+ candidates: list[tuple[int, str]] = []for path, payload in ((broker_halt_path(broker), read_halt(broker)),(halt_path(), read_halt()),@@ -155,8 +178,11 @@ def _active_episodes(broker: str) -> list[str]:continueresolved = _sentinel_identity(path, payload)if resolved is not None:- identities.append(resolved[1])- return identities+ candidates.append(resolved)+ if not candidates:+ return [], None+ newest = max(candidates, key=lambda item: item[0])[1]+ return [identity for _, identity in candidates], newestdef _halt_episode(broker: str) -> str | None:@@ -168,26 +194,23 @@ def _halt_episode(broker: str) -> str | None:a newer global trip — nor an older global trip override a newerper-broker one. The newest sentinel therefore wins."""- candidates: list[tuple[int, str]] = []- for path, payload in (- (broker_halt_path(broker), read_halt(broker)),- (halt_path(), read_halt()),- ):- if payload is None:- continue- resolved = _sentinel_identity(path, payload)- if resolved is not None:- candidates.append(resolved)- if not candidates:- return None- return max(candidates, key=lambda item: item[0])[1]+ return halt_snapshot(broker)[1]++def sweep_already_fired(broker: str, episode: str | None = None) -> bool:+ """Return True when the sweep already fired for ``broker``'s ``episode``.-def sweep_already_fired(broker: str) -> bool:- """Return True when the sweep already fired for the current halt episode."""- episode = _halt_episode(broker)+ ``episode=None`` resolves the newest currently tripped episode (legacy+ signature). The per-episode latch file is authoritative; the legacy+ single-record file is consulted only as a compatibility fallback for+ latches written by pre-per-episode versions.+ """if episode is None:- return False+ episode = _halt_episode(broker)+ if episode is None:+ return False+ if latch_path(broker, episode).exists():+ return Truetry:record = json.loads(latch_path(broker).read_text(encoding="utf-8"))except (OSError, json.JSONDecodeError):@@ -200,52 +223,43 @@ def sweep_already_fired(broker: str) -> bool:return record.get("episode") == episode # legacy single-episode record-def mark_sweep_fired(broker: str) -> None:- """Record that the sweep fired for the currently tripped halt episodes.+def mark_sweep_fired(broker: str, episodes: list[str] | None = None) -> None:+ """Atomically record that the sweep fired for every episode in ``episodes``.- Atomic write (same-directory temp file + ``os.replace``), same contract- as the HALT sentinel. A no-op when no halt is tripped, since there is no- episode to bind the record to. The record accumulates every episode the- sweep has fired for on this broker (``episodes``), so clearing one halt- never re-fires the sweep for an older episode that was already swept.+ One ``O_CREAT|O_EXCL`` latch file per episode identity — no shared+ read-merge-write record exists to race on: each create is independent and+ idempotent (an existing file is a prior completion, left untouched). This+ is what makes concurrent completions for different episodes safe without+ a lock, and a re-trip never loses a completed episode's record.- Both the per-broker and the global sentinel may be tripped at the same- time: the sweep covers the halt state as a whole, so EVERY currently- active episode identity is recorded together. Recording only the newest- would leave the older still-active halt unrecorded — clearing the newer- would then re-fire the sweep while the other halt remains tripped- (redundant no-op re-sweep and audit noise).+ The caller MUST pass the episode snapshot it captured BEFORE claiming+ (see :func:`halt_snapshot`) — never re-read the sentinels here: a+ mid-sweep clear/re-trip would record an episode whose sweep never ran,+ causing the next runner to skip it. ``episodes=None`` resolves the+ currently active episodes (legacy signature, for standalone use)."""- active = _active_episodes(broker)- if not active:- return- path = latch_path(broker)- path.parent.mkdir(parents=True, exist_ok=True)- record: dict[str, Any] = {}- try:- existing = json.loads(path.read_text(encoding="utf-8"))- if isinstance(existing, dict) and isinstance(existing.get("episodes"), list):- record = existing- elif isinstance(existing, dict) and existing.get("episode"):- record = {"episodes": [existing["episode"]]}- except (OSError, json.JSONDecodeError):- pass- episodes = record.setdefault("episodes", [])- to_write = bool(episodes)- for episode in active:- if episode not in episodes:- episodes.append(episode)- to_write = True- if to_write:- record["episode"] = _halt_episode(broker) # legacy field = newest- record["fired_at"] = datetime.now(timezone.utc).isoformat()- fd, tmp = tempfile.mkstemp(dir=path.parent, prefix=".flatten-fired-")+ if episodes is None:+ episodes = _active_episodes(broker)+ for episode in episodes:+ path = latch_path(broker, episode)+ path.parent.mkdir(parents=True, exist_ok=True)+ try:+ fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY)+ except FileExistsError:+ continue # already latched by a prior completiontry:with os.fdopen(fd, "w", encoding="utf-8") as f:- json.dump(record, f)- os.replace(tmp, path)- finally:+ json.dump(+ {+ "fired_at": datetime.now(timezone.utc).isoformat(),+ "episode": episode,+ },+ f,+ )+ except Exception:try:- os.unlink(tmp)+ os.unlink(path)except OSError:pass+ raise # durability failure; caller decides (audit + flag)+diff --git a/agent/tests/test_sweep_latch.py b/agent/tests/test_sweep_latch.pyindex 19cc0325b..b346e5414 100644--- a/agent/tests/test_sweep_latch.py+++ b/agent/tests/test_sweep_latch.py@@ -41,6 +41,12 @@ def _trip_with_timestamp(broker: str | None, tripped_at: str) -> None:)+def _any_latch_files() -> bool:+ """True when any per-episode/legacy latch file exists for BROKER."""+ d = sweep_latch.latch_path(BROKER).parent+ return any(p.name.startswith("FLATTEN_FIRED") for p in d.glob("FLATTEN_FIRED*"))++def test_newer_global_halt_rearms_with_tied_mtimes(live_root: Path) -> None:# Regression for the combined-suite flake: broker + global sentinels# written within one filesystem timestamp quantum share an mtime.@@ -81,7 +87,7 @@ def test_fresh_halt_episode_rearms(live_root: Path) -> None:def test_mark_without_halt_is_noop(live_root: Path) -> None:sweep_latch.mark_sweep_fired(BROKER)- assert not sweep_latch.latch_path(BROKER).exists()+ assert not _any_latch_files()def test_hand_touched_halt_binds_via_mtime(live_root: Path) -> None:@@ -196,7 +202,7 @@ def test_read_failure_does_not_latch_and_restart_replays(live_root: Path) -> Nonasyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER]- assert not sweep_latch.latch_path(BROKER).exists() # nothing persisted+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists() # not latched# Restart: the failed sweep must re-fire, not be suppressed.asyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER, BROKER]@@ -216,7 +222,7 @@ def test_nothing_to_do_does_not_latch_and_rechecks(live_root: Path) -> None:asyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER]- assert not sweep_latch.latch_path(BROKER).exists()+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()asyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER, BROKER]@@ -241,7 +247,7 @@ def test_side_effect_attempted_latches_even_with_read_error(asyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER]- assert sweep_latch.latch_path(BROKER).exists() # latched despite the error+ assert sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()# Restart: the on-disk latch suppresses the duplicate close.asyncio.run(_build_runner(live_root, fired, _flatten).run_once())assert fired == [BROKER]@@ -316,7 +322,7 @@ def test_runner_claim_released_after_read_failure(live_root: Path) -> None:assert fired == [BROKER]ep = sweep_latch.halt_episode(BROKER)assert not sweep_latch.claim_path(BROKER, ep).exists()- assert not sweep_latch.latch_path(BROKER).exists()+ assert not sweep_latch.latch_path(BROKER, "2026-08-27T01:00:00+00:00").exists()def test_mark_records_both_active_episodes_older_global_newer_broker(@@ -445,3 +451,60 @@ def test_persistent_read_failure_audits_the_breach_once_per_episode(total, kinds = _count_audits(live_root, _flatten, ticks=5)assert total == 6, kindsassert kinds.count("breach") == 1, kinds+++def test_mid_sweep_retrip_records_only_swept_episode(live_root: Path) -> None:+ # Reviewer merge-blocker: the latch must record the episodes the sweep+ # COVERED (captured before claiming), never what the sentinels say AFTER+ # the sweep. With a mid-sweep clear + re-trip (ep1 -> ep2), recording ep2+ # would make the next runner skip it — positions stay open.+ _trip_with_timestamp(BROKER, "2026-08-27T01:00:00+00:00")+ fired: list[str] = []++ def _retripping_flatten(broker, submit, read_positions, read_open_orders):+ fired.append(broker)+ clear_halt(BROKER)+ _trip_with_timestamp(BROKER, "2026-08-27T10:00:00+00:00")+ return {"side_effects_attempted": True}++ asyncio.run(_build_runner(live_root, fired, _retripping_flatten).run_once())+ # ep1 (the swept episode) is latched; ep2 (never swept) is not.+ assert sweep_latch.sweep_already_fired(BROKER, "2026-08-27T01:00:00+00:00") is True+ ep2 = sweep_latch.halt_episode(BROKER)+ assert ep2 == "2026-08-27T10:00:00+00:00"+ assert sweep_latch.sweep_already_fired(BROKER, ep2) is False+ # A subsequent runner must therefore sweep the NEW episode (not skip it).+ asyncio.run(_build_runner(live_root, fired).run_once())+ assert fired == [BROKER, BROKER]+++def test_concurrent_latch_updates_both_episodes_recorded(+ live_root: Path,+) -> None:+ # Reviewer finding 2: two concurrent completions for DIFFERENT episodes+ # must not lose one another's record (the old shared read-merge-write+ # could drop an entry). Per-episode latch files make each create+ # independent — even with true thread interleaving, both must survive.+ import threading+ from concurrent.futures import ThreadPoolExecutor++ barriers = [threading.Barrier(2)]+ results: list[bool] = []++ def _marker(episode: str, barrier: threading.Barrier) -> None:+ barrier.wait()+ sweep_latch.mark_sweep_fired(BROKER, [episode])+ results.append(True)++ with ThreadPoolExecutor(max_workers=2) as pool:+ pool.submit(_marker, "2026-08-27T01:00:00+00:00", barriers[0])+ pool.submit(_marker, "2026-08-27T10:00:00+00:00", barriers[0])+ assert len(results) == 2+ assert (+ sweep_latch.sweep_already_fired(BROKER, "2026-08-27T01:00:00+00:00")+ is True+ )+ assert (+ sweep_latch.sweep_already_fired(BROKER, "2026-08-27T10:00:00+00:00")+ is True+ )