Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix: fall back from corrupt loader cache metadata

Rudy Mizrahi Celekliefa93a961793
diff --git a/agent/backtest/loaders/base.py b/agent/backtest/loaders/base.py
index c631fa9e2..e5d05c4f4 100644
--- a/agent/backtest/loaders/base.py
+++ b/agent/backtest/loaders/base.py
@@ -46,7 +46,9 @@ _SINGLE_CURRENCY_VENUES: tuple[tuple[re.Pattern[str], str], ...] = (
def _venue_currency(code: str) -> str | None:
"""Return the one currency a symbol's venue pool accepts, or None if unconstrained."""
code = str(code).strip()
- return next((cur for pattern, cur in _SINGLE_CURRENCY_VENUES if pattern.match(code)), None)
+ return next(
+ (cur for pattern, cur in _SINGLE_CURRENCY_VENUES if pattern.match(code)), None
+ )
def is_lse_symbol(code: str) -> bool:
@@ -152,8 +154,7 @@ def normalize_lse_quote_currency(
else:
label = declared or "missing"
raise ValueError(
- "LSE quote currency must be declared as GBP or GBp; "
- f"got {label!r}"
+ "LSE quote currency must be declared as GBP or GBp; " f"got {label!r}"
)
normalized.attrs["quote_currency"] = "GBP"
@@ -179,7 +180,9 @@ def validate_date_range(start_date: str, end_date: str) -> None:
start = pd.Timestamp(start_date)
end = pd.Timestamp(end_date)
except Exception as exc:
- raise ValueError(f"Invalid date format: start={start_date!r}, end={end_date!r}") from exc
+ raise ValueError(
+ f"Invalid date format: start={start_date!r}, end={end_date!r}"
+ ) from exc
if start > end:
raise ValueError(f"start_date ({start_date}) > end_date ({end_date})")
@@ -232,11 +235,7 @@ def validate_ohlc(
open_, high, low, close = (frame[c] for c in required)
structural = (
- (high < low)
- | (high < open_)
- | (high < close)
- | (low > open_)
- | (low > close)
+ (high < low) | (high < open_) | (high < close) | (low > open_) | (low > close)
)
if allow_nonpositive_prices:
nonpositive = (open_ == 0) | (high == 0) | (low == 0) | (close == 0)
@@ -250,7 +249,9 @@ def validate_ohlc(
if strategy == "raise":
raise ValueError(f"{n_invalid} bar(s) violate OHLC invariants")
if strategy == "warn":
- logger.warning("OHLC validation: %d bar(s) violate invariants (kept)", n_invalid)
+ logger.warning(
+ "OHLC validation: %d bar(s) violate invariants (kept)", n_invalid
+ )
return frame
logger.warning("OHLC validation: dropping %d invalid bar(s)", n_invalid)
return frame[~invalid]
@@ -329,7 +330,9 @@ def resample_bars(frame: pd.DataFrame, interval: str) -> pd.DataFrame:
columns[column] = values.mean()
elif column == "vwap":
if "volume" in frame.columns:
- traded = (frame["vwap"] * frame["volume"]).groupby(periods).sum(min_count=1)
+ traded = (
+ (frame["vwap"] * frame["volume"]).groupby(periods).sum(min_count=1)
+ )
columns[column] = traded / grouped["volume"].sum(min_count=1)
else:
columns[column] = values.first() * float("nan")
@@ -388,7 +391,11 @@ def check_budget(deadline: float, label: str, budget_s: float | None = None) ->
message when present.
"""
if time.monotonic() > deadline:
- suffix = f" exceeded {budget_s:.0f}s budget" if budget_s is not None else " exceeded budget"
+ suffix = (
+ f" exceeded {budget_s:.0f}s budget"
+ if budget_s is not None
+ else " exceeded budget"
+ )
raise TimeoutError(f"{label}{suffix}")
@@ -447,7 +454,9 @@ def retry_with_budget(
f"{label} failed after {attempt + 1} attempt(s): {exc}"
) from exc
time.sleep(min(backoff[attempt], max(0.0, remaining)))
- raise AssertionError("unreachable: retry loop must return or raise") # pragma: no cover
+ raise AssertionError(
+ "unreachable: retry loop must return or raise"
+ ) # pragma: no cover
# ---------------------------------------------------------------------------
@@ -688,7 +697,9 @@ def _normalize_cache_date(value: str) -> str:
def _sanitize_cache_segment(value: str) -> str:
- cleaned = "".join(ch if ch.isalnum() or ch in {"-", "_"} else "_" for ch in value.strip().lower())
+ cleaned = "".join(
+ ch if ch.isalnum() or ch in {"-", "_"} else "_" for ch in value.strip().lower()
+ )
return cleaned or "unknown"
@@ -704,7 +715,9 @@ def _read_loader_cache_frame(cache_path: Path) -> pd.DataFrame | None:
try:
metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
except Exception as exc: # noqa: BLE001 - local cache miss is non-fatal
- logger.warning("loader cache metadata read failed for %s: %s", cache_path.name, exc)
+ logger.warning(
+ "loader cache metadata read failed for %s: %s", cache_path.name, exc
+ )
return None
con = None
@@ -722,26 +735,41 @@ def _read_loader_cache_frame(cache_path: Path) -> pd.DataFrame | None:
if con is not None:
con.close()
- index_columns = metadata.get("index_columns") or []
- if index_columns:
- missing = [column for column in index_columns if column not in frame.columns]
- if missing:
- logger.warning("loader cache %s missing index column(s): %s", cache_path.name, missing)
- return None
- frame = frame.set_index(index_columns)
- frame.index.names = metadata.get("index_names") or index_columns
- frame = _restore_cache_index_dtypes(frame, metadata.get("index_dtypes"))
- frame.columns.name = metadata.get("columns_name")
- frame_attrs = metadata.get("frame_attrs")
- if isinstance(frame_attrs, dict):
- for name in _LOADER_FRAME_METADATA_ATTRS:
- value = frame_attrs.get(name)
- if isinstance(value, str):
- frame.attrs[name] = value
+ try:
+ index_columns = metadata.get("index_columns") or []
+ if index_columns:
+ missing = [
+ column for column in index_columns if column not in frame.columns
+ ]
+ if missing:
+ logger.warning(
+ "loader cache %s missing index column(s): %s",
+ cache_path.name,
+ missing,
+ )
+ return None
+ frame = frame.set_index(index_columns)
+ frame.index.names = metadata.get("index_names") or index_columns
+ frame = _restore_cache_index_dtypes(frame, metadata.get("index_dtypes"))
+ frame.columns.name = metadata.get("columns_name")
+ frame_attrs = metadata.get("frame_attrs")
+ if isinstance(frame_attrs, dict):
+ for name in _LOADER_FRAME_METADATA_ATTRS:
+ value = frame_attrs.get(name)
+ if isinstance(value, str):
+ frame.attrs[name] = value
+ except Exception as exc: # noqa: BLE001 - corrupt metadata is a cache miss
+ logger.warning(
+ "loader cache metadata restore failed for %s: %s", cache_path.name, exc
+ )
+ return None
+
return frame
-def _restore_cache_index_dtypes(frame: pd.DataFrame, index_dtypes: object) -> pd.DataFrame:
+def _restore_cache_index_dtypes(
+ frame: pd.DataFrame, index_dtypes: object
+) -> pd.DataFrame:
"""Best-effort restore of the per-level index dtypes recorded at write time.
Cosmetic and non-fatal: duckdb parquet may rewrite datetime resolution, so
@@ -780,7 +808,9 @@ def _write_loader_cache_frame(cache_path: Path, frame: pd.DataFrame) -> None:
con = duckdb.connect(database=":memory:")
try:
con.register("cache_frame", cache_frame)
- con.execute(f"COPY cache_frame TO {_duckdb_sql_string(tmp_path)} (FORMAT PARQUET)")
+ con.execute(
+ f"COPY cache_frame TO {_duckdb_sql_string(tmp_path)} (FORMAT PARQUET)"
+ )
finally:
con.close()
@@ -790,7 +820,9 @@ def _write_loader_cache_frame(cache_path: Path, frame: pd.DataFrame) -> None:
)
os.replace(tmp_path, cache_path)
os.replace(tmp_metadata_path, metadata_path)
- except Exception as exc: # noqa: BLE001 - cache write failures should not fail fetches
+ except (
+ Exception
+ ) as exc: # noqa: BLE001 - cache write failures should not fail fetches
logger.warning("loader cache write failed for %s: %s", cache_path.name, exc)
for path in (tmp_path, tmp_metadata_path):
try:
@@ -801,7 +833,9 @@ def _write_loader_cache_frame(cache_path: Path, frame: pd.DataFrame) -> None:
pass
-def _frame_for_loader_cache(frame: pd.DataFrame) -> tuple[pd.DataFrame, dict[str, object]]:
+def _frame_for_loader_cache(
+ frame: pd.DataFrame,
+) -> tuple[pd.DataFrame, dict[str, object]]:
cache_frame = frame.copy()
original_index_names = list(cache_frame.index.names)
columns_name = cache_frame.columns.name