Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(alpha): disclose benchmark universe degradation

Haozhe Wu2c1ccd76781a
diff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.py
index ca3ca593c..d2ea6b867 100644
--- a/agent/src/tools/alpha_bench_tool.py
+++ b/agent/src/tools/alpha_bench_tool.py
@@ -323,12 +323,15 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
ed = end.replace("-", "")
codes: list[str] = []
+ constituent_source = "tushare index_weight"
+ constituent_source_date: str | None = None
try:
weights = pro.index_weight(
index_code="399300.SZ", start_date=sd, end_date=ed
)
if weights is not None and not weights.empty:
latest_date = weights["trade_date"].max()
+ constituent_source_date = str(latest_date)
codes = (
weights[weights["trade_date"] == latest_date]["con_code"]
.drop_duplicates()
@@ -340,6 +343,8 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
if not codes:
codes = list(_CSI300_FALLBACK_CODES)
+ constituent_source = "hand-picked fallback"
+ constituent_source_date = None
logger.warning("csi300: using %d-name fallback (degraded run)", len(codes))
# Fetch raw daily in parallel — we need ``amount`` which the standard
@@ -381,6 +386,16 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
panel["vwap"] = safe_div(
panel["amount"] * 1000.0, panel["volume"] * 100.0 + 1.0
)
+ panel["_meta"] = {
+ "universe": "csi300",
+ # A terminal snapshot is forward-looking relative to the start of the
+ # requested interval; the static fallback is survivor-selected too.
+ "survivorship_bias": True,
+ "degraded": constituent_source != "tushare index_weight",
+ "constituent_source": constituent_source,
+ "constituent_source_date": constituent_source_date,
+ "constituent_count": len(codes),
+ }
return panel
@@ -396,14 +411,17 @@ def _load_sp500_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
"""
codes = _fetch_sp500_constituents()
constituent_source = "wikipedia"
+ constituent_source_date: str | None = _SP500_CONSTITUENT_SOURCE_DATE
if not codes:
codes = list(_SP500_FALLBACK_CODES)
constituent_source = "hand-picked fallback"
+ constituent_source_date = None
logger.warning("sp500: using %d-name fallback (degraded run)", len(codes))
logger.warning(
"SP500 universe uses current constituents (%s @ %s) → survivorship-biased",
- constituent_source, _SP500_CONSTITUENT_SOURCE_DATE,
+ constituent_source,
+ constituent_source_date,
)
# yfinance loader expects project-style symbols (``AAPL.US``).
@@ -423,8 +441,9 @@ def _load_sp500_panel(start: str, end: str) -> dict[str, pd.DataFrame]:
panel["_meta"] = {
"universe": "sp500",
"survivorship_bias": True,
+ "degraded": constituent_source == "hand-picked fallback",
"constituent_source": constituent_source,
- "constituent_source_date": _SP500_CONSTITUENT_SOURCE_DATE,
+ "constituent_source_date": constituent_source_date,
"constituent_count": len(codes),
}
return panel
diff --git a/agent/tests/test_alpha_bench_meta_survivorship.py b/agent/tests/test_alpha_bench_meta_survivorship.py
index 89d6caab9..2fb778b65 100644
--- a/agent/tests/test_alpha_bench_meta_survivorship.py
+++ b/agent/tests/test_alpha_bench_meta_survivorship.py
@@ -130,8 +130,8 @@ def test_html_context_includes_universe_meta_when_present(capsys, monkeypatch, _
def test_meta_key_absent_from_envelope_and_context_when_loader_has_none(capsys, monkeypatch, _reg, _captured_html_context):
- """csi300/other loaders return no _meta; final envelope/context must not gain a stray key."""
- rc, cap = _run(monkeypatch, capsys, _ns(universe="csi300"), _result())
+ """Loaders without _meta must not gain a stray downstream key."""
+ rc, cap = _run(monkeypatch, capsys, _ns(universe="btc-usdt"), _result())
assert rc == 0
envelope = _envelope(cap.out)
assert "meta" not in envelope
diff --git a/agent/tests/test_alpha_bench_universe_metadata.py b/agent/tests/test_alpha_bench_universe_metadata.py
new file mode 100644
index 000000000..85f28c788
--- /dev/null
+++ b/agent/tests/test_alpha_bench_universe_metadata.py
@@ -0,0 +1,139 @@
+"""Universe loaders disclose constituent provenance and degradation."""
+
+from __future__ import annotations
+
+import sys
+from types import SimpleNamespace
+
+import pandas as pd
+import pytest
+
+from src.tools import alpha_bench_tool as tool
+
+
+def _daily_frame() -> pd.DataFrame:
+ """Return one deterministic Tushare-style daily row."""
+ return pd.DataFrame(
+ {
+ "trade_date": ["20240102"],
+ "open": [10.0],
+ "high": [11.0],
+ "low": [9.0],
+ "close": [10.5],
+ "vol": [100.0],
+ "amount": [105.0],
+ }
+ )
+
+
+class _FakeTusharePro:
+ """Minimal Tushare Pro surface used by the CSI300 loader."""
+
+ def __init__(self, weights: pd.DataFrame | Exception) -> None:
+ self.weights = weights
+
+ def index_weight(self, **_kwargs) -> pd.DataFrame:
+ """Return configured weights or simulate an upstream failure."""
+ if isinstance(self.weights, Exception):
+ raise self.weights
+ return self.weights
+
+ def daily(self, **_kwargs) -> pd.DataFrame:
+ """Return deterministic OHLCV data for every requested code."""
+ return _daily_frame()
+
+
+def _install_fake_tushare(
+ monkeypatch: pytest.MonkeyPatch,
+ pro: _FakeTusharePro,
+) -> None:
+ """Install deterministic credentials and an in-memory Tushare module."""
+ monkeypatch.setattr(
+ tool,
+ "get_env_config",
+ lambda: SimpleNamespace(data=SimpleNamespace(tushare_token="test-token")),
+ )
+ monkeypatch.setitem(
+ sys.modules,
+ "tushare",
+ SimpleNamespace(pro_api=lambda _token: pro),
+ )
+
+
+def test_csi300_reports_terminal_index_weight_snapshot(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Healthy CSI300 loads disclose their terminal Tushare snapshot."""
+ weights = pd.DataFrame(
+ {
+ "trade_date": ["20240102", "20240131", "20240131"],
+ "con_code": ["600000.SH", "000001.SZ", "000002.SZ"],
+ }
+ )
+ _install_fake_tushare(monkeypatch, _FakeTusharePro(weights))
+
+ panel = tool._load_csi300_panel("2024-01-01", "2024-01-31")
+
+ assert panel["_meta"] == {
+ "universe": "csi300",
+ "survivorship_bias": True,
+ "degraded": False,
+ "constituent_source": "tushare index_weight",
+ "constituent_source_date": "20240131",
+ "constituent_count": 2,
+ }
+
+
+def test_csi300_reports_hand_picked_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Failed index lookup discloses the smaller survivor-selected roster."""
+ monkeypatch.setattr(
+ tool,
+ "_CSI300_FALLBACK_CODES",
+ ["600519.SH", "000001.SZ"],
+ )
+ _install_fake_tushare(monkeypatch, _FakeTusharePro(RuntimeError("offline")))
+
+ panel = tool._load_csi300_panel("2024-01-01", "2024-01-31")
+
+ assert panel["_meta"] == {
+ "universe": "csi300",
+ "survivorship_bias": True,
+ "degraded": True,
+ "constituent_source": "hand-picked fallback",
+ "constituent_source_date": None,
+ "constituent_count": 2,
+ }
+
+
+ ("codes", "source", "source_date", "degraded"),
+ [
+ (["AAPL"], "wikipedia", tool._SP500_CONSTITUENT_SOURCE_DATE, False),
+ ([], "hand-picked fallback", None, True),
+ ],
+)
+def test_sp500_source_date_matches_the_roster(
+ monkeypatch: pytest.MonkeyPatch,
+ codes: list[str],
+ source: str,
+ source_date: str | None,
+ degraded: bool,
+) -> None:
+ """SP500 metadata never assigns Wikipedia's date to a fallback list."""
+
+ class _Loader:
+ def fetch(self, *_args, **_kwargs) -> dict:
+ return {}
+
+ import backtest.loaders.registry as registry
+
+ monkeypatch.setattr(tool, "_fetch_sp500_constituents", lambda: codes)
+ monkeypatch.setattr(registry, "resolve_loader", lambda _market: _Loader())
+
+ panel = tool._load_sp500_panel("2024-01-01", "2024-01-31")
+
+ assert panel["_meta"]["constituent_source"] == source
+ assert panel["_meta"]["constituent_source_date"] == source_date
+ assert panel["_meta"]["degraded"] is degraded