"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(backtest): select and report the benchmark actually evaluated
Haozhe Wu642615560c32
diff --git a/agent/backtest/benchmark.py b/agent/backtest/benchmark.pyindex 0f1c69d3d..42797ced8 100644--- a/agent/backtest/benchmark.py+++ b/agent/backtest/benchmark.py@@ -11,6 +11,7 @@ from typing import Any, Optionalimport pandas as pd+from backtest.engines._market_hooks import matched_market, strip_local_prefixfrom backtest.loaders.base import resample_bars, source_intervalfrom backtest.loaders.registry import is_no_network_fallback_sourcefrom backtest.loaders.yfinance_loader import DataLoader as YfinanceLoader@@ -27,8 +28,12 @@ MARKET_BENCHMARKS: dict[str, Optional[str]] = {"ca_equity": "XIC.TO", # S&P/TSX Capped Composite ETF"br_equity": "BOVA11.SA", # iShares Ibovespa ETF"a_share": "000300.SH", # CSI 300 (China A-share core index)+ "uk_equity": "^FTSE",+ "india_equity": "^NSEI",+ "kr_equity": "^KS11",+ "ar_equity": "^MERV","crypto": "BTC-USDT",- "futures": "ES.CME", # E-mini S&P 500 futures+ "futures": None, # no common reference across contracts and venues"forex": None, # no universal benchmark}@@ -93,7 +98,7 @@ def resolve_benchmark(BenchmarkResult with return series and total return, or None if nobenchmark applies (forex, or fetch failure)."""- ticker = _resolve_ticker(strategy_codes, source, explicit)+ ticker = explicit or default_benchmark(strategy_codes, source)if ticker is None:return None@@ -137,62 +142,26 @@ def resolve_benchmark(# Internal helpers# --------------------------------------------------------------------def _resolve_ticker(- codes: list[str],- source: str,- explicit: Optional[str],-) -> Optional[str]:- """Pick the benchmark ticker to use."""-- if explicit:- return explicit-- # Infer market from source + first code pattern+def default_benchmark(codes: list[str], source: str) -> str | None:+ """Select a default only when all evaluated instruments identify one market."""market = _infer_market(codes, source)- ticker = MARKET_BENCHMARKS.get(market)-- # yfinance is the universal fallback for benchmark fetch- # but it only works for global-equity market types- if ticker and market not in {"us_equity", "hk_equity", "ca_equity"}:- # Only use benchmark if we can actually fetch it- pass-- return ticker---def _infer_market(codes: list[str], source: str) -> str:- """Rough market inference from symbol patterns and source."""- if not codes:- return "us_equity"-- first = codes[0].upper()-- # Yahoo uses .SS for Shanghai; other loaders use the repository's .SH form.- if first.endswith((".SH", ".SS", ".SZ", ".BJ")):- return "a_share"- if first.endswith(".US"):- return "us_equity"- if first.endswith(".HK"):- return "hk_equity"- if first.endswith((".TO", ".V")):- return "ca_equity"- if first.endswith((".NS", ".BO")):- return "india_equity"- if first.endswith(".SA"):- return "br_equity"- if first.endswith((".KS", ".KQ")):- return "kr_equity"- crypto_quotes = ("-USDT", "-USDC", "-USD", "-BTC", "-ETH")- if source in ("okx", "ccxt", "binance") or "/" in first or first.endswith(crypto_quotes):- return "crypto"- if source in ("tushare", "akshare"):- if first.isdigit() and len(first) == 6:- return "a_share"- if first.startswith(("IF", "IC", "IH", "IM", "T", "TF")):- return "futures"- return "a_share"-- return "us_equity"+ return MARKET_BENCHMARKS.get(market) if market else None+++def _infer_market(codes: list[str], source: str) -> str | None:+ """Use the engine's market rules without its default for unknown instruments."""+ markets = set()+ for code in codes:+ symbol = strip_local_prefix(code).strip().upper()+ # Yahoo uses .SS for Shanghai. Bare six-digit codes are unambiguous+ # on the Chinese stock sources, but not on a generic loader.+ if symbol.endswith(".SS") and symbol[:-3].isdigit():+ symbol = symbol[:-3] + ".SH"+ elif source in {"akshare", "tushare", "baostock"} and len(symbol) == 6 and symbol.isdigit():+ markets.add("a_share")+ continue+ markets.add(matched_market(symbol))+ return markets.pop() if len(markets) == 1 else Nonedef _fetch_benchmark(diff --git a/agent/backtest/engines/_market_hooks.py b/agent/backtest/engines/_market_hooks.pyindex b9b78e07d..923a77e7b 100644--- a/agent/backtest/engines/_market_hooks.py+++ b/agent/backtest/engines/_market_hooks.py@@ -372,11 +372,13 @@ def _detect_market(code: str) -> str:``=F`` (futures) and ``=X`` (forex) notations are recognized;any other unknown format defaults to ``a_share``."""+ return matched_market(code) or "a_share"+++def matched_market(code: str) -> str | None:+ """Return the market matched by a routing rule, or None for unknown symbols."""symbol = strip_local_prefix(code)- for pattern, market in _MARKET_PATTERNS:- if pattern.match(symbol):- return market- return "a_share"+ return next((market for pattern, market in _MARKET_PATTERNS if pattern.match(symbol)), None)def _is_china_futures(code: str) -> bool:diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.pyindex d6ba9a422..3b7e462b4 100644--- a/agent/backtest/engines/base.py+++ b/agent/backtest/engines/base.py@@ -1028,54 +1028,37 @@ class BaseEngine(ABC):index=[s.timestamp for s in self.equity_snapshots],)bench_ret = ret_df.mean(axis=1) if ret_df.shape[1] > 0 else pd.Series(0.0, index=dates)- benchmark_metadata = {}- benchmark_warning: str | None = None-- # ── External benchmark fetch ──────────────────────────────────────────- bench_ticker = config.get("benchmark")- if bench_ticker and bench_ticker != "auto":- from backtest.benchmark import resolve_benchmark- bench_source = config.get("source", "yfinance")- bench_result = resolve_benchmark(- strategy_codes=codes,- source=bench_source,- start_date=config.get("start_date", ""),- end_date=config.get("end_date", ""),- interval=interval,- explicit=bench_ticker,- # Explicit source: fetch the benchmark through its own loader- # (keeps e.g. source=local offline). Auto keeps the yfinance- # default — its loader only wraps the preloaded strategy data.- loader=loader if bench_source != "auto" else None,+ from backtest.benchmark import default_benchmark, resolve_benchmark++ basket_codes = list(ret_df.columns)+ basket_label = (+ f"equal-weight basket ({basket_codes[0]})" if len(basket_codes) == 1+ else f"equal-weight basket of {len(basket_codes)} symbols"+ )+ benchmark_metadata = {"benchmark_kind": "equal_weight", "benchmark_label": basket_label}+ requested = config.get("benchmark")+ source = config.get("source", "yfinance")+ ticker = default_benchmark(basket_codes, source) if requested == "auto" else requested+ reason = "no default benchmark for the evaluated symbols" if requested == "auto" else "no benchmark requested"+ if ticker:+ result = resolve_benchmark(+ strategy_codes=basket_codes, source=source, explicit=ticker,+ start_date=config.get("start_date", ""), end_date=config.get("end_date", ""),+ interval=interval, loader=loader if source != "auto" else None,)- if bench_result is not None:- bench_ret = bench_result.ret_series.reindex(dates).fillna(0.0)- # The benchmark is fetched over the requested start_date..- # end_date, but a declared warm-up boundary makes the EVALUATED- # window shorter than that (`dates` is already clipped above).- # Grading total_return over the short window against a- # benchmark measured over the long one is the mismatched-window- # error the warm-up boundary exists to prevent, so re-measure- # the benchmark over the same bars. Falls back to the fetched- # total only when the overlap is too short to measure.- window_ret = bench_result.total_return_over(dates)+ window_return = result.total_return_over(dates) if result is not None else None+ if result is not None and window_return is not None:+ bench_ret = bar_returns(result.close.reindex(dates), label=f"benchmark {result.ticker}")benchmark_metadata = {- "benchmark_ticker": bench_result.ticker,- "benchmark_return": (- window_ret if window_ret is not None else bench_result.total_ret- ),+ "benchmark_kind": "external", "benchmark_label": result.ticker,+ "benchmark_ticker": result.ticker, "benchmark_return": window_return,}else:- # The equal-weight basket stands in for the declared benchmark,- # so every benchmark-relative metric (excess_return,- # information_ratio, tracking_error, benchmark_beta) is graded- # against something the caller did not ask for. Say so.- benchmark_warning = (- f"declared benchmark {bench_ticker!r} could not be resolved; "- "benchmark-relative metrics are graded against the "- "equal-weight basket of the traded symbols instead"- )- # ── External benchmark fetch ──────────────────────────────────────────+ reason = f"benchmark {ticker!r} could not be resolved over the evaluation window"+ benchmark_warning = (+ f"{reason}; benchmark-relative metrics use the {basket_label}"+ if benchmark_metadata["benchmark_kind"] == "equal_weight" else None+ )bench_equity = self.initial_capital * (1 + bench_ret).cumprod()diff --git a/agent/src/api/attribution_core.py b/agent/src/api/attribution_core.pyindex e37ff9380..c32909c99 100644--- a/agent/src/api/attribution_core.py+++ b/agent/src/api/attribution_core.py@@ -823,17 +823,18 @@ def _build_attribution_payload(run_dir: Path) -> Dict[str, Any]:config_benchmark = config.get("benchmark")if isinstance(config_benchmark, dict):config_benchmark = config_benchmark.get("ticker") or config_benchmark.get("code")- if config_benchmark is not None and str(config_benchmark).strip():- benchmark_ticker = str(config_benchmark).strip()- if benchmark_ticker is None:- _metrics_fields, metrics_rows = _attribution_read_csv(run_dir / "artifacts" / "metrics.csv")- if metrics_rows:- candidate = metrics_rows[0].get("benchmark_ticker")- if candidate is not None and str(candidate).strip():- benchmark_ticker = str(candidate).strip()+ _, metrics_rows = _attribution_read_csv(run_dir / "artifacts" / "metrics.csv")+ recorded = metrics_rows[0] if metrics_rows else {}+ # New runs record the reference actually used, including a fallback after+ # an explicit fetch failed. Older runs only had a ticker or the request.+ if recorded.get("benchmark_kind") != "equal_weight":+ candidate = recorded.get("benchmark_ticker") or config_benchmark+ if candidate and str(candidate).strip() not in {"", "auto"}:+ benchmark_ticker = str(candidate).strip()if benchmark_ticker is not None:- payload["benchmark"] = {"ticker": benchmark_ticker, "mode": "explicit"}+ mode = "auto_market" if config_benchmark == "auto" else "explicit"+ payload["benchmark"] = {"ticker": benchmark_ticker, "mode": mode}else:payload["benchmark"] = {"ticker": None, "mode": "auto_equal_weight"}diff --git a/agent/src/skills/backtest-diagnose/SKILL.md b/agent/src/skills/backtest-diagnose/SKILL.mdindex bdc5269e2..35a75c868 100644--- a/agent/src/skills/backtest-diagnose/SKILL.md+++ b/agent/src/skills/backtest-diagnose/SKILL.md@@ -35,6 +35,7 @@ Use this skill when a user reports that a backtest failed, raised an error, or p2. **Late trades** (first trade occurs more than 2 years after the backtest start): data-filtering bug. The lookback window may be too long, or the initial data segment may have been dropped. Shorten the window or check whether `dropna` is too aggressive.3. **Capital utilization < 50%** (mostly in cash): position-sizing bug. Signal triggers may be too sparse, or the position-sizing logic may be wrong.4. **Open position at the end** (a position still exists when the backtest ends): exit-timing bug. Forced liquidation may be missing, or exit logic does not cover the final segment.+5. **Unexpected excess return**: check `benchmark_label` in the run metrics and the warnings in `run_card.json`. An omitted benchmark uses the evaluated symbols’ equal-weight returns. Set `"benchmark": "auto"` to request a market reference, or supply an explicit ticker. If no common default applies or the reference lacks usable data, the run records the equal-weight fallback.### Data Errorsdiff --git a/agent/tests/test_attribution_api.py b/agent/tests/test_attribution_api.pyindex 322371707..ea965951e 100644--- a/agent/tests/test_attribution_api.py+++ b/agent/tests/test_attribution_api.py@@ -190,6 +190,25 @@ def test_attribution_auto_mode_factor_and_symbol_brinson(tmp_path: Path, monkeypassert any("symbol-mode" in note for note in data["notes"])+def test_attribution_uses_executed_reference_after_explicit_fetch_failed(tmp_path, monkeypatch):+ client = _client(tmp_path, monkeypatch)+ run_dir = _build_auto_run(tmp_path, "fallback")+ _write_config(run_dir, [SYMBOL_A, SYMBOL_B], benchmark="SPY")+ _write_metrics(run_dir, extra={"benchmark_kind": "equal_weight"})+ data = client.get("/runs/fallback/attribution").json()+ assert data["benchmark"] == {"ticker": None, "mode": "auto_equal_weight"}+ assert data["brinson"]["mode"] == "symbol"+++def test_attribution_names_the_resolved_auto_ticker(tmp_path, monkeypatch):+ client = _client(tmp_path, monkeypatch)+ run_dir = _build_explicit_run(tmp_path, "market")+ _write_config(run_dir, ["600519.SH"], benchmark="auto")+ _write_metrics(run_dir, extra={"benchmark_kind": "external", "benchmark_ticker": "000300.SH"})+ data = client.get("/runs/market/attribution").json()+ assert data["benchmark"] == {"ticker": "000300.SH", "mode": "auto_market"}++def test_attribution_explicit_mode_asset_class_with_cash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:client = _client(tmp_path, monkeypatch)_build_explicit_run(tmp_path, "attr-explicit")diff --git a/agent/tests/test_benchmark_excess_return_consistency.py b/agent/tests/test_benchmark_excess_return_consistency.pyindex 864dadee0..60fdad5f8 100644--- a/agent/tests/test_benchmark_excess_return_consistency.py+++ b/agent/tests/test_benchmark_excess_return_consistency.py@@ -97,8 +97,7 @@ def test_excess_return_matches_corrected_benchmark_return(monkeypatch, tmp_path)def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):- """No config["benchmark"] set: benchmark_metadata stays empty, so the fix- must not touch excess_return at all on the ordinary, unconfigured path."""+ """Only the loaded symbol contributes to the unconfigured reference."""dates = pd.bdate_range("2026-01-05", periods=3)bars = pd.DataFrame({@@ -111,7 +110,7 @@ def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):index=dates,)config = {- "codes": ["000001.SZ"],+ "codes": ["000002.SZ", "000001.SZ"],"start_date": str(dates[0].date()),"end_date": str(dates[-1].date()),"source": "tushare",@@ -121,5 +120,7 @@ def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):engine = ChinaAEngine({"initial_cash": 1_000_000.0})metrics = engine.run_backtest(config, _FakeLoader(bars), _FlatSignal(), tmp_path)- assert "benchmark_return" not in metrics or metrics.get("benchmark_ticker") is None+ assert metrics["benchmark_kind"] == "equal_weight"+ assert "benchmark_ticker" not in metrics+ assert metrics["benchmark_label"] == "equal-weight basket (000001.SZ)"assert metrics["excess_return"] == pytest.approx(0.0)diff --git a/agent/tests/test_benchmark_exchange_suffix_regressions.py b/agent/tests/test_benchmark_exchange_suffix_regressions.pyindex 7bf809b80..ecb2ef6d1 100644--- a/agent/tests/test_benchmark_exchange_suffix_regressions.py+++ b/agent/tests/test_benchmark_exchange_suffix_regressions.py@@ -27,3 +27,15 @@ def test_explicit_ashare_suffix_selects_csi300_benchmark(symbol, source):assert result is not Noneassert result.ticker == "000300.SH"assert result.total_ret == pytest.approx(0.1)++[email protected](("codes", "ticker"), [+ (["VOD.L"], "^FTSE"), (["GGAL.BA"], "^MERV"), (["005930.KS"], "^KS11"),+ (["7203.T"], None), (["EURUSD=X"], None), (["VIC.VN"], None),+ (["IF2406.CFFEX"], None), (["AAPL.US", "0700.HK"], None),+ (["local:VOD.L", "SHEL.L"], "^FTSE"),+])+def test_auto_requires_a_shared_identified_market(codes, ticker):+ from backtest.benchmark import default_benchmark++ assert default_benchmark(codes, "local") == tickerdiff --git a/agent/tests/test_offline_benchmark.py b/agent/tests/test_offline_benchmark.pyindex 814cd9a22..399f19298 100644--- a/agent/tests/test_offline_benchmark.py+++ b/agent/tests/test_offline_benchmark.py@@ -151,6 +151,18 @@ def test_networked_run_keeps_the_snapshot_wrapper_away_from_its_loader(assert "benchmark_ticker" not in card["metrics"]+def test_nonoverlapping_external_prices_cannot_grade_the_run(tmp_path, monkeypatch):+ unrelated = _bars(_BENCH_CLOSE)+ unrelated.index += pd.Timedelta(days=30)+ card, _ = _run_main(+ tmp_path, monkeypatch, source="local",+ frames={"AAPL.US": _bars(_STOCK_CLOSE), _BENCH: unrelated},+ )+ assert card["metrics"]["benchmark_kind"] == "equal_weight"+ assert card["metrics"]["benchmark_return"] == pytest.approx(0.4)+ assert any(_UNRESOLVED_MARKER in w for w in card["warnings"])++def test_snapshot_wrapper_without_inner_loader_serves_only_the_snapshot() -> None:frame = _bars(_STOCK_CLOSE)loader = runner._AutoLoader({"AAPL.US": frame})@@ -211,15 +223,16 @@ def test_source_bound_benchmark_never_switches_provider(monkeypatch, source, loaassert network_calls == [][email protected]("requested,ticker", [(_BENCH, _BENCH), ("auto", "SPY")])@pytest.mark.parametrize("benchmark_available", [True, False])-def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, benchmark_available):+def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, benchmark_available, requested, ticker):"""Exercise CSV -> actual loader -> runner -> run card with no network."""from backtest.loaders import local_loaderbridge = tmp_path / "bridge.json"symbols = [("AAPL.US", _STOCK_CLOSE)]if benchmark_available:- symbols.append((_BENCH, _BENCH_CLOSE))+ symbols.append((ticker, _BENCH_CLOSE))sources = []for symbol, closes in symbols:path = tmp_path / f"{symbol}.csv"@@ -237,7 +250,7 @@ def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, ben" return {code: frame['close'] * 0 for code, frame in data_map.items()}\n")(run / "config.json").write_text(json.dumps({- "codes": ["AAPL.US"], "source": "local", "benchmark": _BENCH,+ "codes": ["AAPL.US"], "source": "local", "benchmark": requested,"start_date": "2026-01-05", "end_date": "2026-01-09", "initial_cash": 100_000,}))network_calls = []@@ -252,10 +265,12 @@ def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, bencard = json.loads((run / "run_card.json").read_text())assert network_calls == []if benchmark_available:- assert card["metrics"]["benchmark_ticker"] == _BENCH+ assert card["metrics"]["benchmark_ticker"] == tickerassert card["metrics"]["benchmark_return"] == pytest.approx(0.02)assert card["metrics"]["excess_return"] == pytest.approx(-0.02)assert not any(_UNRESOLVED_MARKER in w for w in card.get("warnings") or [])else:+ assert card["metrics"]["benchmark_kind"] == "equal_weight"+ assert card["metrics"]["benchmark_label"] == "equal-weight basket (AAPL.US)"assert "benchmark_ticker" not in card["metrics"]assert any(_UNRESOLVED_MARKER in w for w in card["warnings"])diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.tsindex 78748adac..9c839da20 100644--- a/frontend/src/lib/api.ts+++ b/frontend/src/lib/api.ts@@ -1421,7 +1421,7 @@ export interface FactorReportPayload {export interface AttributionBenchmarkInfo {ticker: string | null;- mode: "auto_equal_weight" | "explicit";+ mode: "auto_equal_weight" | "auto_market" | "explicit";}export interface AttributionRollingPoint {