Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(backtest): select and report the benchmark actually evaluated

Haozhe Wu642615560c32
diff --git a/agent/backtest/benchmark.py b/agent/backtest/benchmark.py
index 0f1c69d3d..42797ced8 100644
--- a/agent/backtest/benchmark.py
+++ b/agent/backtest/benchmark.py
@@ -11,6 +11,7 @@ from typing import Any, Optional
import pandas as pd
+from backtest.engines._market_hooks import matched_market, strip_local_prefix
from backtest.loaders.base import resample_bars, source_interval
from backtest.loaders.registry import is_no_network_fallback_source
from backtest.loaders.yfinance_loader import DataLoader as YfinanceLoader
@@ -27,8 +28,12 @@ MARKET_BENCHMARKS: dict[str, Optional[str]] = {
"ca_equity": "XIC.TO", # S&P/TSX Capped Composite ETF
"br_equity": "BOVA11.SA", # iShares Ibovespa ETF
"a_share": "000300.SH", # CSI 300 (China A-share core index)
+ "uk_equity": "^FTSE",
+ "india_equity": "^NSEI",
+ "kr_equity": "^KS11",
+ "ar_equity": "^MERV",
"crypto": "BTC-USDT",
- "futures": "ES.CME", # E-mini S&P 500 futures
+ "futures": None, # no common reference across contracts and venues
"forex": None, # no universal benchmark
}
@@ -93,7 +98,7 @@ def resolve_benchmark(
BenchmarkResult with return series and total return, or None if no
benchmark applies (forex, or fetch failure).
"""
- ticker = _resolve_ticker(strategy_codes, source, explicit)
+ ticker = explicit or default_benchmark(strategy_codes, source)
if ticker is None:
return None
@@ -137,62 +142,26 @@ def resolve_benchmark(
# Internal helpers
# -------------------------------------------------------------------
-def _resolve_ticker(
- codes: list[str],
- source: str,
- explicit: Optional[str],
-) -> Optional[str]:
- """Pick the benchmark ticker to use."""
-
- if explicit:
- return explicit
-
- # Infer market from source + first code pattern
+def default_benchmark(codes: list[str], source: str) -> str | None:
+ """Select a default only when all evaluated instruments identify one market."""
market = _infer_market(codes, source)
- ticker = MARKET_BENCHMARKS.get(market)
-
- # yfinance is the universal fallback for benchmark fetch
- # but it only works for global-equity market types
- if ticker and market not in {"us_equity", "hk_equity", "ca_equity"}:
- # Only use benchmark if we can actually fetch it
- pass
-
- return ticker
-
-
-def _infer_market(codes: list[str], source: str) -> str:
- """Rough market inference from symbol patterns and source."""
- if not codes:
- return "us_equity"
-
- first = codes[0].upper()
-
- # Yahoo uses .SS for Shanghai; other loaders use the repository's .SH form.
- if first.endswith((".SH", ".SS", ".SZ", ".BJ")):
- return "a_share"
- if first.endswith(".US"):
- return "us_equity"
- if first.endswith(".HK"):
- return "hk_equity"
- if first.endswith((".TO", ".V")):
- return "ca_equity"
- if first.endswith((".NS", ".BO")):
- return "india_equity"
- if first.endswith(".SA"):
- return "br_equity"
- if first.endswith((".KS", ".KQ")):
- return "kr_equity"
- crypto_quotes = ("-USDT", "-USDC", "-USD", "-BTC", "-ETH")
- if source in ("okx", "ccxt", "binance") or "/" in first or first.endswith(crypto_quotes):
- return "crypto"
- if source in ("tushare", "akshare"):
- if first.isdigit() and len(first) == 6:
- return "a_share"
- if first.startswith(("IF", "IC", "IH", "IM", "T", "TF")):
- return "futures"
- return "a_share"
-
- return "us_equity"
+ return MARKET_BENCHMARKS.get(market) if market else None
+
+
+def _infer_market(codes: list[str], source: str) -> str | None:
+ """Use the engine's market rules without its default for unknown instruments."""
+ markets = set()
+ for code in codes:
+ symbol = strip_local_prefix(code).strip().upper()
+ # Yahoo uses .SS for Shanghai. Bare six-digit codes are unambiguous
+ # on the Chinese stock sources, but not on a generic loader.
+ if symbol.endswith(".SS") and symbol[:-3].isdigit():
+ symbol = symbol[:-3] + ".SH"
+ elif source in {"akshare", "tushare", "baostock"} and len(symbol) == 6 and symbol.isdigit():
+ markets.add("a_share")
+ continue
+ markets.add(matched_market(symbol))
+ return markets.pop() if len(markets) == 1 else None
def _fetch_benchmark(
diff --git a/agent/backtest/engines/_market_hooks.py b/agent/backtest/engines/_market_hooks.py
index b9b78e07d..923a77e7b 100644
--- a/agent/backtest/engines/_market_hooks.py
+++ b/agent/backtest/engines/_market_hooks.py
@@ -372,11 +372,13 @@ def _detect_market(code: str) -> str:
``=F`` (futures) and ``=X`` (forex) notations are recognized;
any other unknown format defaults to ``a_share``.
"""
+ return matched_market(code) or "a_share"
+
+
+def matched_market(code: str) -> str | None:
+ """Return the market matched by a routing rule, or None for unknown symbols."""
symbol = strip_local_prefix(code)
- for pattern, market in _MARKET_PATTERNS:
- if pattern.match(symbol):
- return market
- return "a_share"
+ return next((market for pattern, market in _MARKET_PATTERNS if pattern.match(symbol)), None)
def _is_china_futures(code: str) -> bool:
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.py
index d6ba9a422..3b7e462b4 100644
--- a/agent/backtest/engines/base.py
+++ b/agent/backtest/engines/base.py
@@ -1028,54 +1028,37 @@ class BaseEngine(ABC):
index=[s.timestamp for s in self.equity_snapshots],
)
bench_ret = ret_df.mean(axis=1) if ret_df.shape[1] > 0 else pd.Series(0.0, index=dates)
- benchmark_metadata = {}
- benchmark_warning: str | None = None
-
- # ── External benchmark fetch ──────────────────────────────────────────
- bench_ticker = config.get("benchmark")
- if bench_ticker and bench_ticker != "auto":
- from backtest.benchmark import resolve_benchmark
- bench_source = config.get("source", "yfinance")
- bench_result = resolve_benchmark(
- strategy_codes=codes,
- source=bench_source,
- start_date=config.get("start_date", ""),
- end_date=config.get("end_date", ""),
- interval=interval,
- explicit=bench_ticker,
- # Explicit source: fetch the benchmark through its own loader
- # (keeps e.g. source=local offline). Auto keeps the yfinance
- # default — its loader only wraps the preloaded strategy data.
- loader=loader if bench_source != "auto" else None,
+ from backtest.benchmark import default_benchmark, resolve_benchmark
+
+ basket_codes = list(ret_df.columns)
+ basket_label = (
+ f"equal-weight basket ({basket_codes[0]})" if len(basket_codes) == 1
+ else f"equal-weight basket of {len(basket_codes)} symbols"
+ )
+ benchmark_metadata = {"benchmark_kind": "equal_weight", "benchmark_label": basket_label}
+ requested = config.get("benchmark")
+ source = config.get("source", "yfinance")
+ ticker = default_benchmark(basket_codes, source) if requested == "auto" else requested
+ reason = "no default benchmark for the evaluated symbols" if requested == "auto" else "no benchmark requested"
+ if ticker:
+ result = resolve_benchmark(
+ strategy_codes=basket_codes, source=source, explicit=ticker,
+ start_date=config.get("start_date", ""), end_date=config.get("end_date", ""),
+ interval=interval, loader=loader if source != "auto" else None,
)
- if bench_result is not None:
- bench_ret = bench_result.ret_series.reindex(dates).fillna(0.0)
- # The benchmark is fetched over the requested start_date..
- # end_date, but a declared warm-up boundary makes the EVALUATED
- # window shorter than that (`dates` is already clipped above).
- # Grading total_return over the short window against a
- # benchmark measured over the long one is the mismatched-window
- # error the warm-up boundary exists to prevent, so re-measure
- # the benchmark over the same bars. Falls back to the fetched
- # total only when the overlap is too short to measure.
- window_ret = bench_result.total_return_over(dates)
+ window_return = result.total_return_over(dates) if result is not None else None
+ if result is not None and window_return is not None:
+ bench_ret = bar_returns(result.close.reindex(dates), label=f"benchmark {result.ticker}")
benchmark_metadata = {
- "benchmark_ticker": bench_result.ticker,
- "benchmark_return": (
- window_ret if window_ret is not None else bench_result.total_ret
- ),
+ "benchmark_kind": "external", "benchmark_label": result.ticker,
+ "benchmark_ticker": result.ticker, "benchmark_return": window_return,
}
else:
- # The equal-weight basket stands in for the declared benchmark,
- # so every benchmark-relative metric (excess_return,
- # information_ratio, tracking_error, benchmark_beta) is graded
- # against something the caller did not ask for. Say so.
- benchmark_warning = (
- f"declared benchmark {bench_ticker!r} could not be resolved; "
- "benchmark-relative metrics are graded against the "
- "equal-weight basket of the traded symbols instead"
- )
- # ── External benchmark fetch ──────────────────────────────────────────
+ reason = f"benchmark {ticker!r} could not be resolved over the evaluation window"
+ benchmark_warning = (
+ f"{reason}; benchmark-relative metrics use the {basket_label}"
+ if benchmark_metadata["benchmark_kind"] == "equal_weight" else None
+ )
bench_equity = self.initial_capital * (1 + bench_ret).cumprod()
diff --git a/agent/src/api/attribution_core.py b/agent/src/api/attribution_core.py
index e37ff9380..c32909c99 100644
--- a/agent/src/api/attribution_core.py
+++ b/agent/src/api/attribution_core.py
@@ -823,17 +823,18 @@ def _build_attribution_payload(run_dir: Path) -> Dict[str, Any]:
config_benchmark = config.get("benchmark")
if isinstance(config_benchmark, dict):
config_benchmark = config_benchmark.get("ticker") or config_benchmark.get("code")
- if config_benchmark is not None and str(config_benchmark).strip():
- benchmark_ticker = str(config_benchmark).strip()
- if benchmark_ticker is None:
- _metrics_fields, metrics_rows = _attribution_read_csv(run_dir / "artifacts" / "metrics.csv")
- if metrics_rows:
- candidate = metrics_rows[0].get("benchmark_ticker")
- if candidate is not None and str(candidate).strip():
- benchmark_ticker = str(candidate).strip()
+ _, metrics_rows = _attribution_read_csv(run_dir / "artifacts" / "metrics.csv")
+ recorded = metrics_rows[0] if metrics_rows else {}
+ # New runs record the reference actually used, including a fallback after
+ # an explicit fetch failed. Older runs only had a ticker or the request.
+ if recorded.get("benchmark_kind") != "equal_weight":
+ candidate = recorded.get("benchmark_ticker") or config_benchmark
+ if candidate and str(candidate).strip() not in {"", "auto"}:
+ benchmark_ticker = str(candidate).strip()
if benchmark_ticker is not None:
- payload["benchmark"] = {"ticker": benchmark_ticker, "mode": "explicit"}
+ mode = "auto_market" if config_benchmark == "auto" else "explicit"
+ payload["benchmark"] = {"ticker": benchmark_ticker, "mode": mode}
else:
payload["benchmark"] = {"ticker": None, "mode": "auto_equal_weight"}
diff --git a/agent/src/skills/backtest-diagnose/SKILL.md b/agent/src/skills/backtest-diagnose/SKILL.md
index bdc5269e2..35a75c868 100644
--- a/agent/src/skills/backtest-diagnose/SKILL.md
+++ b/agent/src/skills/backtest-diagnose/SKILL.md
@@ -35,6 +35,7 @@ Use this skill when a user reports that a backtest failed, raised an error, or p
2. **Late trades** (first trade occurs more than 2 years after the backtest start): data-filtering bug. The lookback window may be too long, or the initial data segment may have been dropped. Shorten the window or check whether `dropna` is too aggressive.
3. **Capital utilization < 50%** (mostly in cash): position-sizing bug. Signal triggers may be too sparse, or the position-sizing logic may be wrong.
4. **Open position at the end** (a position still exists when the backtest ends): exit-timing bug. Forced liquidation may be missing, or exit logic does not cover the final segment.
+5. **Unexpected excess return**: check `benchmark_label` in the run metrics and the warnings in `run_card.json`. An omitted benchmark uses the evaluated symbols’ equal-weight returns. Set `"benchmark": "auto"` to request a market reference, or supply an explicit ticker. If no common default applies or the reference lacks usable data, the run records the equal-weight fallback.
### Data Errors
diff --git a/agent/tests/test_attribution_api.py b/agent/tests/test_attribution_api.py
index 322371707..ea965951e 100644
--- a/agent/tests/test_attribution_api.py
+++ b/agent/tests/test_attribution_api.py
@@ -190,6 +190,25 @@ def test_attribution_auto_mode_factor_and_symbol_brinson(tmp_path: Path, monkeyp
assert any("symbol-mode" in note for note in data["notes"])
+def test_attribution_uses_executed_reference_after_explicit_fetch_failed(tmp_path, monkeypatch):
+ client = _client(tmp_path, monkeypatch)
+ run_dir = _build_auto_run(tmp_path, "fallback")
+ _write_config(run_dir, [SYMBOL_A, SYMBOL_B], benchmark="SPY")
+ _write_metrics(run_dir, extra={"benchmark_kind": "equal_weight"})
+ data = client.get("/runs/fallback/attribution").json()
+ assert data["benchmark"] == {"ticker": None, "mode": "auto_equal_weight"}
+ assert data["brinson"]["mode"] == "symbol"
+
+
+def test_attribution_names_the_resolved_auto_ticker(tmp_path, monkeypatch):
+ client = _client(tmp_path, monkeypatch)
+ run_dir = _build_explicit_run(tmp_path, "market")
+ _write_config(run_dir, ["600519.SH"], benchmark="auto")
+ _write_metrics(run_dir, extra={"benchmark_kind": "external", "benchmark_ticker": "000300.SH"})
+ data = client.get("/runs/market/attribution").json()
+ assert data["benchmark"] == {"ticker": "000300.SH", "mode": "auto_market"}
+
+
def test_attribution_explicit_mode_asset_class_with_cash(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
client = _client(tmp_path, monkeypatch)
_build_explicit_run(tmp_path, "attr-explicit")
diff --git a/agent/tests/test_benchmark_excess_return_consistency.py b/agent/tests/test_benchmark_excess_return_consistency.py
index 864dadee0..60fdad5f8 100644
--- a/agent/tests/test_benchmark_excess_return_consistency.py
+++ b/agent/tests/test_benchmark_excess_return_consistency.py
@@ -97,8 +97,7 @@ def test_excess_return_matches_corrected_benchmark_return(monkeypatch, tmp_path)
def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):
- """No config["benchmark"] set: benchmark_metadata stays empty, so the fix
- must not touch excess_return at all on the ordinary, unconfigured path."""
+ """Only the loaded symbol contributes to the unconfigured reference."""
dates = pd.bdate_range("2026-01-05", periods=3)
bars = pd.DataFrame(
{
@@ -111,7 +110,7 @@ def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):
index=dates,
)
config = {
- "codes": ["000001.SZ"],
+ "codes": ["000002.SZ", "000001.SZ"],
"start_date": str(dates[0].date()),
"end_date": str(dates[-1].date()),
"source": "tushare",
@@ -121,5 +120,7 @@ def test_excess_return_unaffected_when_no_external_benchmark(tmp_path):
engine = ChinaAEngine({"initial_cash": 1_000_000.0})
metrics = engine.run_backtest(config, _FakeLoader(bars), _FlatSignal(), tmp_path)
- assert "benchmark_return" not in metrics or metrics.get("benchmark_ticker") is None
+ assert metrics["benchmark_kind"] == "equal_weight"
+ assert "benchmark_ticker" not in metrics
+ assert metrics["benchmark_label"] == "equal-weight basket (000001.SZ)"
assert metrics["excess_return"] == pytest.approx(0.0)
diff --git a/agent/tests/test_benchmark_exchange_suffix_regressions.py b/agent/tests/test_benchmark_exchange_suffix_regressions.py
index 7bf809b80..ecb2ef6d1 100644
--- a/agent/tests/test_benchmark_exchange_suffix_regressions.py
+++ b/agent/tests/test_benchmark_exchange_suffix_regressions.py
@@ -27,3 +27,15 @@ def test_explicit_ashare_suffix_selects_csi300_benchmark(symbol, source):
assert result is not None
assert result.ticker == "000300.SH"
assert result.total_ret == pytest.approx(0.1)
+
+
[email protected](("codes", "ticker"), [
+ (["VOD.L"], "^FTSE"), (["GGAL.BA"], "^MERV"), (["005930.KS"], "^KS11"),
+ (["7203.T"], None), (["EURUSD=X"], None), (["VIC.VN"], None),
+ (["IF2406.CFFEX"], None), (["AAPL.US", "0700.HK"], None),
+ (["local:VOD.L", "SHEL.L"], "^FTSE"),
+])
+def test_auto_requires_a_shared_identified_market(codes, ticker):
+ from backtest.benchmark import default_benchmark
+
+ assert default_benchmark(codes, "local") == ticker
diff --git a/agent/tests/test_offline_benchmark.py b/agent/tests/test_offline_benchmark.py
index 814cd9a22..399f19298 100644
--- a/agent/tests/test_offline_benchmark.py
+++ b/agent/tests/test_offline_benchmark.py
@@ -151,6 +151,18 @@ def test_networked_run_keeps_the_snapshot_wrapper_away_from_its_loader(
assert "benchmark_ticker" not in card["metrics"]
+def test_nonoverlapping_external_prices_cannot_grade_the_run(tmp_path, monkeypatch):
+ unrelated = _bars(_BENCH_CLOSE)
+ unrelated.index += pd.Timedelta(days=30)
+ card, _ = _run_main(
+ tmp_path, monkeypatch, source="local",
+ frames={"AAPL.US": _bars(_STOCK_CLOSE), _BENCH: unrelated},
+ )
+ assert card["metrics"]["benchmark_kind"] == "equal_weight"
+ assert card["metrics"]["benchmark_return"] == pytest.approx(0.4)
+ assert any(_UNRESOLVED_MARKER in w for w in card["warnings"])
+
+
def test_snapshot_wrapper_without_inner_loader_serves_only_the_snapshot() -> None:
frame = _bars(_STOCK_CLOSE)
loader = runner._AutoLoader({"AAPL.US": frame})
@@ -211,15 +223,16 @@ def test_source_bound_benchmark_never_switches_provider(monkeypatch, source, loa
assert network_calls == []
[email protected]("requested,ticker", [(_BENCH, _BENCH), ("auto", "SPY")])
@pytest.mark.parametrize("benchmark_available", [True, False])
-def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, benchmark_available):
+def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, benchmark_available, requested, ticker):
"""Exercise CSV -> actual loader -> runner -> run card with no network."""
from backtest.loaders import local_loader
bridge = tmp_path / "bridge.json"
symbols = [("AAPL.US", _STOCK_CLOSE)]
if benchmark_available:
- symbols.append((_BENCH, _BENCH_CLOSE))
+ symbols.append((ticker, _BENCH_CLOSE))
sources = []
for symbol, closes in symbols:
path = tmp_path / f"{symbol}.csv"
@@ -237,7 +250,7 @@ def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, ben
" return {code: frame['close'] * 0 for code, frame in data_map.items()}\n"
)
(run / "config.json").write_text(json.dumps({
- "codes": ["AAPL.US"], "source": "local", "benchmark": _BENCH,
+ "codes": ["AAPL.US"], "source": "local", "benchmark": requested,
"start_date": "2026-01-05", "end_date": "2026-01-09", "initial_cash": 100_000,
}))
network_calls = []
@@ -252,10 +265,12 @@ def test_real_local_csv_benchmark_reaches_run_metrics(tmp_path, monkeypatch, ben
card = json.loads((run / "run_card.json").read_text())
assert network_calls == []
if benchmark_available:
- assert card["metrics"]["benchmark_ticker"] == _BENCH
+ assert card["metrics"]["benchmark_ticker"] == ticker
assert card["metrics"]["benchmark_return"] == pytest.approx(0.02)
assert card["metrics"]["excess_return"] == pytest.approx(-0.02)
assert not any(_UNRESOLVED_MARKER in w for w in card.get("warnings") or [])
else:
+ assert card["metrics"]["benchmark_kind"] == "equal_weight"
+ assert card["metrics"]["benchmark_label"] == "equal-weight basket (AAPL.US)"
assert "benchmark_ticker" not in card["metrics"]
assert any(_UNRESOLVED_MARKER in w for w in card["warnings"])
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index 78748adac..9c839da20 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -1421,7 +1421,7 @@ export interface FactorReportPayload {
export interface AttributionBenchmarkInfo {
ticker: string | null;
- mode: "auto_equal_weight" | "explicit";
+ mode: "auto_equal_weight" | "auto_market" | "explicit";
}
export interface AttributionRollingPoint {