"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(backtest): separate the data window from the evaluation window
Haozhe Wu16c9d28883ce
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.pyindex 2c36ecdd1..7e7e93b36 100644--- a/agent/backtest/engines/base.py+++ b/agent/backtest/engines/base.py@@ -146,6 +146,76 @@ def _ffill_2d(arr: np.ndarray, limit: int = 5) -> np.ndarray:# ─── Signal alignment (reused from daily_portfolio logic) ───+def evaluation_start_index(config: Dict[str, Any], dates: pd.DatetimeIndex) -> int:+ """Return the first bar index that counts as performance.++ A long-lookback indicator needs history before the period the user asked+ about — an MA200 strategy over ten years needs bars from before those ten+ years. Loading that history is correct; *grading* it is not. Without a+ declared boundary the extra bars silently join the evaluation: trades fire+ in them, the equity curve starts in them, and CAGR and the benchmark are+ computed over a window longer than the one that was requested. The run+ still succeeds and its metrics are still internally consistent, which is+ what makes it dangerous.++ Two spellings, because a strategy author knows one or the other: the number+ of warm-up bars the indicator needs, or the date the evaluation should+ start. Declaring both is refused rather than resolved — the two can+ disagree, and picking a winner would silently discard the author's other+ instruction.++ Args:+ config: Backtest configuration. Reads ``warmup_bars`` (int >= 0) and+ ``evaluation_start_date`` (``YYYY-MM-DD``); absent means the whole+ loaded window is evaluated, which is the historical behaviour.+ dates: The aligned bar index the backtest would otherwise run over.++ Returns:+ Index into ``dates`` of the first evaluated bar; ``0`` when nothing is+ declared.++ Raises:+ ValueError: Both keys declared, a malformed value, or a boundary that+ leaves fewer than two bars to evaluate.+ """+ warmup = config.get("warmup_bars")+ eval_start = config.get("evaluation_start_date")+ if warmup in (None, 0) and not eval_start:+ return 0+ if warmup not in (None, 0) and eval_start:+ raise ValueError(+ "declare warmup_bars or evaluation_start_date, not both — they can "+ "disagree, and resolving that silently would discard one of them"+ )++ if eval_start:+ boundary = pd.Timestamp(eval_start)+ if boundary.tz is None and dates.tz is not None:+ boundary = boundary.tz_localize(dates.tz)+ elif boundary.tz is not None and dates.tz is None:+ boundary = boundary.tz_localize(None)+ start = int(np.searchsorted(dates.values, boundary.to_datetime64(), side="left"))+ if start >= len(dates):+ raise ValueError(+ f"evaluation_start_date {eval_start} is after the last loaded bar "+ f"({dates[-1].date()}); nothing would be evaluated"+ )+ else:+ try:+ start = int(warmup)+ except (TypeError, ValueError):+ raise ValueError(f"warmup_bars must be an integer, got {warmup!r}") from None+ if start < 0:+ raise ValueError(f"warmup_bars must be non-negative, got {start}")++ if len(dates) - start < 2:+ raise ValueError(+ f"the evaluation window would hold {max(0, len(dates) - start)} bar(s) of "+ f"{len(dates)} loaded; widen start_date or shorten the warm-up"+ )+ return start++def _align(data_map: Dict[str, pd.DataFrame],signal_map: Dict[str, pd.Series],@@ -736,6 +806,18 @@ class BaseEngine(ABC):# Sync codes after _align may have dropped all-NaN symbolsvalid_codes = [c for c in valid_codes if c in target_pos.columns]+ # 3b. Drop the warm-up prefix from the evaluation window. Signals were+ # generated over the whole loaded panel above, so the first evaluated+ # bar already carries an indicator that saw its full lookback; from here+ # on the warm-up bars simply do not exist, so no trade, no equity point,+ # no benchmark return and no metric can come from them.+ warmup_end = evaluation_start_index(config, dates)+ if warmup_end:+ dates = dates[warmup_end:]+ close_df = close_df.iloc[warmup_end:]+ target_pos = target_pos.iloc[warmup_end:]+ ret_df = ret_df.iloc[warmup_end:]+# 4. Bar-by-bar executionself._execute_bars(dates, data_map, close_df, target_pos, valid_codes)actual_pos = self._actual_positions_frame(valid_codes)diff --git a/agent/backtest/engines/options_portfolio.py b/agent/backtest/engines/options_portfolio.pyindex 8404db6c9..4f7d2a03d 100644--- a/agent/backtest/engines/options_portfolio.py+++ b/agent/backtest/engines/options_portfolio.py@@ -26,6 +26,7 @@ from typing import Any, Dict, List, Optionalimport numpy as npimport pandas as pd+from backtest.engines.base import evaluation_start_indexfrom src.quantlib.options import bs_greeks, bs_price, normalise_option_type@@ -232,6 +233,12 @@ def run_options_backtest(all_dates.update(df.index)dates = sorted(all_dates)+ # Warm-up bars primed the signal engine above; from here they do not exist,+ # so nothing they contain reaches a fill, the equity curve or a metric.+ warmup_end = evaluation_start_index(config, pd.DatetimeIndex(dates))+ if warmup_end:+ dates = dates[warmup_end:]+# Index signals by datesignal_by_date: Dict[str, List[Dict[str, Any]]] = {}for sig in signals:diff --git a/agent/backtest/runner.py b/agent/backtest/runner.pyindex f3d521aaa..a7bd38ec9 100644--- a/agent/backtest/runner.py+++ b/agent/backtest/runner.py@@ -85,6 +85,15 @@ class BacktestConfigSchema(BaseModel):initial_cash: float = Field(default=1_000_000, gt=0, allow_inf_nan=False)fundamental_fields: Optional[Dict[str, List[str]]] = Noneevent_feeds: Optional[List[Dict[str, Any]]] = None+ # An indicator with a long lookback needs bars from before the period the+ # user asked about. Declaring the boundary keeps those bars out of the+ # performance: either as a bar count, or as the date evaluation starts.+ # Only the shapes are checked here -- whether the boundary leaves anything+ # to evaluate depends on the loaded calendar, so the one rule that decides+ # it lives in `engines.base.evaluation_start_index`, which every engine and+ # every direct-API caller passes through.+ warmup_bars: Optional[int] = Field(default=None, ge=0)+ evaluation_start_date: Optional[str] = None@field_validator("codes")@classmethod@@ -104,6 +113,19 @@ class BacktestConfigSchema(BaseModel):raise ValueError(f"invalid date format: {v!r} (expected YYYY-MM-DD)")return v+ @field_validator("evaluation_start_date")+ @classmethod+ def valid_evaluation_start(cls, v: Optional[str]) -> Optional[str]:+ if v is None:+ return v+ try:+ pd.Timestamp(v)+ except Exception:+ raise ValueError(+ f"invalid evaluation_start_date: {v!r} (expected YYYY-MM-DD)"+ ) from None+ return v+@field_validator("interval")@classmethoddef valid_interval(cls, v: str) -> str:diff --git a/agent/src/skills/strategy-generate/SKILL.md b/agent/src/skills/strategy-generate/SKILL.mdindex 2a7c2344a..7dfb57ed1 100644--- a/agent/src/skills/strategy-generate/SKILL.md+++ b/agent/src/skills/strategy-generate/SKILL.md@@ -21,6 +21,7 @@ category: strategyExtract the following from the user's description:- **Instrument codes**: process them according to the normalization rules below- **Time range**: if the user does not specify dates, default to **10 years back from today** (for example, if today is `2026-03-18`, then `start_date=2016-03-18`, `end_date=2026-03-18`)+- **Indicator warm-up**: a long lookback (MA200, a 252-day z-score) needs bars from *before* the requested period. Move `start_date` back to load them **and declare the boundary with `warmup_bars`** — the requested period is what gets graded, and undeclared warm-up bars are graded too. Silently backdating `start_date` by a year turns a 10-year backtest into an 11-year one that still calls itself 10 years: the extra year's trades, CAGR and benchmark all enter the report, the run succeeds, and the numbers look internally consistent- **Strategy logic**: entry / exit conditions and indicator parameters**If critical information is missing, you must ask the user instead of guessing:**@@ -118,6 +119,7 @@ Self-check after writing `signal_engine.py`:"codes": ["000001.SZ"],"start_date": "2016-03-18","end_date": "2026-03-18",+ "warmup_bars": 0,"interval": "1D","initial_cash": 1000000,"commission": 0.001,@@ -138,6 +140,10 @@ Self-check after writing `signal_engine.py`:- `interval`: candlestick interval, default `"1D"`. Supported values: `"1m"` / `"5m"` / `"15m"` / `"30m"` / `"1H"` / `"4H"` / `"1D"`- The annualization factor for minute backtests is inferred automatically from `source` (252 trading days for China A-shares, 365 calendar days for crypto)- Minute backtests can be very data-heavy. Recommended limits are no more than 30 days for `1m`, or 1 year for `1H`+- `warmup_bars`: how many leading bars exist only to prime the indicators. They are loaded and fed to `SignalEngine.generate()`, then excluded from trades, the equity curve, the benchmark and every metric. Default `0` grades the whole loaded window.+ - Use it whenever you widen `start_date` for an indicator's lookback. `start_date` is the **data** window; `start_date` plus `warmup_bars` is the **evaluation** window, and the report describes the second one.+ - Size it from the longest lookback in the strategy, plus a margin: MA200 needs at least 200 daily bars, a 252-day rolling z-score needs 252. Then set `start_date` far enough back to supply them.+ - `evaluation_start_date` (`"YYYY-MM-DD"`) is the same instruction stated as a date, for when the user names the period rather than the lookback. Declare one or the other — declaring both is rejected.- `extra_fields`: China A-shares can use values such as `["pe", "pb", "roe"]`; other markets should use `null`- `fundamental_fields`: optional China A-share statement fields, such as `{"income": ["total_revenue", "n_income"], "fina_indicator": ["roe"]}`; use `null` unless the strategy needs financial statement pre-filtering- `optimizer`: optional, one of `"equal_volatility"` / `"risk_parity"` / `"mean_variance"` / `"max_diversification"` / `"turnover_aware"` / `null` (equal-weight by default)diff --git a/agent/tests/test_warmup_window.py b/agent/tests/test_warmup_window.pynew file mode 100644index 000000000..6f3d88716--- /dev/null+++ b/agent/tests/test_warmup_window.py@@ -0,0 +1,241 @@+"""Warm-up history must feed the indicators without joining the performance.++Issue #1240: a strategy with a long lookback (MA200) needs bars from before the+period the user asked about. The agent loaded them by moving ``start_date``+earlier — and the backtest then graded them. A run billed as ten years reported+eleven: trades fired in the extra year, and CAGR and the benchmark were computed+over the longer window. Nothing raised, and the metrics were internally+consistent, so the wrong evaluation period was invisible in the output.+"""++from __future__ import annotations++import numpy as np+import pandas as pd+import pytest++from backtest.engines.base import BaseEngine, evaluation_start_index+++class _Engine(BaseEngine):+ """Frictionless engine: only the evaluation window is under test."""++ def can_execute(self, symbol, direction, bar):+ return True++ def round_size(self, raw_size, price):+ return float(raw_size)++ def calc_commission(self, size, price, direction, is_open):+ return 0.0++ def apply_slippage(self, price, direction):+ return price+++class _Loader:+ def __init__(self, data_map):+ self._data_map = data_map++ def fetch(self, codes, start_date, end_date, fields=None, interval="1D"):+ return self._data_map+++class _MovingAverageCross:+ """Long while price is above its own moving average — needs `window` bars."""++ def __init__(self, window: int):+ self.window = window++ def generate(self, data):+ out = {}+ for code, frame in data.items():+ ma = frame["close"].rolling(self.window).mean()+ out[code] = (frame["close"] > ma).astype(float)+ return out+++def _prices(n: int, start: str = "2015-01-01") -> pd.DataFrame:+ """A rising series with a dip, so the MA rule both enters and exits."""+ idx = pd.bdate_range(start, periods=n)+ base = np.linspace(100.0, 200.0, n)+ wobble = 8.0 * np.sin(np.arange(n) / 9.0)+ return pd.DataFrame({"open": base + wobble, "close": base + wobble}, index=idx)+++class TestEvaluationStartIndex:+ dates = pd.bdate_range("2015-01-01", periods=100)++ def test_absent_declaration_evaluates_everything(self):+ assert evaluation_start_index({}, self.dates) == 0+ assert evaluation_start_index({"warmup_bars": 0}, self.dates) == 0++ def test_warmup_bars(self):+ assert evaluation_start_index({"warmup_bars": 30}, self.dates) == 30++ def test_evaluation_start_date_lands_on_the_first_bar_at_or_after_it(self):+ boundary = self.dates[40]+ cfg = {"evaluation_start_date": boundary.strftime("%Y-%m-%d")}+ assert evaluation_start_index(cfg, self.dates) == 40+ # A weekend date resolves forward to the next trading bar.+ assert self.dates[evaluation_start_index(+ {"evaluation_start_date": "2015-03-14"}, self.dates+ )] >= pd.Timestamp("2015-03-14")++ def test_declaring_both_is_refused(self):+ with pytest.raises(ValueError, match="not both"):+ evaluation_start_index(+ {"warmup_bars": 10, "evaluation_start_date": "2015-03-01"}, self.dates+ )++ def test_a_boundary_that_leaves_nothing_is_refused(self):+ with pytest.raises(ValueError, match="evaluation window"):+ evaluation_start_index({"warmup_bars": 99}, self.dates)+ with pytest.raises(ValueError, match="after the last loaded bar"):+ evaluation_start_index({"evaluation_start_date": "2030-01-01"}, self.dates)++ def test_malformed_warmup_is_refused(self):+ with pytest.raises(ValueError, match="non-negative"):+ evaluation_start_index({"warmup_bars": -5}, self.dates)+ with pytest.raises(ValueError, match="must be an integer"):+ evaluation_start_index({"warmup_bars": "two hundred"}, self.dates)+++class TestWarmupIsExcludedFromPerformance:+ frame = _prices(600)+ boundary = 250++ def _run(self, tmp_path, config_extra):+ data_map = {"AAA": self.frame}+ engine = _Engine({"initial_cash": 100_000.0})+ config = {+ "codes": ["AAA"],+ "start_date": str(self.frame.index[0].date()),+ "end_date": str(self.frame.index[-1].date()),+ **config_extra,+ }+ metrics = engine.run_backtest(+ config, _Loader(data_map), _MovingAverageCross(60), tmp_path+ )+ return engine, metrics++ def test_no_trade_and_no_equity_point_precedes_the_boundary(self, tmp_path):+ engine, _ = self._run(tmp_path, {"warmup_bars": self.boundary})+ first_bar = self.frame.index[self.boundary]++ assert engine.equity_snapshots, "the evaluated window produced no equity curve"+ assert min(s.timestamp for s in engine.equity_snapshots) >= first_bar+ assert engine.trades, "fixture must trade, or it proves nothing"+ assert min(t.entry_time for t in engine.trades) >= first_bar++ def test_the_warmup_year_changes_the_reported_performance(self, tmp_path):+ """The bug: identical loaded data, and the metrics silently differed."""+ _, graded_everything = self._run(tmp_path / "a", {})+ _, graded_after_warmup = self._run(+ tmp_path / "b", {"warmup_bars": self.boundary}+ )++ assert graded_everything["total_return"] != graded_after_warmup["total_return"]+ assert graded_everything["annual_return"] != graded_after_warmup["annual_return"]++ def test_the_two_spellings_agree(self, tmp_path):+ _, by_bars = self._run(tmp_path / "a", {"warmup_bars": self.boundary})+ _, by_date = self._run(+ tmp_path / "b",+ {"evaluation_start_date": str(self.frame.index[self.boundary].date())},+ )+ assert by_bars["total_return"] == pytest.approx(by_date["total_return"])++ def test_warmup_history_still_reaches_the_indicator(self, tmp_path):+ """The point of loading the extra bars: the strategy starts warmed up.++ Same evaluation window, twice. Given warm-up history the moving average+ is already defined on the first evaluated bar, so the book can hold a+ position immediately; loading only the evaluation window leaves the+ indicator NaN until its lookback fills, and the first trade is late.+ """+ warm_engine, _ = self._run(tmp_path / "warm", {"warmup_bars": self.boundary})++ cold_map = {"AAA": self.frame.iloc[self.boundary:]}+ cold_engine = _Engine({"initial_cash": 100_000.0})+ cold_engine.run_backtest(+ {+ "codes": ["AAA"],+ "start_date": str(self.frame.index[self.boundary].date()),+ "end_date": str(self.frame.index[-1].date()),+ },+ _Loader(cold_map),+ _MovingAverageCross(60),+ tmp_path / "cold",+ )++ assert warm_engine.trades and cold_engine.trades+ assert min(t.entry_time for t in warm_engine.trades) < min(+ t.entry_time for t in cold_engine.trades+ )+++class TestOptionsEngineHonoursTheBoundary:+ """The options pipeline builds its own date sequence and needs the same cut."""++ dates = pd.bdate_range("2025-01-01", periods=8)+ bars = pd.DataFrame(+ {+ "open": np.linspace(100.0, 107.0, 8),+ "high": np.linspace(101.0, 108.0, 8),+ "low": np.linspace(99.0, 106.0, 8),+ "close": np.linspace(100.5, 107.5, 8),+ "volume": [1000] * 8,+ },+ index=dates,+ )++ class _OptionsLoader:+ name = "yfinance"++ def __init__(self, bars):+ self._bars = bars++ def fetch(self, codes, start_date, end_date):+ return {"SPY": self._bars.copy()}++ class _OpenOnDay:+ """Opens on day 1 — inside the warm-up — and never closes."""++ def generate(self, data_map):+ return [+ {+ "date": "2025-01-01",+ "action": "open",+ "underlying": "SPY",+ "legs": [+ {"type": "call", "strike": 101.0, "expiry": "2025-03-21", "qty": 5}+ ],+ }+ ]++ def _run(self, tmp_path, extra):+ from backtest.engines.options_portfolio import run_options_backtest++ run_options_backtest(+ {+ "codes": ["SPY"],+ "start_date": "2025-01-01",+ "end_date": "2025-01-10",+ "source": "yfinance",+ "engine": "options",+ "initial_cash": 100_000,+ **extra,+ },+ self._OptionsLoader(self.bars),+ self._OpenOnDay(),+ tmp_path,+ )+ return tmp_path / "artifacts"++ def test_a_signal_inside_the_warmup_never_fills(self, tmp_path):+ graded_everything = pd.read_csv(self._run(tmp_path / "a", {}) / "trades.csv")+ assert not graded_everything.empty, "fixture must trade, or it proves nothing"++ after_warmup = self._run(tmp_path / "b", {"warmup_bars": 3}) / "trades.csv"+ assert pd.read_csv(after_warmup).empty