Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(backtest): separate the data window from the evaluation window

Haozhe Wu16c9d28883ce
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.py
index 2c36ecdd1..7e7e93b36 100644
--- a/agent/backtest/engines/base.py
+++ b/agent/backtest/engines/base.py
@@ -146,6 +146,76 @@ def _ffill_2d(arr: np.ndarray, limit: int = 5) -> np.ndarray:
# ─── Signal alignment (reused from daily_portfolio logic) ───
+def evaluation_start_index(config: Dict[str, Any], dates: pd.DatetimeIndex) -> int:
+ """Return the first bar index that counts as performance.
+
+ A long-lookback indicator needs history before the period the user asked
+ about — an MA200 strategy over ten years needs bars from before those ten
+ years. Loading that history is correct; *grading* it is not. Without a
+ declared boundary the extra bars silently join the evaluation: trades fire
+ in them, the equity curve starts in them, and CAGR and the benchmark are
+ computed over a window longer than the one that was requested. The run
+ still succeeds and its metrics are still internally consistent, which is
+ what makes it dangerous.
+
+ Two spellings, because a strategy author knows one or the other: the number
+ of warm-up bars the indicator needs, or the date the evaluation should
+ start. Declaring both is refused rather than resolved — the two can
+ disagree, and picking a winner would silently discard the author's other
+ instruction.
+
+ Args:
+ config: Backtest configuration. Reads ``warmup_bars`` (int >= 0) and
+ ``evaluation_start_date`` (``YYYY-MM-DD``); absent means the whole
+ loaded window is evaluated, which is the historical behaviour.
+ dates: The aligned bar index the backtest would otherwise run over.
+
+ Returns:
+ Index into ``dates`` of the first evaluated bar; ``0`` when nothing is
+ declared.
+
+ Raises:
+ ValueError: Both keys declared, a malformed value, or a boundary that
+ leaves fewer than two bars to evaluate.
+ """
+ warmup = config.get("warmup_bars")
+ eval_start = config.get("evaluation_start_date")
+ if warmup in (None, 0) and not eval_start:
+ return 0
+ if warmup not in (None, 0) and eval_start:
+ raise ValueError(
+ "declare warmup_bars or evaluation_start_date, not both — they can "
+ "disagree, and resolving that silently would discard one of them"
+ )
+
+ if eval_start:
+ boundary = pd.Timestamp(eval_start)
+ if boundary.tz is None and dates.tz is not None:
+ boundary = boundary.tz_localize(dates.tz)
+ elif boundary.tz is not None and dates.tz is None:
+ boundary = boundary.tz_localize(None)
+ start = int(np.searchsorted(dates.values, boundary.to_datetime64(), side="left"))
+ if start >= len(dates):
+ raise ValueError(
+ f"evaluation_start_date {eval_start} is after the last loaded bar "
+ f"({dates[-1].date()}); nothing would be evaluated"
+ )
+ else:
+ try:
+ start = int(warmup)
+ except (TypeError, ValueError):
+ raise ValueError(f"warmup_bars must be an integer, got {warmup!r}") from None
+ if start < 0:
+ raise ValueError(f"warmup_bars must be non-negative, got {start}")
+
+ if len(dates) - start < 2:
+ raise ValueError(
+ f"the evaluation window would hold {max(0, len(dates) - start)} bar(s) of "
+ f"{len(dates)} loaded; widen start_date or shorten the warm-up"
+ )
+ return start
+
+
def _align(
data_map: Dict[str, pd.DataFrame],
signal_map: Dict[str, pd.Series],
@@ -736,6 +806,18 @@ class BaseEngine(ABC):
# Sync codes after _align may have dropped all-NaN symbols
valid_codes = [c for c in valid_codes if c in target_pos.columns]
+ # 3b. Drop the warm-up prefix from the evaluation window. Signals were
+ # generated over the whole loaded panel above, so the first evaluated
+ # bar already carries an indicator that saw its full lookback; from here
+ # on the warm-up bars simply do not exist, so no trade, no equity point,
+ # no benchmark return and no metric can come from them.
+ warmup_end = evaluation_start_index(config, dates)
+ if warmup_end:
+ dates = dates[warmup_end:]
+ close_df = close_df.iloc[warmup_end:]
+ target_pos = target_pos.iloc[warmup_end:]
+ ret_df = ret_df.iloc[warmup_end:]
+
# 4. Bar-by-bar execution
self._execute_bars(dates, data_map, close_df, target_pos, valid_codes)
actual_pos = self._actual_positions_frame(valid_codes)
diff --git a/agent/backtest/engines/options_portfolio.py b/agent/backtest/engines/options_portfolio.py
index 8404db6c9..4f7d2a03d 100644
--- a/agent/backtest/engines/options_portfolio.py
+++ b/agent/backtest/engines/options_portfolio.py
@@ -26,6 +26,7 @@ from typing import Any, Dict, List, Optional
import numpy as np
import pandas as pd
+from backtest.engines.base import evaluation_start_index
from src.quantlib.options import bs_greeks, bs_price, normalise_option_type
@@ -232,6 +233,12 @@ def run_options_backtest(
all_dates.update(df.index)
dates = sorted(all_dates)
+ # Warm-up bars primed the signal engine above; from here they do not exist,
+ # so nothing they contain reaches a fill, the equity curve or a metric.
+ warmup_end = evaluation_start_index(config, pd.DatetimeIndex(dates))
+ if warmup_end:
+ dates = dates[warmup_end:]
+
# Index signals by date
signal_by_date: Dict[str, List[Dict[str, Any]]] = {}
for sig in signals:
diff --git a/agent/backtest/runner.py b/agent/backtest/runner.py
index f3d521aaa..a7bd38ec9 100644
--- a/agent/backtest/runner.py
+++ b/agent/backtest/runner.py
@@ -85,6 +85,15 @@ class BacktestConfigSchema(BaseModel):
initial_cash: float = Field(default=1_000_000, gt=0, allow_inf_nan=False)
fundamental_fields: Optional[Dict[str, List[str]]] = None
event_feeds: Optional[List[Dict[str, Any]]] = None
+ # An indicator with a long lookback needs bars from before the period the
+ # user asked about. Declaring the boundary keeps those bars out of the
+ # performance: either as a bar count, or as the date evaluation starts.
+ # Only the shapes are checked here -- whether the boundary leaves anything
+ # to evaluate depends on the loaded calendar, so the one rule that decides
+ # it lives in `engines.base.evaluation_start_index`, which every engine and
+ # every direct-API caller passes through.
+ warmup_bars: Optional[int] = Field(default=None, ge=0)
+ evaluation_start_date: Optional[str] = None
@field_validator("codes")
@classmethod
@@ -104,6 +113,19 @@ class BacktestConfigSchema(BaseModel):
raise ValueError(f"invalid date format: {v!r} (expected YYYY-MM-DD)")
return v
+ @field_validator("evaluation_start_date")
+ @classmethod
+ def valid_evaluation_start(cls, v: Optional[str]) -> Optional[str]:
+ if v is None:
+ return v
+ try:
+ pd.Timestamp(v)
+ except Exception:
+ raise ValueError(
+ f"invalid evaluation_start_date: {v!r} (expected YYYY-MM-DD)"
+ ) from None
+ return v
+
@field_validator("interval")
@classmethod
def valid_interval(cls, v: str) -> str:
diff --git a/agent/src/skills/strategy-generate/SKILL.md b/agent/src/skills/strategy-generate/SKILL.md
index 2a7c2344a..7dfb57ed1 100644
--- a/agent/src/skills/strategy-generate/SKILL.md
+++ b/agent/src/skills/strategy-generate/SKILL.md
@@ -21,6 +21,7 @@ category: strategy
Extract the following from the user's description:
- **Instrument codes**: process them according to the normalization rules below
- **Time range**: if the user does not specify dates, default to **10 years back from today** (for example, if today is `2026-03-18`, then `start_date=2016-03-18`, `end_date=2026-03-18`)
+- **Indicator warm-up**: a long lookback (MA200, a 252-day z-score) needs bars from *before* the requested period. Move `start_date` back to load them **and declare the boundary with `warmup_bars`** — the requested period is what gets graded, and undeclared warm-up bars are graded too. Silently backdating `start_date` by a year turns a 10-year backtest into an 11-year one that still calls itself 10 years: the extra year's trades, CAGR and benchmark all enter the report, the run succeeds, and the numbers look internally consistent
- **Strategy logic**: entry / exit conditions and indicator parameters
**If critical information is missing, you must ask the user instead of guessing:**
@@ -118,6 +119,7 @@ Self-check after writing `signal_engine.py`:
"codes": ["000001.SZ"],
"start_date": "2016-03-18",
"end_date": "2026-03-18",
+ "warmup_bars": 0,
"interval": "1D",
"initial_cash": 1000000,
"commission": 0.001,
@@ -138,6 +140,10 @@ Self-check after writing `signal_engine.py`:
- `interval`: candlestick interval, default `"1D"`. Supported values: `"1m"` / `"5m"` / `"15m"` / `"30m"` / `"1H"` / `"4H"` / `"1D"`
- The annualization factor for minute backtests is inferred automatically from `source` (252 trading days for China A-shares, 365 calendar days for crypto)
- Minute backtests can be very data-heavy. Recommended limits are no more than 30 days for `1m`, or 1 year for `1H`
+- `warmup_bars`: how many leading bars exist only to prime the indicators. They are loaded and fed to `SignalEngine.generate()`, then excluded from trades, the equity curve, the benchmark and every metric. Default `0` grades the whole loaded window.
+ - Use it whenever you widen `start_date` for an indicator's lookback. `start_date` is the **data** window; `start_date` plus `warmup_bars` is the **evaluation** window, and the report describes the second one.
+ - Size it from the longest lookback in the strategy, plus a margin: MA200 needs at least 200 daily bars, a 252-day rolling z-score needs 252. Then set `start_date` far enough back to supply them.
+ - `evaluation_start_date` (`"YYYY-MM-DD"`) is the same instruction stated as a date, for when the user names the period rather than the lookback. Declare one or the other — declaring both is rejected.
- `extra_fields`: China A-shares can use values such as `["pe", "pb", "roe"]`; other markets should use `null`
- `fundamental_fields`: optional China A-share statement fields, such as `{"income": ["total_revenue", "n_income"], "fina_indicator": ["roe"]}`; use `null` unless the strategy needs financial statement pre-filtering
- `optimizer`: optional, one of `"equal_volatility"` / `"risk_parity"` / `"mean_variance"` / `"max_diversification"` / `"turnover_aware"` / `null` (equal-weight by default)
diff --git a/agent/tests/test_warmup_window.py b/agent/tests/test_warmup_window.py
new file mode 100644
index 000000000..6f3d88716
--- /dev/null
+++ b/agent/tests/test_warmup_window.py
@@ -0,0 +1,241 @@
+"""Warm-up history must feed the indicators without joining the performance.
+
+Issue #1240: a strategy with a long lookback (MA200) needs bars from before the
+period the user asked about. The agent loaded them by moving ``start_date``
+earlier — and the backtest then graded them. A run billed as ten years reported
+eleven: trades fired in the extra year, and CAGR and the benchmark were computed
+over the longer window. Nothing raised, and the metrics were internally
+consistent, so the wrong evaluation period was invisible in the output.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from backtest.engines.base import BaseEngine, evaluation_start_index
+
+
+class _Engine(BaseEngine):
+ """Frictionless engine: only the evaluation window is under test."""
+
+ def can_execute(self, symbol, direction, bar):
+ return True
+
+ def round_size(self, raw_size, price):
+ return float(raw_size)
+
+ def calc_commission(self, size, price, direction, is_open):
+ return 0.0
+
+ def apply_slippage(self, price, direction):
+ return price
+
+
+class _Loader:
+ def __init__(self, data_map):
+ self._data_map = data_map
+
+ def fetch(self, codes, start_date, end_date, fields=None, interval="1D"):
+ return self._data_map
+
+
+class _MovingAverageCross:
+ """Long while price is above its own moving average — needs `window` bars."""
+
+ def __init__(self, window: int):
+ self.window = window
+
+ def generate(self, data):
+ out = {}
+ for code, frame in data.items():
+ ma = frame["close"].rolling(self.window).mean()
+ out[code] = (frame["close"] > ma).astype(float)
+ return out
+
+
+def _prices(n: int, start: str = "2015-01-01") -> pd.DataFrame:
+ """A rising series with a dip, so the MA rule both enters and exits."""
+ idx = pd.bdate_range(start, periods=n)
+ base = np.linspace(100.0, 200.0, n)
+ wobble = 8.0 * np.sin(np.arange(n) / 9.0)
+ return pd.DataFrame({"open": base + wobble, "close": base + wobble}, index=idx)
+
+
+class TestEvaluationStartIndex:
+ dates = pd.bdate_range("2015-01-01", periods=100)
+
+ def test_absent_declaration_evaluates_everything(self):
+ assert evaluation_start_index({}, self.dates) == 0
+ assert evaluation_start_index({"warmup_bars": 0}, self.dates) == 0
+
+ def test_warmup_bars(self):
+ assert evaluation_start_index({"warmup_bars": 30}, self.dates) == 30
+
+ def test_evaluation_start_date_lands_on_the_first_bar_at_or_after_it(self):
+ boundary = self.dates[40]
+ cfg = {"evaluation_start_date": boundary.strftime("%Y-%m-%d")}
+ assert evaluation_start_index(cfg, self.dates) == 40
+ # A weekend date resolves forward to the next trading bar.
+ assert self.dates[evaluation_start_index(
+ {"evaluation_start_date": "2015-03-14"}, self.dates
+ )] >= pd.Timestamp("2015-03-14")
+
+ def test_declaring_both_is_refused(self):
+ with pytest.raises(ValueError, match="not both"):
+ evaluation_start_index(
+ {"warmup_bars": 10, "evaluation_start_date": "2015-03-01"}, self.dates
+ )
+
+ def test_a_boundary_that_leaves_nothing_is_refused(self):
+ with pytest.raises(ValueError, match="evaluation window"):
+ evaluation_start_index({"warmup_bars": 99}, self.dates)
+ with pytest.raises(ValueError, match="after the last loaded bar"):
+ evaluation_start_index({"evaluation_start_date": "2030-01-01"}, self.dates)
+
+ def test_malformed_warmup_is_refused(self):
+ with pytest.raises(ValueError, match="non-negative"):
+ evaluation_start_index({"warmup_bars": -5}, self.dates)
+ with pytest.raises(ValueError, match="must be an integer"):
+ evaluation_start_index({"warmup_bars": "two hundred"}, self.dates)
+
+
+class TestWarmupIsExcludedFromPerformance:
+ frame = _prices(600)
+ boundary = 250
+
+ def _run(self, tmp_path, config_extra):
+ data_map = {"AAA": self.frame}
+ engine = _Engine({"initial_cash": 100_000.0})
+ config = {
+ "codes": ["AAA"],
+ "start_date": str(self.frame.index[0].date()),
+ "end_date": str(self.frame.index[-1].date()),
+ **config_extra,
+ }
+ metrics = engine.run_backtest(
+ config, _Loader(data_map), _MovingAverageCross(60), tmp_path
+ )
+ return engine, metrics
+
+ def test_no_trade_and_no_equity_point_precedes_the_boundary(self, tmp_path):
+ engine, _ = self._run(tmp_path, {"warmup_bars": self.boundary})
+ first_bar = self.frame.index[self.boundary]
+
+ assert engine.equity_snapshots, "the evaluated window produced no equity curve"
+ assert min(s.timestamp for s in engine.equity_snapshots) >= first_bar
+ assert engine.trades, "fixture must trade, or it proves nothing"
+ assert min(t.entry_time for t in engine.trades) >= first_bar
+
+ def test_the_warmup_year_changes_the_reported_performance(self, tmp_path):
+ """The bug: identical loaded data, and the metrics silently differed."""
+ _, graded_everything = self._run(tmp_path / "a", {})
+ _, graded_after_warmup = self._run(
+ tmp_path / "b", {"warmup_bars": self.boundary}
+ )
+
+ assert graded_everything["total_return"] != graded_after_warmup["total_return"]
+ assert graded_everything["annual_return"] != graded_after_warmup["annual_return"]
+
+ def test_the_two_spellings_agree(self, tmp_path):
+ _, by_bars = self._run(tmp_path / "a", {"warmup_bars": self.boundary})
+ _, by_date = self._run(
+ tmp_path / "b",
+ {"evaluation_start_date": str(self.frame.index[self.boundary].date())},
+ )
+ assert by_bars["total_return"] == pytest.approx(by_date["total_return"])
+
+ def test_warmup_history_still_reaches_the_indicator(self, tmp_path):
+ """The point of loading the extra bars: the strategy starts warmed up.
+
+ Same evaluation window, twice. Given warm-up history the moving average
+ is already defined on the first evaluated bar, so the book can hold a
+ position immediately; loading only the evaluation window leaves the
+ indicator NaN until its lookback fills, and the first trade is late.
+ """
+ warm_engine, _ = self._run(tmp_path / "warm", {"warmup_bars": self.boundary})
+
+ cold_map = {"AAA": self.frame.iloc[self.boundary:]}
+ cold_engine = _Engine({"initial_cash": 100_000.0})
+ cold_engine.run_backtest(
+ {
+ "codes": ["AAA"],
+ "start_date": str(self.frame.index[self.boundary].date()),
+ "end_date": str(self.frame.index[-1].date()),
+ },
+ _Loader(cold_map),
+ _MovingAverageCross(60),
+ tmp_path / "cold",
+ )
+
+ assert warm_engine.trades and cold_engine.trades
+ assert min(t.entry_time for t in warm_engine.trades) < min(
+ t.entry_time for t in cold_engine.trades
+ )
+
+
+class TestOptionsEngineHonoursTheBoundary:
+ """The options pipeline builds its own date sequence and needs the same cut."""
+
+ dates = pd.bdate_range("2025-01-01", periods=8)
+ bars = pd.DataFrame(
+ {
+ "open": np.linspace(100.0, 107.0, 8),
+ "high": np.linspace(101.0, 108.0, 8),
+ "low": np.linspace(99.0, 106.0, 8),
+ "close": np.linspace(100.5, 107.5, 8),
+ "volume": [1000] * 8,
+ },
+ index=dates,
+ )
+
+ class _OptionsLoader:
+ name = "yfinance"
+
+ def __init__(self, bars):
+ self._bars = bars
+
+ def fetch(self, codes, start_date, end_date):
+ return {"SPY": self._bars.copy()}
+
+ class _OpenOnDay:
+ """Opens on day 1 — inside the warm-up — and never closes."""
+
+ def generate(self, data_map):
+ return [
+ {
+ "date": "2025-01-01",
+ "action": "open",
+ "underlying": "SPY",
+ "legs": [
+ {"type": "call", "strike": 101.0, "expiry": "2025-03-21", "qty": 5}
+ ],
+ }
+ ]
+
+ def _run(self, tmp_path, extra):
+ from backtest.engines.options_portfolio import run_options_backtest
+
+ run_options_backtest(
+ {
+ "codes": ["SPY"],
+ "start_date": "2025-01-01",
+ "end_date": "2025-01-10",
+ "source": "yfinance",
+ "engine": "options",
+ "initial_cash": 100_000,
+ **extra,
+ },
+ self._OptionsLoader(self.bars),
+ self._OpenOnDay(),
+ tmp_path,
+ )
+ return tmp_path / "artifacts"
+
+ def test_a_signal_inside_the_warmup_never_fills(self, tmp_path):
+ graded_everything = pd.read_csv(self._run(tmp_path / "a", {}) / "trades.csv")
+ assert not graded_everything.empty, "fixture must trade, or it proves nothing"
+
+ after_warmup = self._run(tmp_path / "b", {"warmup_bars": 3}) / "trades.csv"
+ assert pd.read_csv(after_warmup).empty