"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Statistical validation for backtest results.
Three independent tools:
- Monte Carlo permutation test: is the strategy significantly better than random?
- Bootstrap Sharpe CI: how stable is the risk-adjusted return?
- Walk-Forward analysis: is performance consistent across time windows?
Usage: called automatically by BaseEngine.run_backtest when config[\"validation\"]
is present, or invoked directly on backtest outputs.
Percentage-based validation requires positive, finite observed equity.
An undefined calculation returns only an error, never a numeric placeholder.
"""
from __future__ import annotations
import json
import math
from numbers import Integral, Real
from pathlib import Path
from typing import Any, Dict, List
import numpy as np
import pandas as pd
from backtest.metrics import effective_bars_per_year, period_returns
from backtest.models import TradeRecord
# ─── Monte Carlo Permutation Test ───
def monte_carlo_test(
trades: List[TradeRecord],
initial_capital: float,
n_simulations: int = 1000,
seed: int = 42,
bars_per_year: int = 252,
) -> Dict[str, Any]:
"""Shuffle trade PnL order to test path significance.
Null hypothesis: the observed Sharpe / max-drawdown is no better than
a random ordering of the same trades.
Args:
trades: Completed round-trip trades from backtest.
initial_capital: Starting capital.
n_simulations: Number of random permutations.
seed: Random seed for reproducibility.
bars_per_year: Annualisation factor (must match bootstrap_sharpe_ci
and walk_forward_analysis so the report's Sharpe figures agree).
Returns:
Dict with actual_sharpe, p_value_sharpe, actual_max_dd,
p_value_max_dd, simulated_sharpes (percentiles).
"""
if isinstance(n_simulations, bool) or not isinstance(n_simulations, Integral) or n_simulations < 1:
return {
"error": f"n_simulations must be >= 1, got {n_simulations}",
}
if isinstance(seed, bool) or not isinstance(seed, Integral) or seed < 0:
return {"error": f"seed must be >= 0, got {seed}"}
if len(trades) < 3:
return {"error": "need at least 3 trades"}
pnls = np.array([t.pnl for t in trades])
actual = _path_metrics(pnls, initial_capital, bars_per_year)
if actual is None:
return {"error": "Monte Carlo requires positive finite equity at every observed trade step"}
rng = np.random.default_rng(seed)
sharpe_count = 0
dd_count = 0
sim_sharpes = []
# The full path matrix feeds the fan-chart payload; skip it for
# pathological sizes so a huge run cannot balloon memory or the JSON.
keep_paths = n_simulations * len(pnls) <= 2_000_000
sim_equities = np.empty((n_simulations, len(pnls))) if keep_paths else None
for i in range(n_simulations):
shuffled = rng.permutation(pnls)
if sim_equities is not None:
sim_equities[i] = initial_capital + np.cumsum(shuffled)
sim = _path_metrics(shuffled, initial_capital, bars_per_year)
if sim is None:
return {"error": f"Monte Carlo requires positive finite equity at every simulated trade step (simulation {i + 1})"}
sim_sharpes.append(sim["sharpe"])
if sim["sharpe"] >= actual["sharpe"]:
sharpe_count += 1
if sim["max_dd"] >= actual["max_dd"]: # less negative = "better"
dd_count += 1
sim_arr = np.array(sim_sharpes)
result = {
"actual_sharpe": round(actual["sharpe"], 4),
"actual_max_dd": round(actual["max_dd"], 4),
"p_value_sharpe": round(sharpe_count / n_simulations, 4),
"p_value_max_dd": round(dd_count / n_simulations, 4),
"simulated_sharpe_mean": round(float(sim_arr.mean()), 4),
"simulated_sharpe_std": round(float(sim_arr.std()), 4),
"simulated_sharpe_p5": round(float(np.percentile(sim_arr, 5)), 4),
"simulated_sharpe_p95": round(float(np.percentile(sim_arr, 95)), 4),
"n_simulations": n_simulations,
"n_trades": len(trades),
"sharpe_samples": [round(float(s), 4) for s in sim_sharpes],
}
if sim_equities is not None:
idx = np.unique(np.linspace(0, len(pnls) - 1, min(len(pnls), 400)).astype(int))
sample_rows = np.unique(
np.linspace(0, n_simulations - 1, min(30, n_simulations)).astype(int)
)
result["equity_paths"] = {
"steps": (idx + 1).tolist(),
"initial_capital": round(float(initial_capital), 2),
"actual": np.round((initial_capital + np.cumsum(pnls))[idx], 2).tolist(),
"band_p5": np.round(np.percentile(sim_equities[:, idx], 5, axis=0), 2).tolist(),
"band_p25": np.round(np.percentile(sim_equities[:, idx], 25, axis=0), 2).tolist(),
"band_p50": np.round(np.percentile(sim_equities[:, idx], 50, axis=0), 2).tolist(),
"band_p75": np.round(np.percentile(sim_equities[:, idx], 75, axis=0), 2).tolist(),
"band_p95": np.round(np.percentile(sim_equities[:, idx], 95, axis=0), 2).tolist(),
"samples": np.round(sim_equities[np.ix_(sample_rows, idx)], 2).tolist(),
}
return result
def _path_metrics(
pnls: np.ndarray, initial_capital: float, bars_per_year: int = 252
) -> Dict[str, float] | None:
"""Compute path metrics, or return None when percentage returns are undefined."""
# Include starting cash so the first trade contributes to returns and
# drawdown in both the observed path and every shuffled path.
equity = np.concatenate(([initial_capital], initial_capital + np.cumsum(pnls)))
if not np.isfinite(equity).all() or (equity <= 0).any():
return None
if len(equity) > 1:
prev = equity[:-1]
diff = np.diff(equity)
returns = np.where(prev != 0, diff / np.where(prev != 0, prev, 1.0), 0.0)
else:
returns = np.array([0.0])
std = returns.std()
sharpe = float(returns.mean() / (std + 1e-10) * np.sqrt(bars_per_year))
peak = np.maximum.accumulate(equity)
dd = (equity - peak) / np.where(peak > 0, peak, 1.0)
max_dd = float(dd.min())
return {"sharpe": sharpe, "max_dd": max_dd}
# ─── Bootstrap Sharpe CI ───
def bootstrap_sharpe_ci(
equity_curve: pd.Series,
n_bootstrap: int = 1000,
confidence: float = 0.95,
bars_per_year: int = 252,
seed: int = 42,
initial_capital: float | None = None,
) -> Dict[str, Any]:
"""Resample daily returns to estimate Sharpe confidence interval.
Args:
equity_curve: Equity time series.
n_bootstrap: Number of bootstrap samples.
confidence: Confidence level (e.g. 0.95 for 95% CI).
bars_per_year: Annualisation factor.
seed: Random seed.
initial_capital: Starting capital. When given, the first return is
measured from this cash mark (the calc_metrics convention), so
``observed_sharpe`` agrees with the report's own Sharpe on paths
whose first recorded bar carries real PnL. None keeps the legacy
adjacent-pair series.
Returns:
Dict with observed_sharpe, ci_lower, ci_upper, median_sharpe,
prob_positive (fraction of samples with Sharpe > 0).
"""
if isinstance(n_bootstrap, bool) or not isinstance(n_bootstrap, Integral) or n_bootstrap < 1:
return {"error": f"n_bootstrap must be >= 1, got {n_bootstrap}"}
if (
isinstance(confidence, bool)
or not isinstance(confidence, Real)
or not math.isfinite(float(confidence))
or not 0.0 < confidence < 1.0
):
return {"error": f"confidence must be in (0, 1), got {confidence}"}
if isinstance(seed, bool) or not isinstance(seed, Integral) or seed < 0:
return {"error": f"seed must be >= 0, got {seed}"}
observed = equity_curve.dropna()
if not np.isfinite(observed).all() or (observed <= 0).any():
return {"error": "Bootstrap Sharpe CI requires positive finite observed equity"}
if initial_capital is not None and (not math.isfinite(initial_capital) or initial_capital <= 0):
return {"error": "Bootstrap Sharpe CI requires positive finite initial capital"}
if initial_capital is not None:
series = period_returns(equity_curve, initial_capital)
else:
series = equity_curve.pct_change(fill_method=None)
# Missing equity is not a flat observation or a one-period recovery return.
returns = (
series
.replace([np.inf, -np.inf], 0.0)
.dropna()
.values
)
if len(returns) < 5:
return {"error": "need at least 5 return observations"}
observed = _sharpe(returns, bars_per_year)
rng = np.random.default_rng(seed)
boot_sharpes = []
for _ in range(n_bootstrap):
sample = rng.choice(returns, size=len(returns), replace=True)
boot_sharpes.append(_sharpe(sample, bars_per_year))
arr = np.array(boot_sharpes)
alpha = (1 - confidence) / 2
lower = float(np.percentile(arr, alpha * 100))
upper = float(np.percentile(arr, (1 - alpha) * 100))
prob_pos = float(np.mean(arr > 0))
result = {
"observed_sharpe": round(observed, 4),
"ci_lower": round(lower, 4),
"ci_upper": round(upper, 4),
"median_sharpe": round(float(np.median(arr)), 4),
"prob_positive": round(prob_pos, 4),
"confidence": confidence,
"n_bootstrap": n_bootstrap,
}
if n_bootstrap <= 20_000:
result["sharpe_samples"] = [round(float(s), 4) for s in boot_sharpes]
return result
def _sharpe(returns: np.ndarray, bars_per_year: int = 252) -> float:
std = returns.std()
return float(returns.mean() / (std + 1e-10) * np.sqrt(bars_per_year))
# ─── Walk-Forward Analysis ───
def walk_forward_analysis(
equity_curve: pd.Series,
trades: List[TradeRecord],
n_windows: int = 5,
bars_per_year: int = 252,
) -> Dict[str, Any]:
"""Split backtest into sequential windows, check consistency.
Each window is evaluated independently (returns normalised to window start).
Args:
equity_curve: Equity time series.
trades: Completed trades.
n_windows: Number of non-overlapping windows.
bars_per_year: Annualisation factor.
Returns:
Dict with per_window stats, consistency metrics.
"""
if isinstance(n_windows, bool) or not isinstance(n_windows, Integral) or n_windows < 1:
return {"error": f"n_windows must be >= 1, got {n_windows}"}
if len(equity_curve) < n_windows * 2:
return {"error": f"need at least {n_windows * 2} bars for {n_windows} windows"}
indices = equity_curve.index
window_size = len(indices) // n_windows
windows = []
for i in range(n_windows):
start_idx = i * window_size
end_idx = (i + 1) * window_size if i < n_windows - 1 else len(indices)
win_eq = equity_curve.iloc[start_idx:end_idx]
win_start = indices[start_idx]
win_end = indices[end_idx - 1]
if not np.isfinite(win_eq.iloc[[0, -1]]).all():
return {"error": f"window {i + 1} requires observed start and end equity"}
observed = win_eq.dropna()
if not np.isfinite(observed).all() or (observed <= 0).any():
return {"error": f"window {i + 1} requires positive finite observed equity"}
# Per-window trades
win_trades = [t for t in trades if win_start <= t.entry_time <= win_end]
# Per-window metrics
ret = float(win_eq.iloc[-1] / win_eq.iloc[0] - 1) if win_eq.iloc[0] > 0 else 0.0
# Use only observed adjacent pairs within this window.
win_returns = (
win_eq.pct_change(fill_method=None)
.replace([np.inf, -np.inf], 0.0)
.dropna()
.values
)
if not len(win_returns):
return {"error": f"window {i + 1} has no observed adjacent equity returns"}
sharpe = _sharpe(win_returns, bars_per_year) if len(win_returns) > 1 else 0.0
peak = win_eq.cummax()
dd = (win_eq - peak) / peak.replace(0, 1)
max_dd = float(dd.min())
win_pnls = [t.pnl for t in win_trades]
win_rate = len([p for p in win_pnls if p > 0]) / len(win_pnls) if win_pnls else 0.0
windows.append(
{
"window": i + 1,
"start": str(win_start.date()) if hasattr(win_start, "date") else str(win_start),
"end": str(win_end.date()) if hasattr(win_end, "date") else str(win_end),
"return": round(ret, 6),
"sharpe": round(sharpe, 4),
"max_dd": round(max_dd, 6),
"trades": len(win_trades),
"win_rate": round(win_rate, 4),
}
)
# Consistency metrics
returns_list = [w["return"] for w in windows]
sharpes_list = [w["sharpe"] for w in windows]
profitable_windows = sum(1 for r in returns_list if r > 0)
return {
"n_windows": n_windows,
"windows": windows,
"profitable_windows": profitable_windows,
"consistency_rate": round(profitable_windows / n_windows, 4),
"return_mean": round(float(np.mean(returns_list)), 6),
"return_std": round(float(np.std(returns_list)), 6),
"sharpe_mean": round(float(np.mean(sharpes_list)), 4),
"sharpe_std": round(float(np.std(sharpes_list)), 4),
}
# ─── Runner integration ───
def run_validation(
config: Dict[str, Any],
equity_curve: pd.Series,
trades: List[TradeRecord],
initial_capital: float,
bars_per_year: int | None = 252,
) -> Dict[str, Any]:
"""Run configured validation checks.
Reads from config["validation"]:
- monte_carlo: {n_simulations, seed}
- bootstrap: {n_bootstrap, confidence, seed}
- walk_forward: {n_windows}
Args:
config: Backtest config (must contain "validation" key).
equity_curve: Equity time series.
trades: Completed trades.
initial_capital: Starting capital.
bars_per_year: Annualisation factor.
Returns:
Dict keyed by validation type with results.
"""
v_cfg = config.get("validation", {})
results: Dict[str, Any] = {}
# Cross-market convention (runner.py passes bars_per_year=None): resolve
# it through the shared span-derived factor — otherwise _sharpe's
# np.sqrt(bars_per_year) raises TypeError for every validation-enabled
# cross-market run.
if bars_per_year is None:
bars_per_year = effective_bars_per_year(equity_curve.index)
if "monte_carlo" in v_cfg:
mc_cfg = v_cfg["monte_carlo"] if isinstance(v_cfg["monte_carlo"], dict) else {}
results["monte_carlo"] = monte_carlo_test(
trades,
initial_capital,
n_simulations=mc_cfg.get("n_simulations", 1000),
seed=mc_cfg.get("seed", 42),
bars_per_year=bars_per_year,
)
if "bootstrap" in v_cfg:
bs_cfg = v_cfg["bootstrap"] if isinstance(v_cfg["bootstrap"], dict) else {}
results["bootstrap"] = bootstrap_sharpe_ci(
equity_curve,
bars_per_year=bars_per_year,
n_bootstrap=bs_cfg.get("n_bootstrap", 1000),
confidence=bs_cfg.get("confidence", 0.95),
seed=bs_cfg.get("seed", 42),
initial_capital=initial_capital,
)
if "walk_forward" in v_cfg:
wf_cfg = v_cfg["walk_forward"] if isinstance(v_cfg["walk_forward"], dict) else {}
results["walk_forward"] = walk_forward_analysis(
equity_curve,
trades,
n_windows=wf_cfg.get("n_windows", 5),
bars_per_year=bars_per_year,
)
return results
# ─── Standalone CLI ───
def _load_equity(run_dir: Path) -> pd.Series:
"""Load equity curve from artifacts/equity.csv."""
path = run_dir / "artifacts" / "equity.csv"
df = pd.read_csv(path, index_col=0, parse_dates=True)
for col in ("equity", "nav", "value"):
if col in df.columns:
return df[col]
raise ValueError(
f"equity.csv must contain an equity/nav/value column; got {list(df.columns)}"
)
def _load_trades(run_dir: Path) -> List[TradeRecord]:
"""Load trades from artifacts/trades.csv and convert to TradeRecord list."""
path = run_dir / "artifacts" / "trades.csv"
df = pd.read_csv(path)
if df.empty:
return []
# New artifacts identify exits explicitly, including break-even trades.
# Unmarked historical/other-engine files retain their legacy convention.
trades = []
exit_mask = df["event"].eq("exit") if "event" in df.columns else df["pnl"].ne(0)
exit_rows = df[exit_mask]
for row_idx, row in exit_rows.iterrows():
exit_time = pd.Timestamp(row.get("timestamp", "2000-01-01"))
entry_time = exit_time
# BaseEngine emits adjacent entry/exit pairs. Recover entry dates only
# from an explicit matching pair; legacy/exit-only files stay unchanged.
if "event" in df.columns and row_idx > 0:
entry = df.iloc[row_idx - 1]
expected_side = {"sell": "buy", "buy": "sell"}.get(row.get("side"))
if (
entry["event"] == "entry"
and entry.get("code") == row.get("code")
and expected_side is not None
and entry.get("side") == expected_side
):
candidate_time = pd.to_datetime(entry.get("timestamp"), errors="coerce")
try:
if pd.notna(candidate_time) and candidate_time <= exit_time:
entry_time = candidate_time
except TypeError:
pass # Incompatible timestamp timezones cannot form a pair.
hold = pd.to_numeric(row.get("holding_bars"), errors="coerce")
if pd.isna(hold):
hold = pd.to_numeric(row.get("holding_days", 0), errors="coerce")
holding_bars = 0.0 if pd.isna(hold) else float(hold)
trades.append(
TradeRecord(
symbol=str(row.get("code", "")),
direction=1 if row.get("side") == "sell" else -1,
entry_price=0.0,
exit_price=float(row.get("price", 0)),
entry_time=entry_time,
exit_time=exit_time,
size=float(row.get("qty", 0)),
leverage=1.0,
pnl=float(row.get("pnl", 0)),
pnl_pct=float(row.get("return_pct", 0)),
exit_reason=str(row.get("reason", "signal")),
holding_bars=holding_bars,
commission=0.0,
)
)
return trades
def _parse_run_dir(argv: List[str]) -> Path:
"""Validate CLI input and return a usable run directory path."""
if len(argv) < 2:
raise SystemExit("Usage: python -m backtest.validation <run_dir>")
raw_run_dir = argv[1]
if not raw_run_dir.strip():
raise SystemExit("run_dir must be a non-empty path")
if "\0" in raw_run_dir:
raise SystemExit("Invalid run_dir path: embedded NUL byte")
try:
run_dir = Path(raw_run_dir).expanduser()
exists = run_dir.exists()
is_dir = run_dir.is_dir() if exists else False
except (OSError, RuntimeError, ValueError) as exc:
raise SystemExit(f"Invalid run_dir path: {exc}") from exc
if not exists:
raise SystemExit(f"run_dir does not exist: {run_dir}")
if not is_dir:
raise SystemExit(f"run_dir is not a directory: {run_dir}")
return run_dir
def _json_safe(value: Any) -> Any:
"""Return a JSON-strict copy of validation results."""
if isinstance(value, np.ndarray):
return [_json_safe(item) for item in value.tolist()]
if isinstance(value, np.generic):
return _json_safe(value.item())
if isinstance(value, float):
return value if math.isfinite(value) else None
if isinstance(value, dict):
return {str(key): _json_safe(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_json_safe(item) for item in value]
return value
def write_validation_json(path: Path, results: Dict[str, Any]) -> Dict[str, Any]:
"""Write validation results to ``path`` as strict, RFC-8259 JSON.
A validation metric can be non-finite (e.g. a Sharpe computed from a path
whose equity touches zero), and ``json.dumps`` emits bare ``NaN`` /
``Infinity`` tokens for those by default (``allow_nan=True``) — tokens that
strict parsers reject. Sanitize with :func:`_json_safe` (non-finite → null)
and serialize with ``allow_nan=False`` so every writer of
``artifacts/validation.json`` produces the same valid JSON. Returns the
sanitized payload that was written.
"""
safe_results = _json_safe(results)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(safe_results, indent=2, ensure_ascii=False, allow_nan=False) + "\n",
encoding="utf-8",
)
return safe_results
def _saved_bars_per_year(config: Dict[str, Any], equity: pd.Series, run_dir: Path) -> int | None:
"""Annualisation factor for a saved run, chosen the way the runner chose it.
The engine passes the run's ``bars_per_year`` into :func:`run_validation`;
Prefer the factor recorded by the runner after checking the served bars.
For legacy cards, apply the runner's spacing check to the saved equity
index. This is a reconstruction: the original per-symbol price indexes
are not available. A config without codes keeps the historical default.
Args:
config: The saved run config.
equity: Saved equity observations for the legacy spacing fallback.
run_dir: Directory containing the optional run card.
Returns:
Bars per year, or ``None`` for calendar-day annualisation.
"""
card_path = run_dir / "run_card.json"
if card_path.exists():
card = json.loads(card_path.read_text(encoding="utf-8"))
if "annualisation" in card:
value = card["annualisation"]["bars_per_year"]
if value is None:
return None
if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
raise ValueError("run-card bars_per_year must be a positive integer or null")
return value
codes = [code for code in config.get("codes") or [] if isinstance(code, str) and code]
if not codes:
return 252
from backtest.runner import _annualisation_bars, _detect_market, _detect_primary_source
if len({_detect_market(code) for code in codes}) > 1:
return None
source = _detect_primary_source(codes, config.get("source", "tushare"))
return _annualisation_bars(
config.get("interval", "1D"), source, {"equity": equity.to_frame()}, ["equity"]
)
def main(run_dir: Path) -> Dict[str, Any]:
"""Run all three validations on existing backtest artifacts.
Reads equity.csv, trades.csv, and config.json from run_dir.
Runs all three tools with saved validation settings, defaulting missing ones.
Args:
run_dir: Directory with artifacts/ subdirectory.
Returns:
Validation results dict.
"""
# Load initial cash and saved validation settings.
config_path = run_dir / "config.json"
if config_path.exists():
config = json.loads(config_path.read_text(encoding="utf-8"))
else:
config = {}
initial_capital = config.get("initial_cash", 1_000_000)
equity = _load_equity(run_dir)
trades = _load_trades(run_dir)
# Standalone validation always runs all tools, even when the original run
# enabled only a subset. Reuse dispatcher defaults and parameter guards.
settings = {name: {} for name in ("monte_carlo", "bootstrap", "walk_forward")}
saved_settings = config.get("validation")
if isinstance(saved_settings, dict):
settings.update(saved_settings)
results = run_validation(
{"validation": settings}, equity, trades, initial_capital,
_saved_bars_per_year(config, equity, run_dir),
)
out = run_dir / "artifacts" / "validation.json"
safe_results = write_validation_json(out, results)
print(json.dumps(safe_results, indent=2, allow_nan=False))
return safe_results
if __name__ == "__main__":
import sys
main(_parse_run_dir(sys.argv))