"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(metrics): use full-sample downside deviation for Sortino
Rix95389681ea0a
diff --git a/agent/backtest/metrics.py b/agent/backtest/metrics.pyindex 79f85409a..486d2fad7 100644--- a/agent/backtest/metrics.py+++ b/agent/backtest/metrics.py@@ -580,7 +580,7 @@ def calc_metrics(ann_ret = float("inf")# ``Series.std()`` uses ddof=1, so a single-observation return series# (e.g. a one-bar backtest) yields NaN and poisons the Sharpe ratio.- # Guard the small sample the same way ``downside_std`` is guarded below.+ # Guard the small sample before calculating volatility.vol = float(port_ret.std()) if len(port_ret) > 1 and returns_finite else 0.0sharpe = (float(port_ret.mean() / (vol + 1e-10) * np.sqrt(bpy))@@ -601,11 +601,15 @@ def calc_metrics(calmar = ann_ret / abs(max_dd) if abs(max_dd) > 1e-10 else 0.0- # Sortino+ # Sortino: zero-target downside RMS, with all return periods in the divisor.if returns_finite:downside = port_ret[port_ret < 0]- downside_std = float(downside.std()) if len(downside) > 1 else 1e-10- sortino = float(port_ret.mean() / (downside_std + 1e-10) * np.sqrt(bpy))+ downside_deviation = (+ float(np.sqrt((downside**2).sum() / len(port_ret)))+ if len(downside) > 0+ else 1e-10 # Preserve the existing no-downside fallback.+ )+ sortino = float(port_ret.mean() / (downside_deviation + 1e-10) * np.sqrt(bpy))else:sortino = 0.0if not np.isfinite(sortino):@@ -636,7 +640,7 @@ def calc_metrics(excess = total_ret - bench_returnaligned_bench = bench_ret.reindex(port_ret.index).fillna(0.0)active_ret = port_ret - aligned_bench- # Same ddof=1 small-sample guard as ``vol`` / ``downside_std`` so the+ # Same ddof=1 small-sample guard as ``vol`` so the# information ratio stays finite for a single-observation series.active_std = float(active_ret.std()) if len(active_ret) > 1 and returns_finite else 0.0ir = (diff --git a/agent/tests/test_metrics_sortino.py b/agent/tests/test_metrics_sortino.pynew file mode 100644index 000000000..581a19962--- /dev/null+++ b/agent/tests/test_metrics_sortino.py@@ -0,0 +1,59 @@+"""Sortino uses zero-target downside RMS over all observed return periods."""++import math++import pandas as pd+import pytest++from backtest.metrics import calc_metrics+++ "equity, mean_return, downside_mean_square",+ [+ # Returns include the existing initial zero: [0, -.01, -.01, .03].+ ([100.0, 99.0, 98.01, 100.9503], 0.0025, 0.0002 / 4),+ # One loss is enough to define downside risk: [0, -.01, .03].+ ([100.0, 99.0, 101.97], 0.02 / 3, 0.0001 / 3),+ # Upside periods still count in the denominator: [0, -.01, -.02, .06].+ ([100.0, 99.0, 97.02, 102.8412], 0.03 / 4, 0.0005 / 4),+ # Equal losses have real risk even though their standard deviation is zero.+ ([100.0, 99.0, 98.01], -0.02 / 3, 0.0002 / 3),+ ],+ ids=["equal-losses", "single-loss", "unequal-losses", "only-losses"],+)[email protected]("bars_per_year", [1, 252])+def test_sortino_uses_full_sample_downside_deviation(+ equity: list[float],+ mean_return: float,+ downside_mean_square: float,+ bars_per_year: int,+) -> None:+ result = calc_metrics(pd.Series(equity), [], 100.0, bars_per_year)+ expected = mean_return / math.sqrt(downside_mean_square) * math.sqrt(bars_per_year)+ assert result["sortino"] == pytest.approx(expected, abs=1e-4)+++def test_sortino_preserves_no_downside_convention() -> None:+ # Zero downside retains the existing finite fallback, not a defined ratio.+ equity = pd.Series([100.0, 110.0, 121.0])+ result = calc_metrics(equity, [], 100.0, bars_per_year=1)+ assert result["sortino"] == pytest.approx((0.2 / 3) / 2e-10)++[email protected]("equity", [[], [100.0], [100.0, 100.0], [100.0, 0.0, 50.0]])+def test_sortino_preserves_degenerate_path_conventions(equity: list[float]) -> None:+ result = calc_metrics(pd.Series(equity, dtype=float), [], 100.0)+ assert result["sortino"] == 0.0+++def test_sortino_preserves_calendar_annualization() -> None:+ # Four bars across 366 days resolve to int(4 / (366 / 365.25)) == 3 bars/year.+ equity = pd.Series(+ [100.0, 99.0, 98.01, 100.9503],+ index=pd.to_datetime(["2024-01-01", "2024-05-01", "2024-09-01", "2025-01-01"]),+ )+ result = calc_metrics(equity, [], 100.0, bars_per_year=None)+ assert result["sortino"] == pytest.approx(+ 0.0025 / math.sqrt(0.00005) * math.sqrt(3), abs=1e-4+ )