Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(metrics): use full-sample downside deviation for Sortino

Rix95389681ea0a
diff --git a/agent/backtest/metrics.py b/agent/backtest/metrics.py
index 79f85409a..486d2fad7 100644
--- a/agent/backtest/metrics.py
+++ b/agent/backtest/metrics.py
@@ -580,7 +580,7 @@ def calc_metrics(
ann_ret = float("inf")
# ``Series.std()`` uses ddof=1, so a single-observation return series
# (e.g. a one-bar backtest) yields NaN and poisons the Sharpe ratio.
- # Guard the small sample the same way ``downside_std`` is guarded below.
+ # Guard the small sample before calculating volatility.
vol = float(port_ret.std()) if len(port_ret) > 1 and returns_finite else 0.0
sharpe = (
float(port_ret.mean() / (vol + 1e-10) * np.sqrt(bpy))
@@ -601,11 +601,15 @@ def calc_metrics(
calmar = ann_ret / abs(max_dd) if abs(max_dd) > 1e-10 else 0.0
- # Sortino
+ # Sortino: zero-target downside RMS, with all return periods in the divisor.
if returns_finite:
downside = port_ret[port_ret < 0]
- downside_std = float(downside.std()) if len(downside) > 1 else 1e-10
- sortino = float(port_ret.mean() / (downside_std + 1e-10) * np.sqrt(bpy))
+ downside_deviation = (
+ float(np.sqrt((downside**2).sum() / len(port_ret)))
+ if len(downside) > 0
+ else 1e-10 # Preserve the existing no-downside fallback.
+ )
+ sortino = float(port_ret.mean() / (downside_deviation + 1e-10) * np.sqrt(bpy))
else:
sortino = 0.0
if not np.isfinite(sortino):
@@ -636,7 +640,7 @@ def calc_metrics(
excess = total_ret - bench_return
aligned_bench = bench_ret.reindex(port_ret.index).fillna(0.0)
active_ret = port_ret - aligned_bench
- # Same ddof=1 small-sample guard as ``vol`` / ``downside_std`` so the
+ # Same ddof=1 small-sample guard as ``vol`` so the
# information ratio stays finite for a single-observation series.
active_std = float(active_ret.std()) if len(active_ret) > 1 and returns_finite else 0.0
ir = (
diff --git a/agent/tests/test_metrics_sortino.py b/agent/tests/test_metrics_sortino.py
new file mode 100644
index 000000000..581a19962
--- /dev/null
+++ b/agent/tests/test_metrics_sortino.py
@@ -0,0 +1,59 @@
+"""Sortino uses zero-target downside RMS over all observed return periods."""
+
+import math
+
+import pandas as pd
+import pytest
+
+from backtest.metrics import calc_metrics
+
+
+ "equity, mean_return, downside_mean_square",
+ [
+ # Returns include the existing initial zero: [0, -.01, -.01, .03].
+ ([100.0, 99.0, 98.01, 100.9503], 0.0025, 0.0002 / 4),
+ # One loss is enough to define downside risk: [0, -.01, .03].
+ ([100.0, 99.0, 101.97], 0.02 / 3, 0.0001 / 3),
+ # Upside periods still count in the denominator: [0, -.01, -.02, .06].
+ ([100.0, 99.0, 97.02, 102.8412], 0.03 / 4, 0.0005 / 4),
+ # Equal losses have real risk even though their standard deviation is zero.
+ ([100.0, 99.0, 98.01], -0.02 / 3, 0.0002 / 3),
+ ],
+ ids=["equal-losses", "single-loss", "unequal-losses", "only-losses"],
+)
[email protected]("bars_per_year", [1, 252])
+def test_sortino_uses_full_sample_downside_deviation(
+ equity: list[float],
+ mean_return: float,
+ downside_mean_square: float,
+ bars_per_year: int,
+) -> None:
+ result = calc_metrics(pd.Series(equity), [], 100.0, bars_per_year)
+ expected = mean_return / math.sqrt(downside_mean_square) * math.sqrt(bars_per_year)
+ assert result["sortino"] == pytest.approx(expected, abs=1e-4)
+
+
+def test_sortino_preserves_no_downside_convention() -> None:
+ # Zero downside retains the existing finite fallback, not a defined ratio.
+ equity = pd.Series([100.0, 110.0, 121.0])
+ result = calc_metrics(equity, [], 100.0, bars_per_year=1)
+ assert result["sortino"] == pytest.approx((0.2 / 3) / 2e-10)
+
+
[email protected]("equity", [[], [100.0], [100.0, 100.0], [100.0, 0.0, 50.0]])
+def test_sortino_preserves_degenerate_path_conventions(equity: list[float]) -> None:
+ result = calc_metrics(pd.Series(equity, dtype=float), [], 100.0)
+ assert result["sortino"] == 0.0
+
+
+def test_sortino_preserves_calendar_annualization() -> None:
+ # Four bars across 366 days resolve to int(4 / (366 / 365.25)) == 3 bars/year.
+ equity = pd.Series(
+ [100.0, 99.0, 98.01, 100.9503],
+ index=pd.to_datetime(["2024-01-01", "2024-05-01", "2024-09-01", "2025-01-01"]),
+ )
+ result = calc_metrics(equity, [], 100.0, bars_per_year=None)
+ assert result["sortino"] == pytest.approx(
+ 0.0025 / math.sqrt(0.00005) * math.sqrt(3), abs=1e-4
+ )