Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(alpha-bench): skip insufficient IC samples instead of emitting NaN (#1918)

Rudy Mizrahi Celekli96160132ca32
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 7afab180a..97afce16a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -32,6 +32,9 @@ filing dates and require the S&P 500 universe. Results report observed coverage,
missing symbols and current-constituent survivorship bias. A first large-universe
download can take minutes; later evaluations reuse the existing panel cache.
+Factor benchmarks skip alphas with fewer than two IC observations and report
+the reason. They do not rank an unestimable sample standard deviation as zero.
+
Analysis drafts and completed results survive navigation and reload in the same
browser tab. Factor progress reports data loading separately from computation;
Reconnect resumes the existing job or repeats its identical request ID. Running
diff --git a/agent/src/factors/bench_runner.py b/agent/src/factors/bench_runner.py
index b954a94ce..ad1134107 100644
--- a/agent/src/factors/bench_runner.py
+++ b/agent/src/factors/bench_runner.py
@@ -76,8 +76,8 @@ def _compute_single_alpha(args: Any) -> dict[str, Any]:
try:
factor_df = reg.compute(alpha_id, panel)
ic = compute_ic_series(factor_df, return_df)
- if ic.empty:
- return {"skip": {"id": alpha_id, "reason": "empty IC series", "kind": "typed"}}
+ if len(ic) < 2:
+ return {"skip": {"id": alpha_id, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}}
ic_mean = float(ic.mean())
ic_std = float(ic.std())
ir = ic_mean / ic_std if ic_std > 0 else 0.0
@@ -324,9 +324,9 @@ def run_bench(
try:
factor_df = reg.compute(aid, panel)
ic = compute_ic_series(factor_df, return_df)
- if ic.empty:
+ if len(ic) < 2:
skipped.append(
- {"id": aid, "reason": "empty IC series", "kind": "typed"}
+ {"id": aid, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}
)
else:
ic_mean = float(ic.mean())
diff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.py
index 617b3b4e6..65d2f3b6f 100644
--- a/agent/src/tools/alpha_bench_tool.py
+++ b/agent/src/tools/alpha_bench_tool.py
@@ -993,9 +993,9 @@ def _bench_one_alpha(
factor_df = registry.compute(alpha_id, panel)
ic_series = compute_ic_series(factor_df, return_df)
- if ic_series.empty:
+ if len(ic_series) < 2:
raise RuntimeError(
- f"{alpha_id}: IC series empty — insufficient overlap between factor and returns"
+ f"{alpha_id}: insufficient IC observations (need at least 2)"
)
ic_mean = float(ic_series.mean())
ic_std = float(ic_series.std())
diff --git a/agent/tests/test_bench_single_ic_observation.py b/agent/tests/test_bench_single_ic_observation.py
new file mode 100644
index 000000000..236706c0d
--- /dev/null
+++ b/agent/tests/test_bench_single_ic_observation.py
@@ -0,0 +1,77 @@
+"""A single IC cannot estimate sample spread or enter the factor ranking."""
+
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from src.factors import bench_runner
+from src.factors.bench_runner import _compute_single_alpha
+from src.tools import alpha_bench_tool
+
+
+def _panel_and_one_return_row():
+ rng = np.random.default_rng(7)
+ dates = pd.date_range("2024-01-01", periods=30, freq="B")
+ symbols = [f"S{i}" for i in range(10)]
+ close = pd.DataFrame(100 + rng.normal(0, 1, (30, 10)).cumsum(axis=0), index=dates, columns=symbols)
+ factor = pd.DataFrame(rng.normal(size=(30, 10)), index=dates, columns=symbols)
+ return_df = close.pct_change(fill_method=None).shift(-1).iloc[[10]]
+ return {"close": close}, factor, return_df
+
+
+def test_bench_runner_single_ic_is_explicitly_skipped() -> None:
+ panel, factor, return_df = _panel_and_one_return_row()
+ with patch("src.factors.bench_runner.get_default_registry") as get_registry:
+ registry = MagicMock()
+ get_registry.return_value = registry
+ registry.compute.return_value = factor
+ registry.get.return_value = MagicMock(meta={})
+ result = _compute_single_alpha(("alpha_0", panel, return_df))
+ assert "row" not in result
+ assert "insufficient IC" in result["skip"]["reason"]
+ json.dumps(result, allow_nan=False)
+
+
+def test_alpha_bench_single_ic_is_explicitly_rejected() -> None:
+ panel, factor, return_df = _panel_and_one_return_row()
+ registry = SimpleNamespace(
+ compute=lambda alpha_id, panel: factor,
+ get=lambda alpha_id: SimpleNamespace(zoo="z", meta={}),
+ )
+ with pytest.raises(RuntimeError, match="insufficient IC"):
+ alpha_bench_tool._bench_one_alpha(registry, "alpha_0", panel, return_df)
+
+
+def test_sequential_bench_does_not_rank_a_single_observation(monkeypatch):
+ panel, factor, return_df = _panel_and_one_return_row()
+ registry = MagicMock()
+ registry.list.return_value = ["alpha_0"]
+ registry.compute.return_value = factor
+ registry.get.return_value = SimpleNamespace(meta={}, inputs={})
+ monkeypatch.setattr(bench_runner, "_load_universe_panel", lambda *a: panel)
+ monkeypatch.setattr(bench_runner, "_compute_forward_returns", lambda *a, **kw: return_df)
+ result = bench_runner.run_bench("test", "mock", "2024-2025", registry=registry)
+ assert result["n_alphas_tested"] == 0
+ assert "insufficient IC" in result["skipped"][0]["reason"]
+ json.dumps(result, allow_nan=False)
+
+
+def test_two_observations_keep_the_measured_sample_std():
+ panel, factor, _ = _panel_and_one_return_row()
+ returns = panel["close"].pct_change(fill_method=None).shift(-1).iloc[[10, 11]]
+ registry = SimpleNamespace(
+ compute=lambda *args: factor,
+ get=lambda *args: SimpleNamespace(zoo="z", meta={}),
+ )
+ from src.factors.factor_analysis_core import compute_ic_series
+ expected = compute_ic_series(factor, returns).std()
+ row = alpha_bench_tool._bench_one_alpha(registry, "alpha_0", panel, returns)
+ assert row["ic_count"] == 2
+ assert row["ic_std"] == round(expected, 6)
+ json.dumps(row, allow_nan=False)