"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(alpha): surface strict alpha t-stats in bench JSON and HTML report
jay79-boopea2e29a0f108
diff --git a/agent/src/factors/cli_handlers.py b/agent/src/factors/cli_handlers.pyindex f0b0afc5d..1fd188451 100644--- a/agent/src/factors/cli_handlers.py+++ b/agent/src/factors/cli_handlers.py@@ -80,6 +80,17 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]# fail at universe load, not produce a Korean benchmark._UNIVERSE_CHOICES = ["csi300", "sp500", "btc-usdt"]+# Per-row fields that only ``bench_runner_strict`` produces. They are the+# statistics ``categorise_strict`` actually gates on, so a strict run that+# omits them reports a verdict the reader cannot check. Forwarded to both the+# JSON envelope and the HTML report when present.+_STRICT_ROW_FIELDS = (+ "alpha_t_full",+ "alpha_t_train",+ "alpha_t_test",+ "random_ic_mean",+)+# Factor-metadata universes accepted by ``alpha list --universe`` — these are# the values carried in each alpha's ``universe`` metadata, which is what# ``Registry.list`` filters on. Keep in sync with@@ -666,7 +677,7 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:# Normalise rows + skipped for the report template (and for the JSON# envelope so consumers don't see the internal _category key).def _normalise_row(r: dict[str, Any]) -> dict[str, Any]:- return {+ normalised = {"id": r.get("id"),"zoo": r.get("zoo") or _zoo_for_id(reg, r.get("id")),"theme": r.get("theme") or [],@@ -678,6 +689,14 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:"ic_count": r.get("ic_count", 0),"category": r.get("_category") or r.get("category"),}+ # Strict mode decides on the alpha t-stats and the random-control+ # baseline, none of which survived this projection — so a strict+ # run reported a category with no way to see the numbers behind+ # it. Forward them when present; non-strict rows are unchanged.+ for key in _STRICT_ROW_FIELDS:+ if key in r:+ normalised[key] = r[key]+ return normaliseddef _normalise_skipped(s: dict[str, Any]) -> dict[str, Any]:return {"alpha_id": s.get("id") or s.get("alpha_id"), "reason": s.get("reason", "")}@@ -712,6 +731,7 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:"n_skipped": len(skipped),"top": top_rows,"failures": failures_for_report,+ "strict": bool(getattr(args, "strict", False)),}if universe_meta:context["meta"] = universe_metadiff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.pyindex be38bda23..8c7c995b0 100644--- a/agent/src/tools/alpha_bench_tool.py+++ b/agent/src/tools/alpha_bench_tool.py@@ -723,6 +723,28 @@ _JINJA_TEMPLATE = """<!doctype html>{% endfor %}</table>+{% if strict %}+<h2>Strict gate</h2>+<div class="meta">+ Alpha t-stats against the same-universe random control. The strict gate+ decides on these, not on IC.+</div>+<table>+<tr><th>Alpha ID</th><th>alpha_t full</th><th>alpha_t train</th>+ <th>alpha_t test</th><th>random IC mean</th><th>Category</th></tr>+{% for row in top %}+<tr>+ <td>{{ row.id }}</td>+ <td>{{ "%.4f"|format(row.get('alpha_t_full')) if row.get('alpha_t_full') is not none else "n/a" }}</td>+ <td>{{ "%.4f"|format(row.get('alpha_t_train')) if row.get('alpha_t_train') is not none else "n/a" }}</td>+ <td>{{ "%.4f"|format(row.get('alpha_t_test')) if row.get('alpha_t_test') is not none else "n/a" }}</td>+ <td>{{ "%.6f"|format(row.get('random_ic_mean')) if row.get('random_ic_mean') is not none else "n/a" }}</td>+ <td>{{ row.category }}</td>+</tr>+{% endfor %}+</table>+{% endif %}+<h2>Formulas</h2><table><tr><th>Alpha ID</th><th>Formula (LaTeX source)</th></tr>@@ -794,7 +816,30 @@ def _render_html_manual(ctx: dict[str, Any]) -> str:f"<td>{ic_pos}</td>"f"<td>{_esc(row['ic_count'])}</td></tr>")- parts.append("</table><h2>Formulas</h2><table>")+ parts.append("</table>")+ if ctx.get("strict"):+ parts.append(+ "<h2>Strict gate</h2>"+ "<div class=\"meta\">Alpha t-stats against the same-universe random "+ "control. The strict gate decides on these, not on IC.</div><table>"+ "<tr><th>Alpha ID</th><th>alpha_t full</th><th>alpha_t train</th>"+ "<th>alpha_t test</th><th>random IC mean</th><th>Category</th></tr>"+ )++ def _fmt(value: Any, places: int = 4) -> str:+ return "n/a" if value is None else _esc(f"{value:.{places}f}")++ for row in ctx["top"]:+ parts.append(+ f"<tr><td>{_esc(row['id'])}</td>"+ f"<td>{_fmt(row.get('alpha_t_full'))}</td>"+ f"<td>{_fmt(row.get('alpha_t_train'))}</td>"+ f"<td>{_fmt(row.get('alpha_t_test'))}</td>"+ f"<td>{_fmt(row.get('random_ic_mean'), 6)}</td>"+ f"<td>{_esc(row.get('category'))}</td></tr>"+ )+ parts.append("</table>")+ parts.append("<h2>Formulas</h2><table>")parts.append("<tr><th>Alpha ID</th><th>Formula (LaTeX source)</th></tr>")for row in ctx["top"]:parts.append(diff --git a/agent/tests/test_alpha_bench_strict_cli.py b/agent/tests/test_alpha_bench_strict_cli.pyindex c60a00f54..050dfb5af 100644--- a/agent/tests/test_alpha_bench_strict_cli.py+++ b/agent/tests/test_alpha_bench_strict_cli.py@@ -44,6 +44,10 @@ def _strict_result():"theme": ["momentum"],"formula_latex": "x","_category": "confirmed_alive",+ "alpha_t_full": 2.7697,+ "alpha_t_train": 0.7931,+ "alpha_t_test": 3.0179,+ "random_ic_mean": 0.000333,}],"skipped": [],@@ -185,3 +189,60 @@ def test_strict_result_envelope_marks_counts(capsys, monkeypatch, _reg, _no_repoassert called["random_control"] is Trueassert called["oos_split"] is Noneassert called["n_random_seeds"] == 5+++# -- strict statistics must reach the output surfaces -----------------------+# categorise_strict() gates on alpha_t_full / alpha_t_train / alpha_t_test and+# the random-control baseline, but the CLI row projection dropped all four, so+# a strict run printed a verdict with no way to check the numbers behind it.+++def test_strict_envelope_forwards_alpha_t_stats(capsys, monkeypatch, _reg, _no_report):+ """The JSON envelope must expose the statistics the strict gate decides on."""+ _, _, cap = _run(capsys, _ns(strict=True, oos_split="2023-01-01"), monkeypatch)+ row = _envelope(cap.out)["top"][0]+ assert row["alpha_t_full"] == pytest.approx(2.7697)+ assert row["alpha_t_train"] == pytest.approx(0.7931)+ assert row["alpha_t_test"] == pytest.approx(3.0179)+ assert row["random_ic_mean"] == pytest.approx(0.000333)+++def test_legacy_envelope_gains_no_strict_fields(capsys, monkeypatch, _reg, _no_report):+ """A non-strict run's rows are unchanged — the fields are additive only."""+ import src.factors.bench_runner as legacy_mod++ monkeypatch.setattr(legacy_mod, "run_bench", lambda *a, **k: _legacy_result())+ cli_handlers.cmd_alpha_bench(_ns(strict=False))+ row = _envelope(capsys.readouterr().out)["top"][0]+ for key in ("alpha_t_full", "alpha_t_train", "alpha_t_test", "random_ic_mean"):+ assert key not in row+++def _report_html(tmp_path):+ reports = sorted(tmp_path.glob("alpha_bench_*.html"))+ assert reports, "no HTML report was written"+ return reports[-1].read_text(encoding="utf-8")+++def test_strict_report_renders_alpha_t_table(capsys, monkeypatch, tmp_path, _reg):+ """The HTML report gains a strict section carrying the same numbers."""+ import src.tools.alpha_bench_tool as tool++ monkeypatch.setattr(tool, "_default_output_dir", lambda: tmp_path)+ _run(capsys, _ns(strict=True, oos_split="2023-01-01"), monkeypatch)+ html_out = _report_html(tmp_path)+ assert "Strict gate" in html_out+ assert "2.7697" in html_out+ assert "3.0179" in html_out+++def test_legacy_report_has_no_strict_section(capsys, monkeypatch, tmp_path, _reg):+ """Non-strict reports are untouched by the strict section."""+ import src.factors.bench_runner as legacy_mod+ import src.tools.alpha_bench_tool as tool++ monkeypatch.setattr(tool, "_default_output_dir", lambda: tmp_path)+ monkeypatch.setattr(legacy_mod, "run_bench", lambda *a, **k: _legacy_result())+ cli_handlers.cmd_alpha_bench(_ns(strict=False))+ capsys.readouterr()+ assert "Strict gate" not in _report_html(tmp_path)