Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(alpha): surface strict alpha t-stats in bench JSON and HTML report

jay79-boopea2e29a0f108
diff --git a/agent/src/factors/cli_handlers.py b/agent/src/factors/cli_handlers.py
index f0b0afc5d..1fd188451 100644
--- a/agent/src/factors/cli_handlers.py
+++ b/agent/src/factors/cli_handlers.py
@@ -80,6 +80,17 @@ _REPO_ROOT = Path(__file__).resolve().parents[3]
# fail at universe load, not produce a Korean benchmark.
_UNIVERSE_CHOICES = ["csi300", "sp500", "btc-usdt"]
+# Per-row fields that only ``bench_runner_strict`` produces. They are the
+# statistics ``categorise_strict`` actually gates on, so a strict run that
+# omits them reports a verdict the reader cannot check. Forwarded to both the
+# JSON envelope and the HTML report when present.
+_STRICT_ROW_FIELDS = (
+ "alpha_t_full",
+ "alpha_t_train",
+ "alpha_t_test",
+ "random_ic_mean",
+)
+
# Factor-metadata universes accepted by ``alpha list --universe`` — these are
# the values carried in each alpha's ``universe`` metadata, which is what
# ``Registry.list`` filters on. Keep in sync with
@@ -666,7 +677,7 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:
# Normalise rows + skipped for the report template (and for the JSON
# envelope so consumers don't see the internal _category key).
def _normalise_row(r: dict[str, Any]) -> dict[str, Any]:
- return {
+ normalised = {
"id": r.get("id"),
"zoo": r.get("zoo") or _zoo_for_id(reg, r.get("id")),
"theme": r.get("theme") or [],
@@ -678,6 +689,14 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:
"ic_count": r.get("ic_count", 0),
"category": r.get("_category") or r.get("category"),
}
+ # Strict mode decides on the alpha t-stats and the random-control
+ # baseline, none of which survived this projection — so a strict
+ # run reported a category with no way to see the numbers behind
+ # it. Forward them when present; non-strict rows are unchanged.
+ for key in _STRICT_ROW_FIELDS:
+ if key in r:
+ normalised[key] = r[key]
+ return normalised
def _normalise_skipped(s: dict[str, Any]) -> dict[str, Any]:
return {"alpha_id": s.get("id") or s.get("alpha_id"), "reason": s.get("reason", "")}
@@ -712,6 +731,7 @@ def cmd_alpha_bench(args: argparse.Namespace) -> int:
"n_skipped": len(skipped),
"top": top_rows,
"failures": failures_for_report,
+ "strict": bool(getattr(args, "strict", False)),
}
if universe_meta:
context["meta"] = universe_meta
diff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.py
index be38bda23..8c7c995b0 100644
--- a/agent/src/tools/alpha_bench_tool.py
+++ b/agent/src/tools/alpha_bench_tool.py
@@ -723,6 +723,28 @@ _JINJA_TEMPLATE = """<!doctype html>
{% endfor %}
</table>
+{% if strict %}
+<h2>Strict gate</h2>
+<div class="meta">
+ Alpha t-stats against the same-universe random control. The strict gate
+ decides on these, not on IC.
+</div>
+<table>
+<tr><th>Alpha ID</th><th>alpha_t full</th><th>alpha_t train</th>
+ <th>alpha_t test</th><th>random IC mean</th><th>Category</th></tr>
+{% for row in top %}
+<tr>
+ <td>{{ row.id }}</td>
+ <td>{{ "%.4f"|format(row.get('alpha_t_full')) if row.get('alpha_t_full') is not none else "n/a" }}</td>
+ <td>{{ "%.4f"|format(row.get('alpha_t_train')) if row.get('alpha_t_train') is not none else "n/a" }}</td>
+ <td>{{ "%.4f"|format(row.get('alpha_t_test')) if row.get('alpha_t_test') is not none else "n/a" }}</td>
+ <td>{{ "%.6f"|format(row.get('random_ic_mean')) if row.get('random_ic_mean') is not none else "n/a" }}</td>
+ <td>{{ row.category }}</td>
+</tr>
+{% endfor %}
+</table>
+{% endif %}
+
<h2>Formulas</h2>
<table>
<tr><th>Alpha ID</th><th>Formula (LaTeX source)</th></tr>
@@ -794,7 +816,30 @@ def _render_html_manual(ctx: dict[str, Any]) -> str:
f"<td>{ic_pos}</td>"
f"<td>{_esc(row['ic_count'])}</td></tr>"
)
- parts.append("</table><h2>Formulas</h2><table>")
+ parts.append("</table>")
+ if ctx.get("strict"):
+ parts.append(
+ "<h2>Strict gate</h2>"
+ "<div class=\"meta\">Alpha t-stats against the same-universe random "
+ "control. The strict gate decides on these, not on IC.</div><table>"
+ "<tr><th>Alpha ID</th><th>alpha_t full</th><th>alpha_t train</th>"
+ "<th>alpha_t test</th><th>random IC mean</th><th>Category</th></tr>"
+ )
+
+ def _fmt(value: Any, places: int = 4) -> str:
+ return "n/a" if value is None else _esc(f"{value:.{places}f}")
+
+ for row in ctx["top"]:
+ parts.append(
+ f"<tr><td>{_esc(row['id'])}</td>"
+ f"<td>{_fmt(row.get('alpha_t_full'))}</td>"
+ f"<td>{_fmt(row.get('alpha_t_train'))}</td>"
+ f"<td>{_fmt(row.get('alpha_t_test'))}</td>"
+ f"<td>{_fmt(row.get('random_ic_mean'), 6)}</td>"
+ f"<td>{_esc(row.get('category'))}</td></tr>"
+ )
+ parts.append("</table>")
+ parts.append("<h2>Formulas</h2><table>")
parts.append("<tr><th>Alpha ID</th><th>Formula (LaTeX source)</th></tr>")
for row in ctx["top"]:
parts.append(
diff --git a/agent/tests/test_alpha_bench_strict_cli.py b/agent/tests/test_alpha_bench_strict_cli.py
index c60a00f54..050dfb5af 100644
--- a/agent/tests/test_alpha_bench_strict_cli.py
+++ b/agent/tests/test_alpha_bench_strict_cli.py
@@ -44,6 +44,10 @@ def _strict_result():
"theme": ["momentum"],
"formula_latex": "x",
"_category": "confirmed_alive",
+ "alpha_t_full": 2.7697,
+ "alpha_t_train": 0.7931,
+ "alpha_t_test": 3.0179,
+ "random_ic_mean": 0.000333,
}
],
"skipped": [],
@@ -185,3 +189,60 @@ def test_strict_result_envelope_marks_counts(capsys, monkeypatch, _reg, _no_repo
assert called["random_control"] is True
assert called["oos_split"] is None
assert called["n_random_seeds"] == 5
+
+
+# -- strict statistics must reach the output surfaces -----------------------
+# categorise_strict() gates on alpha_t_full / alpha_t_train / alpha_t_test and
+# the random-control baseline, but the CLI row projection dropped all four, so
+# a strict run printed a verdict with no way to check the numbers behind it.
+
+
+def test_strict_envelope_forwards_alpha_t_stats(capsys, monkeypatch, _reg, _no_report):
+ """The JSON envelope must expose the statistics the strict gate decides on."""
+ _, _, cap = _run(capsys, _ns(strict=True, oos_split="2023-01-01"), monkeypatch)
+ row = _envelope(cap.out)["top"][0]
+ assert row["alpha_t_full"] == pytest.approx(2.7697)
+ assert row["alpha_t_train"] == pytest.approx(0.7931)
+ assert row["alpha_t_test"] == pytest.approx(3.0179)
+ assert row["random_ic_mean"] == pytest.approx(0.000333)
+
+
+def test_legacy_envelope_gains_no_strict_fields(capsys, monkeypatch, _reg, _no_report):
+ """A non-strict run's rows are unchanged — the fields are additive only."""
+ import src.factors.bench_runner as legacy_mod
+
+ monkeypatch.setattr(legacy_mod, "run_bench", lambda *a, **k: _legacy_result())
+ cli_handlers.cmd_alpha_bench(_ns(strict=False))
+ row = _envelope(capsys.readouterr().out)["top"][0]
+ for key in ("alpha_t_full", "alpha_t_train", "alpha_t_test", "random_ic_mean"):
+ assert key not in row
+
+
+def _report_html(tmp_path):
+ reports = sorted(tmp_path.glob("alpha_bench_*.html"))
+ assert reports, "no HTML report was written"
+ return reports[-1].read_text(encoding="utf-8")
+
+
+def test_strict_report_renders_alpha_t_table(capsys, monkeypatch, tmp_path, _reg):
+ """The HTML report gains a strict section carrying the same numbers."""
+ import src.tools.alpha_bench_tool as tool
+
+ monkeypatch.setattr(tool, "_default_output_dir", lambda: tmp_path)
+ _run(capsys, _ns(strict=True, oos_split="2023-01-01"), monkeypatch)
+ html_out = _report_html(tmp_path)
+ assert "Strict gate" in html_out
+ assert "2.7697" in html_out
+ assert "3.0179" in html_out
+
+
+def test_legacy_report_has_no_strict_section(capsys, monkeypatch, tmp_path, _reg):
+ """Non-strict reports are untouched by the strict section."""
+ import src.factors.bench_runner as legacy_mod
+ import src.tools.alpha_bench_tool as tool
+
+ monkeypatch.setattr(tool, "_default_output_dir", lambda: tmp_path)
+ monkeypatch.setattr(legacy_mod, "run_bench", lambda *a, **k: _legacy_result())
+ cli_handlers.cmd_alpha_bench(_ns(strict=False))
+ capsys.readouterr()
+ assert "Strict gate" not in _report_html(tmp_path)