"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""Bench runner: compute IC stats for every alpha in a zoo over one universe.
Extracted from ``agent/scripts/w4a_run_benches.py`` so the same pipeline can be
called by:
- the CLI bench driver (``w4a_run_benches.py``)
- the Web UI background worker (``src/api/alpha_routes.py``)
The math is unchanged — only the carrier moved. Categorisation thresholds:
- ``alive`` : ic_mean > 0.02 and ic_positive_ratio >= 0.55 and |t| > 2
- ``reversed`` : ic_mean < -0.02 and |t| > 2
- ``dead`` : everything else
"""
from __future__ import annotations
import logging
import math
import os
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from typing import Any, Callable, Iterable, Mapping
import numpy as np
from src.config.accessor import get_env_config
from src.factors.factor_analysis_core import compute_ic_series
from src.quantlib.multipletesting import (
MIN_OBSERVATIONS,
deflated_sharpe_ratio,
expected_maximum_sharpe,
)
from src.factors.registry import (
Registry,
RegistryError,
SkipAlpha,
get_default_registry,
)
from src.factors.data_availability import build_availability_contract
from src.factors.research_contract import build_bench_research_contract
from src.tools.alpha_bench_tool import _MAX_HORIZON, _compute_forward_returns, _load_universe_panel, _prepare_bench_panel, _split_alphas_by_universe_support, _validate_bench_data_requirements
logger = logging.getLogger(__name__)
ProgressCb = Callable[[int, int, str], None]
"""Signature: ``on_progress(n_done, n_total, current_alpha_id)``."""
_WORKER_PANEL: dict[str, Any] | None = None
_WORKER_RETURN_DF: Any | None = None
def _init_bench_worker(panel: dict[str, Any], return_df: Any) -> None:
"""Store large bench inputs once per worker process."""
global _WORKER_PANEL, _WORKER_RETURN_DF
_WORKER_PANEL = panel
_WORKER_RETURN_DF = return_df
def _compute_single_alpha(args: Any) -> dict[str, Any]:
"""Picklable worker for parallel bench.
Normal parallel execution passes only an alpha id after ``_init_bench_worker``
has stored the large panel and return matrix once per worker. The tuple
form is kept for direct unit tests and small one-off calls.
"""
if isinstance(args, tuple):
alpha_id, panel, return_df = args
else:
alpha_id = str(args)
panel = _WORKER_PANEL
return_df = _WORKER_RETURN_DF
if panel is None or return_df is None:
return {"skip": {"id": alpha_id, "reason": "worker inputs not initialized", "kind": "unexpected"}}
reg = get_default_registry()
try:
factor_df = reg.compute(alpha_id, panel)
ic = compute_ic_series(factor_df, return_df)
if len(ic) < 2:
return {"skip": {"id": alpha_id, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}}
ic_mean = float(ic.mean())
ic_std = float(ic.std())
ir = ic_mean / ic_std if ic_std > 0 else 0.0
meta = reg.get(alpha_id).meta or {}
return {
"row": {
"id": alpha_id,
"ic_mean": round(ic_mean, 6),
"ic_std": round(ic_std, 6),
"ir": round(ir, 4),
"ic_positive_ratio": round(float((ic > 0).mean()), 4),
"ic_count": int(len(ic)),
"theme": meta.get("theme", []),
"formula_latex": meta.get("formula_latex", ""),
}
}
except (SkipAlpha, RegistryError, RuntimeError, KeyError, ValueError) as exc:
return {"skip": {"id": alpha_id, "reason": str(exc), "kind": "typed"}}
except Exception as exc: # noqa: BLE001
return {"skip": {"id": alpha_id, "reason": f"unexpected: {exc}", "kind": "unexpected"}}
def t_stat(ic_mean: float, ic_std: float, n: int) -> float:
"""Two-sided t-statistic of the IC series."""
if not (n > 0 and ic_std > 0 and math.isfinite(ic_std)):
return 0.0
return ic_mean / (ic_std / math.sqrt(n))
def categorise(row: dict[str, Any]) -> str:
"""Bucket a per-alpha row into alive / reversed / dead.
Thresholds are calibrated on daily next-bar ICs; at horizon h > 1 the
|IC| > 0.02 floor is a looser bar (h-bar ICs are naturally larger in
magnitude). The run envelope states its own horizon.
"""
ic_mean = row["ic_mean"]
pos = row["ic_positive_ratio"]
t = t_stat(ic_mean, row["ic_std"], row["ic_count"])
if ic_mean > 0.02 and pos >= 0.55 and abs(t) > 2:
return "alive"
if ic_mean < -0.02 and abs(t) > 2:
return "reversed"
return "dead"
def theme_breakdown(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
"""Aggregate alive/reversed/dead counts by theme tag."""
by_theme: dict[str, dict[str, int]] = {}
for row in rows:
cat = row["_category"]
themes = row.get("theme", []) or ["uncategorised"]
for theme in themes:
bucket = by_theme.setdefault(
theme, {"alive": 0, "reversed": 0, "dead": 0, "count": 0}
)
bucket[cat] += 1
bucket["count"] += 1
return by_theme
def run_bench(
zoo: str,
universe: str,
period: str,
top: int = 20,
on_progress: ProgressCb | None = None,
registry: Registry | None = None,
only: Iterable[str] | None = None,
on_stage: Callable[[str], None] | None = None,
horizon: int = 1,
research_mode: str = "explore",
research_stage: str = "single_factor_evaluation",
research_lineage: Iterable[str | Mapping[str, Any]] | None = None,
research_transformations: Iterable[str | Mapping[str, Any]] | None = None,
) -> dict[str, Any]:
"""Run a bench end-to-end and return the API-shaped summary.
Args:
zoo: Zoo id (e.g. ``gtja191``, ``alpha101``, ``qlib158``).
universe: Universe key understood by ``_load_universe_panel``
(e.g. ``csi300``, ``sp500``, ``btc-usdt``).
period: ``YYYY-YYYY`` or ``YYYY-MM-DD/YYYY-MM-DD`` window.
top: Number of top-IR alphas to keep in ``top5_by_ir`` and
``dead_examples`` (capped at ``top``).
on_progress: Optional callback fired after every alpha completes
(success or skip). Signature: ``(n_done, n_total, alpha_id)``.
registry: Optional pre-built registry (test injection); defaults to a
fresh ``Registry()``.
only: Optional subset of alpha ids to evaluate. When provided, the zoo's
alpha list is restricted to this set — used by ``alpha compare`` to
bench just a handful of named alphas instead of the whole zoo. Ids
not registered under ``zoo`` are silently dropped from the subset.
horizon: Forward-return horizon in bars (default 1 = next-bar,
unchanged behaviour). ``h > 1`` computes h-bar forward returns
sampled non-overlapping (end-anchored) so the naive-iid t-stat
stays honest; 21 ≈ monthly, the conventional horizon for
fundamental factors.
research_mode: Declared evidence mode for the additive research
contract (``explore``, ``strict`` or ``locked``). It is metadata
only in this version and does not change the bench math.
research_stage: Stage name recorded in the research contract.
research_lineage: Optional structured inputs for future artifact
lineage. Strings are accepted as a compact form.
research_transformations: Optional transformations applied before the
bench, such as neutralization or factor combination.
Returns:
Dict with keys: ``status``, ``zoo``, ``universe``, ``period``,
``horizon``, ``forward_return_sampling``, ``n_alphas_tested``,
``n_skipped``, ``alive``, ``reversed``, ``dead``, ``by_theme``,
``top5_by_ir``, ``dead_examples``, ``wall_seconds`` —
and on failure: ``status="error"``, ``error``.
"""
start = time.monotonic()
entry: dict[str, Any] = {
"status": "pending",
"zoo": zoo,
"universe": universe,
"period": period,
"horizon": horizon,
"forward_return_sampling": (
"non_overlapping"
if isinstance(horizon, int) and not isinstance(horizon, bool) and horizon > 1
else "daily"
),
}
# Fail fast before any panel load (network): same 1.._MAX_HORIZON contract
# the tool and MCP wrapper enforce.
if isinstance(horizon, bool) or not isinstance(horizon, int) or not 1 <= horizon <= _MAX_HORIZON:
entry["status"] = "error"
entry["error"] = (
f"horizon must be an integer between 1 and {_MAX_HORIZON}, got {horizon!r}"
)
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
reg = registry if registry is not None else get_default_registry()
alpha_ids = reg.list(zoo=zoo)
if not alpha_ids:
entry["status"] = "error"
entry["error"] = f"no alphas registered under zoo={zoo!r}"
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
if only is not None:
only_set = {str(aid) for aid in only}
alpha_ids = [aid for aid in alpha_ids if aid in only_set]
if not alpha_ids:
entry["status"] = "error"
entry["error"] = f"none of the requested alphas are registered under zoo={zoo!r}"
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
alpha_ids, universe_excluded = _split_alphas_by_universe_support(reg, alpha_ids, universe)
if universe_excluded:
entry["universe_excluded"] = universe_excluded
logger.info(
"bench: %d alpha(s) excluded on %s — fund:* inputs need SEC US filings (sp500): %s",
len(universe_excluded),
universe,
", ".join(universe_excluded[:10]),
)
if not alpha_ids:
entry["status"] = "error"
entry["error"] = (
f"all {len(universe_excluded)} selected alphas require SEC US filings "
f"(fund:* inputs); the {universe!r} universe cannot serve them. "
"Choose the S&P 500 universe."
)
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
try:
_validate_bench_data_requirements(reg, alpha_ids, universe)
if on_stage:
on_stage("loading_data")
panel = _load_universe_panel(universe, period)
except (ValueError, NotImplementedError, RuntimeError) as exc:
entry["status"] = "error"
entry["error"] = f"universe load failed: {exc}"
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
try:
panel = _prepare_bench_panel(reg, alpha_ids, panel, universe, period)
if on_stage:
on_stage("preparing_returns")
return_df = _compute_forward_returns(panel, horizon=horizon)
except Exception as exc: # noqa: BLE001
entry["status"] = "error"
entry["error"] = f"forward returns failed: {exc}"
entry["wall_seconds"] = round(time.monotonic() - start, 2)
return entry
rows: list[dict[str, Any]] = []
skipped: list[dict[str, Any]] = []
n_total = len(alpha_ids)
if on_stage:
on_stage("computing")
try:
n_workers = get_env_config().agent_tuning.vibe_trading_bench_workers or os.cpu_count() or 1
except ValueError:
logger.warning("invalid VIBE_TRADING_BENCH_WORKERS; falling back to sequential")
n_workers = 1
n_workers = max(1, min(n_workers, n_total))
use_parallel = registry is None and n_workers > 1 and n_total > 1
if use_parallel:
ctx_kwargs: dict[str, Any] = {}
try:
import multiprocessing
ctx = multiprocessing.get_context("fork")
ctx_kwargs["mp_context"] = ctx
except (ImportError, ValueError):
pass
future_to_id: dict[Any, str] = {}
with ProcessPoolExecutor(
max_workers=n_workers,
initializer=_init_bench_worker,
initargs=(panel, return_df),
**ctx_kwargs,
) as pool:
for aid in alpha_ids:
fut = pool.submit(_compute_single_alpha, aid)
future_to_id[fut] = aid
n_done = 0
for fut in as_completed(future_to_id):
n_done += 1
aid = future_to_id[fut]
try:
result = fut.result()
except Exception as exc: # noqa: BLE001
logger.exception("bench: worker crash on %s", aid)
skipped.append({"id": aid, "reason": f"worker crash: {exc}", "kind": "unexpected"})
result = None
if result is not None:
if "row" in result:
rows.append(result["row"])
elif "skip" in result:
skipped.append(result["skip"])
if on_progress is not None:
try:
on_progress(n_done, n_total, aid)
except Exception: # noqa: BLE001
logger.exception("on_progress callback raised; ignoring")
else:
for idx, aid in enumerate(alpha_ids, start=1):
try:
factor_df = reg.compute(aid, panel)
ic = compute_ic_series(factor_df, return_df)
if len(ic) < 2:
skipped.append(
{"id": aid, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}
)
else:
ic_mean = float(ic.mean())
ic_std = float(ic.std())
ir = ic_mean / ic_std if ic_std > 0 else 0.0
meta = reg.get(aid).meta or {}
rows.append(
{
"id": aid,
"ic_mean": round(ic_mean, 6),
"ic_std": round(ic_std, 6),
"ir": round(ir, 4),
"ic_positive_ratio": round(float((ic > 0).mean()), 4),
"ic_count": int(len(ic)),
"theme": meta.get("theme", []),
"formula_latex": meta.get("formula_latex", ""),
}
)
except (SkipAlpha, RegistryError, RuntimeError, KeyError, ValueError) as exc:
skipped.append({"id": aid, "reason": str(exc), "kind": "typed"})
except Exception as exc: # noqa: BLE001
logger.exception("bench: unexpected failure on %s", aid)
skipped.append(
{"id": aid, "reason": f"unexpected: {exc}", "kind": "unexpected"}
)
if on_progress is not None:
try:
on_progress(idx, n_total, aid)
except Exception: # noqa: BLE001
logger.exception("on_progress callback raised; ignoring")
for row in rows:
row["_category"] = categorise(row)
counts = {"alive": 0, "reversed": 0, "dead": 0}
for row in rows:
counts[row["_category"]] += 1
rows_by_ir = sorted(rows, key=lambda r: r["ir"], reverse=True)
rows_by_ic = sorted(rows, key=lambda r: r["ic_mean"])
def _slim(r: dict[str, Any]) -> dict[str, Any]:
return {
"id": r["id"],
"ic_mean": r["ic_mean"],
"ir": r["ir"],
"theme": r["theme"],
"formula_latex": r["formula_latex"],
"category": r["_category"],
}
# Forward universe metadata (e.g. survivorship_bias flag from sp500
# loader). Panel meta lives in panel["_meta"] when the loader chooses to
# set it; absence is fine.
universe_meta_raw = panel.get("_meta") if isinstance(panel, dict) else None
universe_meta: dict[str, Any] = {}
if universe_meta_raw is not None:
try:
# Could be a pandas object or a plain dict.
if hasattr(universe_meta_raw, "to_dict"):
universe_meta = dict(universe_meta_raw.to_dict())
else:
universe_meta = dict(universe_meta_raw)
except Exception: # noqa: BLE001
universe_meta = {"raw": str(universe_meta_raw)}
required_fields = sorted(
{
str(column)
for aid in alpha_ids
for column in (reg.get(aid).meta or {}).get("columns_required", [])
}
)
universe_meta["data_availability"] = build_availability_contract(
panel,
universe=universe,
period=period,
required_fields=required_fields,
)
universe_meta["research"] = build_bench_research_contract(
mode=research_mode,
stage=research_stage,
universe=universe,
period=period,
factor_ids=alpha_ids,
zoo=zoo,
lineage=research_lineage,
transformations=research_transformations,
horizon=horizon,
method="rank_ic",
)
# Multiple-testing correction. `n_alphas_tested` has always been carried
# here and has only ever been displayed; the best IR out of hundreds of
# searched alphas is the maximum of hundreds of draws, and under a null
# where none of them works that maximum is comfortably positive. The block
# below prices that. Every key is NEW -- nothing existing changes value, so
# no consumer of this dict is affected.
#
# Note on what is being deflated: `ir` here is mean(IC)/std(IC), not a
# return Sharpe. The deflation family applies to any max-of-N t-like
# statistic, but the resulting probability must be read as "this IC-IR
# survives the search", never as a statement about realised returns.
multiple_testing: dict[str, Any] = {"applicable": False}
if len(rows) >= 2:
ir_values = np.array([float(r["ir"]) for r in rows], dtype=float)
best_row = rows_by_ir[0]
best_ir = float(best_row["ir"])
ir_dispersion = float(ir_values.std(ddof=1))
observations = int(best_row.get("ic_count", 0))
expected_max = expected_maximum_sharpe(len(rows), ir_dispersion)
multiple_testing = {
"applicable": True,
"n_trials": len(rows),
"best_id": best_row["id"],
"best_ir": round(best_ir, 6),
"ir_dispersion": round(ir_dispersion, 6),
"expected_max_ir_under_null": round(expected_max, 6),
"ir_haircut": round(best_ir - expected_max, 6),
"ic_observations": observations,
"statistic": "ic_information_ratio",
}
if observations >= MIN_OBSERVATIONS:
deflated = deflated_sharpe_ratio(
observed_sharpe=best_ir,
n_trials=len(rows),
n_observations=observations,
trial_sharpe_std=ir_dispersion,
)
multiple_testing["deflated_probability"] = round(
deflated.deflated_sharpe_ratio, 6
)
multiple_testing["survives_deflation"] = deflated.survives
else:
multiple_testing["deflated_probability"] = None
multiple_testing["survives_deflation"] = None
multiple_testing["note"] = (
f"only {observations} IC observation(s); the deflated statistic "
f"needs at least {MIN_OBSERVATIONS} to mean anything"
)
entry.update(
{
"status": "ok",
"n_alphas_tested": len(rows),
"multiple_testing": multiple_testing,
"n_skipped": len(skipped),
"alive": counts["alive"],
"reversed": counts["reversed"],
"dead": counts["dead"],
"by_theme": theme_breakdown(rows),
"top5_by_ir": [_slim(r) for r in rows_by_ir[: min(5, top)]],
"dead_examples": [_slim(r) for r in rows_by_ic[:5]],
"rows": rows,
"skipped": skipped,
"meta": universe_meta,
"wall_seconds": round(time.monotonic() - start, 2),
}
)
return entry