Skip to content

"Vibe-Trading: Your Personal Trading Agent"

19.9 KB
Raw Blame
"""Bench runner: compute IC stats for every alpha in a zoo over one universe.

Extracted from ``agent/scripts/w4a_run_benches.py`` so the same pipeline can be
called by:

- the CLI bench driver (``w4a_run_benches.py``)
- the Web UI background worker (``src/api/alpha_routes.py``)

The math is unchanged — only the carrier moved. Categorisation thresholds:

- ``alive``     : ic_mean > 0.02 and ic_positive_ratio >= 0.55 and |t| > 2
- ``reversed``  : ic_mean < -0.02 and |t| > 2
- ``dead``      : everything else
"""

from __future__ import annotations

import logging
import math
import os
import time
from concurrent.futures import ProcessPoolExecutor, as_completed
from typing import Any, Callable, Iterable, Mapping

import numpy as np

from src.config.accessor import get_env_config
from src.factors.factor_analysis_core import compute_ic_series
from src.quantlib.multipletesting import (
    MIN_OBSERVATIONS,
    deflated_sharpe_ratio,
    expected_maximum_sharpe,
)
from src.factors.registry import (
    Registry,
    RegistryError,
    SkipAlpha,
    get_default_registry,
)
from src.factors.data_availability import build_availability_contract
from src.factors.research_contract import build_bench_research_contract
from src.tools.alpha_bench_tool import _MAX_HORIZON, _compute_forward_returns, _load_universe_panel, _prepare_bench_panel, _split_alphas_by_universe_support, _validate_bench_data_requirements

logger = logging.getLogger(__name__)


ProgressCb = Callable[[int, int, str], None]
"""Signature: ``on_progress(n_done, n_total, current_alpha_id)``."""

_WORKER_PANEL: dict[str, Any] | None = None
_WORKER_RETURN_DF: Any | None = None


def _init_bench_worker(panel: dict[str, Any], return_df: Any) -> None:
    """Store large bench inputs once per worker process."""
    global _WORKER_PANEL, _WORKER_RETURN_DF
    _WORKER_PANEL = panel
    _WORKER_RETURN_DF = return_df


def _compute_single_alpha(args: Any) -> dict[str, Any]:
    """Picklable worker for parallel bench.

    Normal parallel execution passes only an alpha id after ``_init_bench_worker``
    has stored the large panel and return matrix once per worker. The tuple
    form is kept for direct unit tests and small one-off calls.
    """
    if isinstance(args, tuple):
        alpha_id, panel, return_df = args
    else:
        alpha_id = str(args)
        panel = _WORKER_PANEL
        return_df = _WORKER_RETURN_DF
        if panel is None or return_df is None:
            return {"skip": {"id": alpha_id, "reason": "worker inputs not initialized", "kind": "unexpected"}}

    reg = get_default_registry()
    try:
        factor_df = reg.compute(alpha_id, panel)
        ic = compute_ic_series(factor_df, return_df)
        if len(ic) < 2:
            return {"skip": {"id": alpha_id, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}}
        ic_mean = float(ic.mean())
        ic_std = float(ic.std())
        ir = ic_mean / ic_std if ic_std > 0 else 0.0
        meta = reg.get(alpha_id).meta or {}
        return {
            "row": {
                "id": alpha_id,
                "ic_mean": round(ic_mean, 6),
                "ic_std": round(ic_std, 6),
                "ir": round(ir, 4),
                "ic_positive_ratio": round(float((ic > 0).mean()), 4),
                "ic_count": int(len(ic)),
                "theme": meta.get("theme", []),
                "formula_latex": meta.get("formula_latex", ""),
            }
        }
    except (SkipAlpha, RegistryError, RuntimeError, KeyError, ValueError) as exc:
        return {"skip": {"id": alpha_id, "reason": str(exc), "kind": "typed"}}
    except Exception as exc:  # noqa: BLE001
        return {"skip": {"id": alpha_id, "reason": f"unexpected: {exc}", "kind": "unexpected"}}


def t_stat(ic_mean: float, ic_std: float, n: int) -> float:
    """Two-sided t-statistic of the IC series."""
    if not (n > 0 and ic_std > 0 and math.isfinite(ic_std)):
        return 0.0
    return ic_mean / (ic_std / math.sqrt(n))


def categorise(row: dict[str, Any]) -> str:
    """Bucket a per-alpha row into alive / reversed / dead.

    Thresholds are calibrated on daily next-bar ICs; at horizon h > 1 the
    |IC| > 0.02 floor is a looser bar (h-bar ICs are naturally larger in
    magnitude). The run envelope states its own horizon.
    """
    ic_mean = row["ic_mean"]
    pos = row["ic_positive_ratio"]
    t = t_stat(ic_mean, row["ic_std"], row["ic_count"])
    if ic_mean > 0.02 and pos >= 0.55 and abs(t) > 2:
        return "alive"
    if ic_mean < -0.02 and abs(t) > 2:
        return "reversed"
    return "dead"


def theme_breakdown(rows: list[dict[str, Any]]) -> dict[str, dict[str, int]]:
    """Aggregate alive/reversed/dead counts by theme tag."""
    by_theme: dict[str, dict[str, int]] = {}
    for row in rows:
        cat = row["_category"]
        themes = row.get("theme", []) or ["uncategorised"]
        for theme in themes:
            bucket = by_theme.setdefault(
                theme, {"alive": 0, "reversed": 0, "dead": 0, "count": 0}
            )
            bucket[cat] += 1
            bucket["count"] += 1
    return by_theme


def run_bench(
    zoo: str,
    universe: str,
    period: str,
    top: int = 20,
    on_progress: ProgressCb | None = None,
    registry: Registry | None = None,
    only: Iterable[str] | None = None,
    on_stage: Callable[[str], None] | None = None,
    horizon: int = 1,
    research_mode: str = "explore",
    research_stage: str = "single_factor_evaluation",
    research_lineage: Iterable[str | Mapping[str, Any]] | None = None,
    research_transformations: Iterable[str | Mapping[str, Any]] | None = None,
) -> dict[str, Any]:
    """Run a bench end-to-end and return the API-shaped summary.

    Args:
        zoo: Zoo id (e.g. ``gtja191``, ``alpha101``, ``qlib158``).
        universe: Universe key understood by ``_load_universe_panel``
            (e.g. ``csi300``, ``sp500``, ``btc-usdt``).
        period: ``YYYY-YYYY`` or ``YYYY-MM-DD/YYYY-MM-DD`` window.
        top: Number of top-IR alphas to keep in ``top5_by_ir`` and
            ``dead_examples`` (capped at ``top``).
        on_progress: Optional callback fired after every alpha completes
            (success or skip). Signature: ``(n_done, n_total, alpha_id)``.
        registry: Optional pre-built registry (test injection); defaults to a
            fresh ``Registry()``.
        only: Optional subset of alpha ids to evaluate. When provided, the zoo's
            alpha list is restricted to this set — used by ``alpha compare`` to
            bench just a handful of named alphas instead of the whole zoo. Ids
            not registered under ``zoo`` are silently dropped from the subset.
        horizon: Forward-return horizon in bars (default 1 = next-bar,
            unchanged behaviour). ``h > 1`` computes h-bar forward returns
            sampled non-overlapping (end-anchored) so the naive-iid t-stat
            stays honest; 21 ≈ monthly, the conventional horizon for
            fundamental factors.
        research_mode: Declared evidence mode for the additive research
            contract (``explore``, ``strict`` or ``locked``). It is metadata
            only in this version and does not change the bench math.
        research_stage: Stage name recorded in the research contract.
        research_lineage: Optional structured inputs for future artifact
            lineage. Strings are accepted as a compact form.
        research_transformations: Optional transformations applied before the
            bench, such as neutralization or factor combination.

    Returns:
        Dict with keys: ``status``, ``zoo``, ``universe``, ``period``,
        ``horizon``, ``forward_return_sampling``, ``n_alphas_tested``,
        ``n_skipped``, ``alive``, ``reversed``, ``dead``, ``by_theme``,
        ``top5_by_ir``, ``dead_examples``, ``wall_seconds`` —
        and on failure: ``status="error"``, ``error``.
    """
    start = time.monotonic()
    entry: dict[str, Any] = {
        "status": "pending",
        "zoo": zoo,
        "universe": universe,
        "period": period,
        "horizon": horizon,
        "forward_return_sampling": (
            "non_overlapping"
            if isinstance(horizon, int) and not isinstance(horizon, bool) and horizon > 1
            else "daily"
        ),
    }

    # Fail fast before any panel load (network): same 1.._MAX_HORIZON contract
    # the tool and MCP wrapper enforce.
    if isinstance(horizon, bool) or not isinstance(horizon, int) or not 1 <= horizon <= _MAX_HORIZON:
        entry["status"] = "error"
        entry["error"] = (
            f"horizon must be an integer between 1 and {_MAX_HORIZON}, got {horizon!r}"
        )
        entry["wall_seconds"] = round(time.monotonic() - start, 2)
        return entry

    reg = registry if registry is not None else get_default_registry()
    alpha_ids = reg.list(zoo=zoo)
    if not alpha_ids:
        entry["status"] = "error"
        entry["error"] = f"no alphas registered under zoo={zoo!r}"
        entry["wall_seconds"] = round(time.monotonic() - start, 2)
        return entry

    if only is not None:
        only_set = {str(aid) for aid in only}
        alpha_ids = [aid for aid in alpha_ids if aid in only_set]
        if not alpha_ids:
            entry["status"] = "error"
            entry["error"] = f"none of the requested alphas are registered under zoo={zoo!r}"
            entry["wall_seconds"] = round(time.monotonic() - start, 2)
            return entry

    alpha_ids, universe_excluded = _split_alphas_by_universe_support(reg, alpha_ids, universe)
    if universe_excluded:
        entry["universe_excluded"] = universe_excluded
        logger.info(
            "bench: %d alpha(s) excluded on %s — fund:* inputs need SEC US filings (sp500): %s",
            len(universe_excluded),
            universe,
            ", ".join(universe_excluded[:10]),
        )
    if not alpha_ids:
        entry["status"] = "error"
        entry["error"] = (
            f"all {len(universe_excluded)} selected alphas require SEC US filings "
            f"(fund:* inputs); the {universe!r} universe cannot serve them. "
            "Choose the S&P 500 universe."
        )
        entry["wall_seconds"] = round(time.monotonic() - start, 2)
        return entry

    try:
        _validate_bench_data_requirements(reg, alpha_ids, universe)
        if on_stage:
            on_stage("loading_data")
        panel = _load_universe_panel(universe, period)
    except (ValueError, NotImplementedError, RuntimeError) as exc:
        entry["status"] = "error"
        entry["error"] = f"universe load failed: {exc}"
        entry["wall_seconds"] = round(time.monotonic() - start, 2)
        return entry

    try:
        panel = _prepare_bench_panel(reg, alpha_ids, panel, universe, period)
        if on_stage:
            on_stage("preparing_returns")
        return_df = _compute_forward_returns(panel, horizon=horizon)
    except Exception as exc:  # noqa: BLE001
        entry["status"] = "error"
        entry["error"] = f"forward returns failed: {exc}"
        entry["wall_seconds"] = round(time.monotonic() - start, 2)
        return entry

    rows: list[dict[str, Any]] = []
    skipped: list[dict[str, Any]] = []
    n_total = len(alpha_ids)
    if on_stage:
        on_stage("computing")

    try:
        n_workers = get_env_config().agent_tuning.vibe_trading_bench_workers or os.cpu_count() or 1
    except ValueError:
        logger.warning("invalid VIBE_TRADING_BENCH_WORKERS; falling back to sequential")
        n_workers = 1
    n_workers = max(1, min(n_workers, n_total))
    use_parallel = registry is None and n_workers > 1 and n_total > 1

    if use_parallel:
        ctx_kwargs: dict[str, Any] = {}
        try:
            import multiprocessing

            ctx = multiprocessing.get_context("fork")
            ctx_kwargs["mp_context"] = ctx
        except (ImportError, ValueError):
            pass

        future_to_id: dict[Any, str] = {}
        with ProcessPoolExecutor(
            max_workers=n_workers,
            initializer=_init_bench_worker,
            initargs=(panel, return_df),
            **ctx_kwargs,
        ) as pool:
            for aid in alpha_ids:
                fut = pool.submit(_compute_single_alpha, aid)
                future_to_id[fut] = aid

            n_done = 0
            for fut in as_completed(future_to_id):
                n_done += 1
                aid = future_to_id[fut]
                try:
                    result = fut.result()
                except Exception as exc:  # noqa: BLE001
                    logger.exception("bench: worker crash on %s", aid)
                    skipped.append({"id": aid, "reason": f"worker crash: {exc}", "kind": "unexpected"})
                    result = None

                if result is not None:
                    if "row" in result:
                        rows.append(result["row"])
                    elif "skip" in result:
                        skipped.append(result["skip"])

                if on_progress is not None:
                    try:
                        on_progress(n_done, n_total, aid)
                    except Exception:  # noqa: BLE001
                        logger.exception("on_progress callback raised; ignoring")
    else:
        for idx, aid in enumerate(alpha_ids, start=1):
            try:
                factor_df = reg.compute(aid, panel)
                ic = compute_ic_series(factor_df, return_df)
                if len(ic) < 2:
                    skipped.append(
                        {"id": aid, "reason": "insufficient IC observations (need at least 2)", "kind": "typed"}
                    )
                else:
                    ic_mean = float(ic.mean())
                    ic_std = float(ic.std())
                    ir = ic_mean / ic_std if ic_std > 0 else 0.0
                    meta = reg.get(aid).meta or {}
                    rows.append(
                        {
                            "id": aid,
                            "ic_mean": round(ic_mean, 6),
                            "ic_std": round(ic_std, 6),
                            "ir": round(ir, 4),
                            "ic_positive_ratio": round(float((ic > 0).mean()), 4),
                            "ic_count": int(len(ic)),
                            "theme": meta.get("theme", []),
                            "formula_latex": meta.get("formula_latex", ""),
                        }
                    )
            except (SkipAlpha, RegistryError, RuntimeError, KeyError, ValueError) as exc:
                skipped.append({"id": aid, "reason": str(exc), "kind": "typed"})
            except Exception as exc:  # noqa: BLE001
                logger.exception("bench: unexpected failure on %s", aid)
                skipped.append(
                    {"id": aid, "reason": f"unexpected: {exc}", "kind": "unexpected"}
                )

            if on_progress is not None:
                try:
                    on_progress(idx, n_total, aid)
                except Exception:  # noqa: BLE001
                    logger.exception("on_progress callback raised; ignoring")

    for row in rows:
        row["_category"] = categorise(row)

    counts = {"alive": 0, "reversed": 0, "dead": 0}
    for row in rows:
        counts[row["_category"]] += 1

    rows_by_ir = sorted(rows, key=lambda r: r["ir"], reverse=True)
    rows_by_ic = sorted(rows, key=lambda r: r["ic_mean"])

    def _slim(r: dict[str, Any]) -> dict[str, Any]:
        return {
            "id": r["id"],
            "ic_mean": r["ic_mean"],
            "ir": r["ir"],
            "theme": r["theme"],
            "formula_latex": r["formula_latex"],
            "category": r["_category"],
        }

    # Forward universe metadata (e.g. survivorship_bias flag from sp500
    # loader). Panel meta lives in panel["_meta"] when the loader chooses to
    # set it; absence is fine.
    universe_meta_raw = panel.get("_meta") if isinstance(panel, dict) else None
    universe_meta: dict[str, Any] = {}
    if universe_meta_raw is not None:
        try:
            # Could be a pandas object or a plain dict.
            if hasattr(universe_meta_raw, "to_dict"):
                universe_meta = dict(universe_meta_raw.to_dict())
            else:
                universe_meta = dict(universe_meta_raw)
        except Exception:  # noqa: BLE001
            universe_meta = {"raw": str(universe_meta_raw)}

    required_fields = sorted(
        {
            str(column)
            for aid in alpha_ids
            for column in (reg.get(aid).meta or {}).get("columns_required", [])
        }
    )
    universe_meta["data_availability"] = build_availability_contract(
        panel,
        universe=universe,
        period=period,
        required_fields=required_fields,
    )
    universe_meta["research"] = build_bench_research_contract(
        mode=research_mode,
        stage=research_stage,
        universe=universe,
        period=period,
        factor_ids=alpha_ids,
        zoo=zoo,
        lineage=research_lineage,
        transformations=research_transformations,
        horizon=horizon,
        method="rank_ic",
    )

    # Multiple-testing correction. `n_alphas_tested` has always been carried
    # here and has only ever been displayed; the best IR out of hundreds of
    # searched alphas is the maximum of hundreds of draws, and under a null
    # where none of them works that maximum is comfortably positive. The block
    # below prices that. Every key is NEW -- nothing existing changes value, so
    # no consumer of this dict is affected.
    #
    # Note on what is being deflated: `ir` here is mean(IC)/std(IC), not a
    # return Sharpe. The deflation family applies to any max-of-N t-like
    # statistic, but the resulting probability must be read as "this IC-IR
    # survives the search", never as a statement about realised returns.
    multiple_testing: dict[str, Any] = {"applicable": False}
    if len(rows) >= 2:
        ir_values = np.array([float(r["ir"]) for r in rows], dtype=float)
        best_row = rows_by_ir[0]
        best_ir = float(best_row["ir"])
        ir_dispersion = float(ir_values.std(ddof=1))
        observations = int(best_row.get("ic_count", 0))
        expected_max = expected_maximum_sharpe(len(rows), ir_dispersion)
        multiple_testing = {
            "applicable": True,
            "n_trials": len(rows),
            "best_id": best_row["id"],
            "best_ir": round(best_ir, 6),
            "ir_dispersion": round(ir_dispersion, 6),
            "expected_max_ir_under_null": round(expected_max, 6),
            "ir_haircut": round(best_ir - expected_max, 6),
            "ic_observations": observations,
            "statistic": "ic_information_ratio",
        }
        if observations >= MIN_OBSERVATIONS:
            deflated = deflated_sharpe_ratio(
                observed_sharpe=best_ir,
                n_trials=len(rows),
                n_observations=observations,
                trial_sharpe_std=ir_dispersion,
            )
            multiple_testing["deflated_probability"] = round(
                deflated.deflated_sharpe_ratio, 6
            )
            multiple_testing["survives_deflation"] = deflated.survives
        else:
            multiple_testing["deflated_probability"] = None
            multiple_testing["survives_deflation"] = None
            multiple_testing["note"] = (
                f"only {observations} IC observation(s); the deflated statistic "
                f"needs at least {MIN_OBSERVATIONS} to mean anything"
            )

    entry.update(
        {
            "status": "ok",
            "n_alphas_tested": len(rows),
            "multiple_testing": multiple_testing,
            "n_skipped": len(skipped),
            "alive": counts["alive"],
            "reversed": counts["reversed"],
            "dead": counts["dead"],
            "by_theme": theme_breakdown(rows),
            "top5_by_ir": [_slim(r) for r in rows_by_ir[: min(5, top)]],
            "dead_examples": [_slim(r) for r in rows_by_ic[:5]],
            "rows": rows,
            "skipped": skipped,
            "meta": universe_meta,
            "wall_seconds": round(time.monotonic() - start, 2),
        }
    )
    return entry