Skip to content

"Vibe-Trading: Your Personal Trading Agent"

17.0 KB
Raw Blame
"""Portfolio risk x-ray: concentration, volatility, drawdown, tail risk, and
co-movement for a weighted basket, computed from close-price panels.

This is the Portfolio Studio risk-x-ray backend (roadmap slice). Everything
here is a pure function of prices and weights — no I/O, no network, no
loader imports — so the same computation serves the agent tool, the API,
and tests. Data fetching lives in the caller.

Conventions:
- Long-only weights are normalized to sum 1. Long/short weights keep their
  original exposure and signed returns; only concentration is normalized by
  gross share.
- Non-finite inputs are never trusted and non-finite outputs are never
  emitted: every reported float passes through ``_finite`` so the result
  survives ``json.dumps(..., allow_nan=False)``.
- Look-ahead is structurally impossible: all statistics are computed over
  trailing returns only.
"""

from __future__ import annotations

import json
import math
from pathlib import Path
from typing import Any, Mapping, Sequence

import numpy as np
import pandas as pd

from backtest.metrics import effective_bars_per_year
from backtest.validation import _json_safe

MIN_HISTORY_DAYS = 30
PERIODS_PER_YEAR = 252
VAR_LEVELS = (0.95, 0.99)


def _finite(value: float | None) -> float | None:
    """Return ``value`` when finite, else ``None`` (strict-JSON safe)."""
    if value is None:
        return None
    try:
        out = float(value)
    except (TypeError, ValueError):
        return None
    return out if math.isfinite(out) else None


def _validate_weights(
    closes: pd.DataFrame, weights: Mapping[str, float]
) -> tuple[dict[str, float], list[str]]:
    """Validate weights and preserve the exposure semantics for each book type.

    Long-only weights retain the historical normalization to sum 1. A book with
    a short leg keeps its original scale for exposure and return calculations;
    concentration alone is normalized by gross share in ``_concentration``.

    Returns the cleaned weight map (restricted to available columns) and any
    warnings worth surfacing to the caller.
    """
    warnings: list[str] = []
    if not weights:
        raise ValueError("weights must name at least one symbol")

    unknown = [sym for sym in weights if sym not in closes.columns]
    if unknown:
        raise ValueError(f"weights reference symbols with no price data: {sorted(unknown)}")

    cleaned: dict[str, float] = {}
    for sym, raw in weights.items():
        try:
            value = float(raw)
        except (TypeError, ValueError) as exc:
            raise ValueError(f"weight for {sym!r} is not a number: {raw!r}") from exc
        if not math.isfinite(value):
            raise ValueError(f"weight for {sym!r} is not finite: {raw!r}")
        cleaned[sym] = value

    if any(value < 0 for value in cleaned.values()):
        gross = sum(abs(value) for value in cleaned.values())
        if abs(gross - 1.0) > 1e-6:
            warnings.append(f"weights preserved at gross exposure {gross:.6f}; concentration uses gross shares")
        return cleaned, warnings

    total = sum(cleaned.values())
    if total <= 0:
        raise ValueError("weights must sum to a positive value")
    if abs(total - 1.0) > 1e-6:
        warnings.append(f"weights summed to {total:.6f}; renormalized to 1.0")
        cleaned = {sym: value / total for sym, value in cleaned.items()}
    return cleaned, warnings


def compute_risk_xray(
    closes: pd.DataFrame,
    weights: Mapping[str, float],
    *,
    periods_per_year: int | None = PERIODS_PER_YEAR,
    var_levels: Sequence[float] = VAR_LEVELS,
    min_history: int = MIN_HISTORY_DAYS,
) -> dict[str, Any]:
    """Compute the risk x-ray for a weighted basket.

    Args:
        closes: Close-price panel, one column per symbol, sorted by date.
        weights: Symbol → weight. Long-only weights are normalized to sum 1 with
            a warning when needed. Long/short weights preserve their actual
            exposure; concentration is computed from gross shares instead.
        periods_per_year: Annualization factor for the bar interval; ``None``
            falls back to span-derived calendar annualization (mirrors
            ``calc_metrics``) — the runner's cross-market convention.
        var_levels: Tail levels for historical VaR / expected shortfall.
        min_history: Minimum valid bars a symbol must have to be included.

    Returns:
        A strict-JSON-safe dict (``json.dumps(..., allow_nan=False)`` must
        never fail on it) with concentration, volatility, drawdown, tail
        risk, diversification, and correlation sections, plus ``skipped``
        and ``warnings`` bookkeeping.

    Raises:
        ValueError: On unusable input (empty panel, bad weights, or nothing
            left after the history filter / calendar alignment).
    """
    if closes is None or closes.empty:
        raise ValueError("price panel is empty")
    frame = closes.dropna(axis=1, how="all")
    if frame.empty:
        raise ValueError("price panel has no non-NaN closes")

    weights, warnings = _validate_weights(frame, weights)

    # History filter: thin symbols are excluded rather than allowed to skew
    # the whole x-ray on a handful of bars.
    kept: list[str] = []
    skipped: list[dict[str, str]] = []
    for sym in weights:
        valid = int(frame[sym].count())
        if valid < min_history:
            skipped.append({"symbol": sym, "reason": f"only {valid} valid bars (min {min_history})"})
        else:
            kept.append(sym)
    if not kept:
        raise ValueError(f"no symbol has at least {min_history} valid bars")
    if skipped:
        kept_weights = {sym: weights[sym] for sym in kept}
        if any(value < 0 for value in weights.values()):
            # Filtering any leg of a long/short book, including its last short
            # leg, must not change the surviving positions' original scale.
            warnings.append("history filter preserved original long/short exposure")
        else:
            # Preserve the long-only behavior: renormalize over survivors so the
            # x-ray still describes a fully invested basket, and say so.
            total = sum(kept_weights.values())
            if total <= 0:
                raise ValueError("surviving symbols have zero total weight")
            weights = {sym: value / total for sym, value in kept_weights.items()}
            warnings.append("weights renormalized over symbols that survived the history filter")

    aligned = frame[kept].dropna(axis=0, how="any")
    if len(aligned) < 2:
        raise ValueError(
            "fewer than 2 shared trading days after aligning calendars across symbols"
        )

    # Compute each one-bar return before dropping incomplete shared dates.
    # Otherwise an interior missing close becomes a multi-bar move presented
    # as one observation and distorts volatility and tail risk.
    returns = frame[kept].pct_change(fill_method=None).dropna(how="any")
    if returns.empty:
        raise ValueError("no overlapping return observations across symbols")

    w = np.array([weights[sym] for sym in kept], dtype=float)
    port = returns.to_numpy(dtype=float) @ w
    port_returns = pd.Series(port, index=returns.index)

    result: dict[str, Any] = {
        "inputs": {
            "symbols": kept,
            "weights": {sym: round(float(weights[sym]), 8) for sym in kept},
            "aligned_days": int(len(aligned)),
            "return_observations": int(len(returns)),
            "first_date": str(aligned.index[0]),
            "last_date": str(aligned.index[-1]),
        },
        "concentration": _concentration(w),
        "exposure": _exposure(w),
        "volatility": _volatility(port_returns, periods_per_year),
        "drawdown": _drawdown(port_returns),
        "tail_risk": _tail_risk(port_returns, var_levels),
        "diversification": _diversification(returns, w, port_returns, periods_per_year),
        "correlation": _correlation(returns, port_returns),
        "skipped": skipped,
        "warnings": warnings,
    }
    return result


def _concentration(w: np.ndarray) -> dict[str, Any]:
    # Concentration is a share-of-book measure, so normalize by gross exposure
    # here and nowhere else. The original signed weights still drive returns.
    gross = np.abs(w)
    total = float(gross.sum())
    if total > 0:
        gross = gross / total
    hhi = float(np.sum(gross**2))
    order = np.argsort(gross)[::-1]
    return {
        "hhi": _finite(hhi),
        "effective_n": _finite(1.0 / hhi) if hhi > 0 else None,
        "top1_weight": _finite(float(gross[order[0]])) if len(order) else None,
        "top3_weight": _finite(float(gross[order[:3]].sum())) if len(order) else None,
    }


def _exposure(w: np.ndarray) -> dict[str, Any]:
    """Report actual gross/net exposure and the long and short sides."""
    return {
        "gross": _finite(float(np.abs(w).sum())),
        "net": _finite(float(w.sum())),
        "long_weight": _finite(float(w[w > 0].sum())),
        "short_weight": _finite(float(w[w < 0].sum())),
        "long_count": int(np.count_nonzero(w > 0)),
        "short_count": int(np.count_nonzero(w < 0)),
    }


def _volatility(port: pd.Series, ppy: int | None) -> dict[str, Any]:
    vol = float(port.std(ddof=1)) if len(port) > 1 else None
    # Zero-target downside deviation: root mean square of the losses over every
    # return period, matching ``calc_metrics`` Sortino. The sample standard
    # deviation of the losing bars alone measures how much the losses differ
    # from each other, so a book that loses 1% every losing bar reads as zero.
    downside_dev = (
        float(np.sqrt((port.clip(upper=0.0) ** 2).mean())) if len(port) > 0 else None
    )
    annualize = _annualize_factor(ppy, port.index)
    return {
        "daily_vol": _finite(vol),
        "annualized_vol": _finite(vol * annualize) if vol is not None else None,
        "downside_deviation_annualized": (
            _finite(downside_dev * annualize) if downside_dev is not None else None
        ),
    }


def _annualize_factor(ppy: int | None, index: Any) -> float:
    """Return the annualization factor ``sqrt(periods_per_year)``.

    ``ppy is None`` means the caller deliberately declined to specify a
    per-market bar count — the runner's cross-market convention
    (``bars_per_year=None``). Resolving it through the shared
    ``effective_bars_per_year`` is what keeps the x-ray's annualized
    volatility on the same footing as the Sharpe in the same run card (a
    fixed 365 would sit ~18% higher for a 252-trading-day daily series).
    """
    if ppy is not None:
        return math.sqrt(float(ppy))
    return math.sqrt(float(effective_bars_per_year(index)))


def _drawdown(port: pd.Series) -> dict[str, Any]:
    if port.empty:
        return {"max_drawdown": None, "max_drawdown_start": None, "max_drawdown_trough": None}
    equity = (1.0 + port).cumprod()
    # The portfolio starts at wealth 1 before the first return observation.
    # Keeping that initial high-water mark also preserves the drawdown sign
    # after a sub -100% return sends compounded wealth through zero.
    peak = equity.cummax().clip(lower=1.0)
    dd = (equity - peak) / peak
    trough_idx = dd.idxmin()
    pre_trough = equity.loc[:trough_idx]
    start_idx = port.index[0] if float(pre_trough.max()) < 1.0 else pre_trough.idxmax()
    return {
        "max_drawdown": _finite(float(dd.loc[trough_idx])),
        "max_drawdown_start": str(start_idx),
        "max_drawdown_trough": str(trough_idx),
    }


def _tail_risk(port: pd.Series, levels: Sequence[float]) -> dict[str, Any]:
    out: dict[str, Any] = {}
    losses = -port.to_numpy(dtype=float)
    for level in levels:
        if len(losses) < 2:
            var = es = None
        else:
            var = float(np.quantile(losses, level))
            tail = losses[losses >= var]
            es = float(tail.mean()) if len(tail) else None
        key = f"{int(round(level * 100))}"
        out[f"var_{key}"] = _finite(var)
        out[f"expected_shortfall_{key}"] = _finite(es)
    out["method"] = "historical simulation (non-parametric)"
    return out


def _diversification(
    returns: pd.DataFrame, w: np.ndarray, port: pd.Series, ppy: int
) -> dict[str, Any]:
    if returns.shape[1] < 2:
        return {"diversification_ratio": None, "note": "needs at least 2 assets"}
    asset_vols = returns.std(ddof=1).to_numpy(dtype=float)
    port_vol = float(port.std(ddof=1)) if len(port) > 1 else 0.0
    if port_vol <= 0 or not math.isfinite(port_vol):
        return {"diversification_ratio": None, "note": "portfolio variance is zero"}
    ratio = float(np.dot(np.abs(w), asset_vols) / port_vol)
    return {"diversification_ratio": _finite(ratio)}


def _correlation(returns: pd.DataFrame, port: pd.Series) -> dict[str, Any]:
    if returns.shape[1] < 2:
        return {
            "avg_pairwise_abs": None,
            "max_pair": None,
            "beta_to_equal_weight": None,
            "note": "needs at least 2 assets",
        }
    corr = returns.corr().to_numpy(dtype=float)
    n = corr.shape[0]
    off_diag = [abs(corr[i, j]) for i in range(n) for j in range(i + 1, n)]
    avg_pairwise = float(np.mean(off_diag)) if off_diag else None

    max_pair = None
    if off_diag:
        i, j = max(
            ((i, j) for i in range(n) for j in range(i + 1, n)),
            key=lambda pair: abs(corr[pair[0], pair[1]]),
        )
        max_pair = {
            "symbols": [str(returns.columns[i]), str(returns.columns[j])],
            "corr": _finite(float(corr[i, j])),
        }

    market = returns.mean(axis=1)
    market_var = float(market.var(ddof=1)) if len(market) > 1 else 0.0
    if market_var > 0 and math.isfinite(market_var):
        beta = float(port.cov(market) / market_var)
    else:
        beta = None

    return {
        "avg_pairwise_abs": _finite(avg_pairwise),
        "max_pair": max_pair,
        "beta_to_equal_weight": _finite(beta),
    }


def average_invested_weights(target_pos: pd.DataFrame) -> tuple[dict[str, float], float]:
    """Derive the run's average basket from the target position frame.

    Returns ``(weights, avg_invested)``: the mean target weight per symbol
    restricted to symbols with a non-zero average signed exposure, and the mean
    signed row sum retained by the existing ``risk_xray_avg_invested`` metric.
    The final row alone would be the wrong object here, since many strategies
    end flat.

    Raises:
        ValueError: when the frame is empty or every symbol has zero average
            signed exposure. Short legs remain negative in the returned weights
            instead of shrinking the report to the long half of the book.
    """
    if target_pos is None or target_pos.empty:
        raise ValueError("target position frame is empty")
    means = target_pos.mean(axis=0)
    weights = {str(sym): float(w) for sym, w in means.items() if float(w) != 0.0}
    avg_invested = float(target_pos.sum(axis=1).mean())
    if not weights:
        raise ValueError("strategy held no average exposure")
    return weights, avg_invested


def render_risk_xray_markdown(report: dict[str, Any]) -> str:
    """Render an x-ray report as a compact Markdown summary."""
    inputs = report["inputs"]
    conc = report["concentration"]
    vol = report["volatility"]
    dd = report["drawdown"]
    tail = report["tail_risk"]
    lines = [
        "# Portfolio Risk X-Ray",
        "",
        f"- basket: {', '.join(inputs['symbols'])}",
        f"- window: {inputs['first_date']} .. {inputs['last_date']} "
        f"({inputs['aligned_days']} aligned days)",
    ]
    exposure = report.get("exposure")
    if exposure:
        lines.append(
            f"- exposure: gross {exposure['gross']:.2%}, net {exposure['net']:.2%} "
            f"(long {exposure['long_weight']:.2%} / short {exposure['short_weight']:.2%})"
        )
    if conc.get("hhi") is not None:
        lines.append(
            f"- concentration: hhi {conc['hhi']:.4f}, "
            f"effective n {conc['effective_n']:.2f}, "
            f"top1 {conc['top1_weight']:.2%}, top3 {conc['top3_weight']:.2%}"
        )
    if vol.get("annualized_vol") is not None:
        lines.append(f"- annualized vol: {vol['annualized_vol']:.2%}")
    if dd.get("max_drawdown") is not None:
        lines.append(f"- max drawdown: {dd['max_drawdown']:.2%}")
    if tail.get("var_95") is not None:
        lines.append(
            f"- tail (historical): VaR95 {tail['var_95']:.2%}, ES95 {tail['expected_shortfall_95']:.2%}"
        )
    if report["skipped"]:
        joined = ", ".join(item["symbol"] for item in report["skipped"])
        lines.append(f"- skipped: {joined}")
    lines.append("")
    return "\n".join(lines)


def write_risk_xray(path: Path, report: dict[str, Any]) -> dict[str, Any]:
    """Write the report to ``path`` as strict, RFC-8259 JSON.

    Same contract as ``write_rebalance_notes``: sanitize with ``_json_safe``
    and serialize with ``allow_nan=False``. Returns the sanitized payload.
    """
    safe_report = _json_safe(report)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(
        json.dumps(safe_report, indent=2, ensure_ascii=False, allow_nan=False) + "\n",
        encoding="utf-8",
    )
    return safe_report