Skip to content

"Vibe-Trading: Your Personal Trading Agent"

9.9 KB
Raw Blame
"""Structured backtest summary for the tool result envelope (FE-1 R1 + R4).

The backtest tool envelope historically carried engine results only as raw
stdout, so consumers (e.g. the web frontend) had to ``JSON.parse`` a truncated
log tail to recover metrics.  This module builds a bounded, JSON-safe
``summary`` dict straight from the run directory's own artifacts —
``run_card.json``, ``artifacts/equity.csv`` and the per-symbol
``artifacts/ohlcv_*.csv`` files — and a code → path map of the OHLCV artifacts.

Everything here is best-effort by contract: :func:`try_build_backtest_summary`
never raises and returns ``None`` when a run cannot be summarised (missing or
corrupt ``run_card.json``, unreadable CSV), so the tool envelope stays ``ok``
and simply omits the ``summary`` key.  Payloads are bounded the same way other
tool results are (see :mod:`src.tools._result_paging`): the equity preview is
equal-stride sampled down to at most :data:`MAX_PREVIEW_POINTS` points with the
first and last row always kept, while ``metrics`` stays complete.
"""

from __future__ import annotations

import csv
import json
import logging
import math
from pathlib import Path
from typing import Any, Mapping

logger = logging.getLogger(__name__)

#: Version of the structured tool-result summary contract.
SCHEMA_VERSION = "1.0"
#: Hard cap on ``equity_preview`` points; longer curves are equal-stride
#: sampled with the first and last row always included.
MAX_PREVIEW_POINTS = 50

_OHLCV_PREFIX = "ohlcv_"
_OHLCV_SUFFIX = ".csv"
_OHLCV_GLOB = f"{_OHLCV_PREFIX}*{_OHLCV_SUFFIX}"
# U+00B7 MIDDLE DOT, spelled as an escape so the separator is unambiguous.
_TITLE_SEPARATOR = " \u00b7 "
_PREVIEW_NUMERIC_FIELDS = ("equity", "drawdown", "benchmark_equity")


def collect_ohlcv_paths(run_dir: Path) -> dict[str, str]:
    """Map each symbol code to the absolute path of its OHLCV CSV.

    Globs ``<run_dir>/artifacts/ohlcv_*.csv``; the code is the filename minus
    the ``ohlcv_`` prefix and ``.csv`` suffix (``ohlcv_BTC-USDT.csv`` maps to
    ``BTC-USDT``).  Paths are absolute when ``run_dir`` is absolute, which the
    backtest tool guarantees via ``safe_run_dir``.

    Args:
        run_dir: Backtest run directory root.

    Returns:
        Code → path mapping, empty when there are no OHLCV artifacts or the
        directory cannot be scanned.  Never raises.
    """
    mapping: dict[str, str] = {}
    try:
        candidates = sorted((Path(run_dir) / "artifacts").glob(_OHLCV_GLOB))
        for path in candidates:
            code = path.name[len(_OHLCV_PREFIX) : -len(_OHLCV_SUFFIX)]
            if code and path.is_file() and path.resolve().is_relative_to(Path(run_dir).resolve()):
                mapping[code] = str(path)
    except OSError as exc:  # pragma: no cover - defensive, glob rarely raises
        logger.debug("ohlcv glob failed for %s: %s", run_dir, exc)
    return mapping


def build_backtest_summary(
    run_dir: Path,
    ohlcv_paths: Mapping[str, str] | None = None,
) -> dict[str, Any]:
    """Build the structured summary dict for a finished backtest run.

    Reads ``run_card.json`` (schema 1.0: ``card["backtest"]`` carries
    codes/dates/interval/initial_cash, ``card["metrics"]`` the scalar metrics,
    ``card["warnings"]`` the warning list) and previews ``artifacts/equity.csv``.
    The full metrics object is carried — sanitized recursively so non-finite
    floats become ``None`` and ``json.dumps`` stays valid — while the equity
    curve is sampled down to at most :data:`MAX_PREVIEW_POINTS` points.

    Args:
        run_dir: Backtest run directory root containing ``run_card.json``.
        ohlcv_paths: Optional precomputed code → path mapping; gathered with
            :func:`collect_ohlcv_paths` when omitted.

    Returns:
        JSON-safe summary dict following the FE-1 contract.

    Raises:
        OSError: ``run_card.json`` or ``artifacts/equity.csv`` is unreadable.
        json.JSONDecodeError: ``run_card.json`` is not valid JSON.
        ValueError: ``run_card.json`` does not contain a JSON object.
    """
    run_dir = Path(run_dir).resolve()
    card = _load_run_card(run_dir)
    backtest = card.get("backtest")
    if not isinstance(backtest, dict):
        backtest = {}
    raw_codes = backtest.get("codes")
    codes = [str(code) for code in raw_codes] if isinstance(raw_codes, list) else []

    if ohlcv_paths is None:
        ohlcv_paths = collect_ohlcv_paths(run_dir)

    artifacts_dir = run_dir / "artifacts"
    for name in ("equity.csv", "trades.csv", "metrics.csv"):
        _contained(run_dir, artifacts_dir / name)
    ohlcv_paths = {code: str(_contained(run_dir, Path(path))) for code, path in ohlcv_paths.items()}
    return _json_safe({
        "schema_version": SCHEMA_VERSION,
        "run_id": run_dir.name,
        "title": _TITLE_SEPARATOR.join(codes),
        "codes": codes,
        "start_date": backtest.get("start_date"),
        "end_date": backtest.get("end_date"),
        "interval": backtest.get("interval"),
        "initial_cash": backtest.get("initial_cash"),
        "metrics": _json_safe(_as_dict(card.get("metrics"))),
        "structured_metrics": _json_safe(_as_dict(card.get("structured_metrics"))),
        "validation": _json_safe(_as_dict(card.get("validation"))),
        "equity_preview": _equity_preview(artifacts_dir / "equity.csv"),
        "artifact_paths": {
            "equity": _existing_file(artifacts_dir / "equity.csv"),
            "trades": _existing_file(artifacts_dir / "trades.csv"),
            "metrics": _existing_file(artifacts_dir / "metrics.csv"),
            "run_card_json": str(run_dir / "run_card.json"),
            "ohlcv": dict(ohlcv_paths),
        },
        "warnings": _json_safe(
            card.get("warnings") if isinstance(card.get("warnings"), list) else []
        ),
    })


def try_build_backtest_summary(
    run_dir: Path,
    ohlcv_paths: Mapping[str, str] | None = None,
) -> dict[str, Any] | None:
    """Best-effort :func:`build_backtest_summary` that never raises.

    Args:
        run_dir: Backtest run directory root.
        ohlcv_paths: Optional precomputed code → path mapping.

    Returns:
        The summary dict, or ``None`` when anything about the run directory
        prevents building one (missing/corrupt ``run_card.json``, unreadable
        CSV).  Callers omit the ``summary`` key on ``None``.
    """
    try:
        return build_backtest_summary(run_dir, ohlcv_paths)
    except Exception as exc:  # noqa: BLE001 — contract: never fail the envelope
        logger.debug("backtest summary omitted for %s: %s", run_dir, exc)
        return None


def _load_run_card(run_dir: Path) -> dict[str, Any]:
    """Read and validate ``run_card.json`` at the run directory root."""
    payload = json.loads(_contained(run_dir, run_dir / "run_card.json").read_text(encoding="utf-8"))
    if not isinstance(payload, dict):
        raise ValueError(
            f"run_card.json must contain a JSON object, got {type(payload).__name__}"
        )
    return payload


def _contained(root: Path, path: Path) -> Path:
    """Refuse artifacts whose resolved destination escapes the run directory."""
    resolved = path.resolve()
    if not resolved.is_relative_to(root.resolve()):
        raise ValueError("Summary artifact resolves outside run directory")
    return path


def _as_dict(value: Any) -> dict[str, Any]:
    """Return ``value`` when it is a dict, else an empty dict."""
    return value if isinstance(value, dict) else {}


def _existing_file(path: Path) -> str | None:
    """Absolute-style path string when the file exists, else ``None``."""
    return str(path) if path.is_file() else None


def _json_safe(value: Any) -> Any:
    """Recursively replace non-finite floats with ``None``.

    Mirrors ``backtest.run_card._json_safe``: ``json.dumps(..., allow_nan=False)``
    and strict frontend parsers reject ``NaN``/``Infinity``, and Python's
    ``json.loads`` happily produces both from non-standard literals.
    """
    if isinstance(value, float):
        return value if math.isfinite(value) else None
    if isinstance(value, Mapping):
        return {str(key): _json_safe(item) for key, item in value.items()}
    if isinstance(value, (list, tuple)):
        return [_json_safe(item) for item in value]
    return value


def _parse_float(cell: str | None) -> float | None:
    """Parse one CSV cell into a finite float, else ``None``.

    Covers blank cells, non-numeric text and the ``NaN``/``inf`` spellings
    ``float()`` accepts, so a dirty equity row degrades to nulls instead of
    poisoning the preview.
    """
    if cell is None:
        return None
    try:
        value = float(cell)
    except (TypeError, ValueError):
        return None
    return value if math.isfinite(value) else None


def _equity_preview(equity_path: Path) -> list[dict[str, Any]]:
    """Read ``equity.csv`` into bounded preview points.

    Columns follow the engine's artifact contract: ``timestamp,ret,equity,
    drawdown,benchmark_equity,active_ret``.  ``time`` keeps the raw timestamp
    string; numeric fields become finite floats or ``None``.  A missing or
    empty file yields ``[]`` — the summary is still built.

    Raises:
        OSError: The file exists but cannot be read.
        csv.Error: The file is not parseable CSV.
    """
    if not equity_path.is_file():
        return []
    with equity_path.open(encoding="utf-8", newline="") as stream:
        total = sum(1 for _ in csv.DictReader(stream))
    indices = set(range(total)) if total <= MAX_PREVIEW_POINTS else {
        i * (total - 1) // (MAX_PREVIEW_POINTS - 1) for i in range(MAX_PREVIEW_POINTS)
    }
    rows: list[dict[str, Any]] = []
    with equity_path.open(encoding="utf-8", newline="") as stream:
        for index, record in enumerate(csv.DictReader(stream)):
            if index not in indices:
                continue
            point: dict[str, Any] = {"time": record.get("timestamp")}
            for field in _PREVIEW_NUMERIC_FIELDS:
                point[field] = _parse_float(record.get(field))
            rows.append(point)
    return rows