Skip to content

"Vibe-Trading: Your Personal Trading Agent"

12.9 KB
Raw Blame
"""yfinance-backed loader for global equity and crypto OHLCV data."""

from __future__ import annotations

import logging
from collections import defaultdict
from collections.abc import Mapping
from typing import Dict, List, Optional, Union

import pandas as pd
import yfinance as yf

from backtest.loaders.base import (
    declared_currency_required,
    loader_cache_get,
    loader_cache_put,
    normalize_declared_quote_currency,
    validate_date_range,
    validate_ohlc,
)
from backtest.loaders.registry import register

logger = logging.getLogger(__name__)

_OHLCV_COLUMNS = ["open", "high", "low", "close", "volume"]
_COLUMN_RENAMES = {
    "Open": "open",
    "High": "high",
    "Low": "low",
    "Close": "close",
    "Volume": "volume",
    "open": "open",
    "high": "high",
    "low": "low",
    "close": "close",
    "volume": "volume",
}
_INTERVAL_MAP = {
    "1D": "1d",
    "1H": "1h",
    "4H": "1h",
    "4h": "1h",  # yfinance has no 4h; match project ``4H`` → ``1h``
    "1W": "1wk",
    "1w": "1wk",
    "1M": "1mo",
    # Minute tokens stay lowercase; do not fold ``1M`` (month) via ``.lower()``.
    "1m": "1m",
    "5m": "5m",
    "15m": "15m",
    "30m": "30m",
}


def _to_yfinance_symbol(code: str) -> str:
    """Convert project symbols into yfinance symbols.

    Args:
        code: Project symbol, for example ``AAPL.US``, ``700.HK``, or
            ``TD.TO``.

    Returns:
        yfinance-compatible symbol.
    """
    upper = code.strip().upper()
    if upper.endswith(".US"):
        # US class shares are hyphenated on Yahoo/yfinance (BRK-B): the dot
        # form returns empty data (live-verified), so map BRK.B.US -> BRK-B.
        return upper[:-3].replace(".", "-")
    if upper.endswith(".HK"):
        digits = upper[:-3]
        width = max(4, len(digits))
        return f"{digits.zfill(width)}.HK"
    # Crypto: BTC-USDT -> BTC-USD, ETH-USDT -> ETH-USD, etc.
    if upper.endswith("-USDT"):
        return upper[:-5] + "-USD"
    if upper.endswith("-USDC"):
        return upper[:-5] + "-USD"
    # India NSE/BSE (RELIANCE.NS, 500325.BO), Korea KRX (005930.KS,
    # 247540.KQ), Canada TSX/TSXV (TD.TO, PNG.V), and Vietnam HOSE (VIC.VN):
    # yfinance carries these suffixes as-is.
    return upper


def _to_yfinance_interval(interval: str) -> str:
    """Map project interval strings to yfinance interval strings.

    Args:
        interval: Backtest interval such as ``1D`` or ``5m``.

    Returns:
        yfinance interval string.
    """
    normalized = str(interval or "1D").strip()
    return _INTERVAL_MAP.get(normalized, normalized.lower())


def _to_yfinance_exclusive_end(end_date: str) -> str:
    """Convert the project-inclusive end date to yfinance's exclusive end."""
    return (pd.Timestamp(end_date).normalize() + pd.Timedelta(days=1)).strftime("%Y-%m-%d")


def _declared_currency(symbol: str) -> Optional[str]:
    """Return Yahoo's declared currency for ``symbol``, or ``None`` when absent.

    ``yf.Ticker(...).history_metadata`` carries the exchange's declared quote
    currency. Absence (or any probe failure) MUST NOT be treated as GBp. The
    LSE loader contract rejects a missing or non-GBP currency rather than
    allowing a USD line into static GBP accounting.
    """
    try:
        meta = yf.Ticker(symbol).history_metadata
    except Exception:  # noqa: BLE001 — a metadata probe failure is not data
        return None
    # yfinance>=1.x returns a lazy HistoryMetadata Mapping, not a dict.
    currency = meta.get("currency") if isinstance(meta, Mapping) else None
    return currency if isinstance(currency, str) and currency else None


def _download_history(
    tickers: Union[List[str], str],
    start_date: str,
    end_date: str,
    interval: str,
) -> pd.DataFrame:
    """Download raw historical data via yfinance.

    Args:
        tickers: One or more yfinance symbols.
        start_date: Inclusive start date string.
        end_date: End date string passed directly to yfinance.
        interval: yfinance interval string.

    Returns:
        Raw dataframe from ``yf.download``.
    """
    return yf.download(
        tickers,
        start=start_date,
        end=end_date,
        interval=interval,
        # Adjusted OHLC like every other loader on the chain (qfq caliber);
        # volume stays raw on both sides of the comparison.
        auto_adjust=True,
        progress=False,
    )


def _flatten_columns(frame: pd.DataFrame, symbol: str) -> pd.DataFrame:
    """Flatten any leftover multi-index columns after symbol selection.

    Args:
        frame: Price dataframe.
        symbol: yfinance symbol used for column cleanup.

    Returns:
        Dataframe with flat string columns.
    """
    if not isinstance(frame.columns, pd.MultiIndex):
        return frame

    cleaned_columns = []
    for column in frame.columns:
        pieces = [str(part) for part in column if str(part) and str(part).upper() != symbol.upper()]
        cleaned_columns.append(pieces[-1] if pieces else str(column[-1]))
    flattened = frame.copy()
    flattened.columns = cleaned_columns
    return flattened


def _extract_symbol_frame(frame: pd.DataFrame, symbol: str, total_symbols: int) -> pd.DataFrame:
    """Extract a single symbol slice from a raw yfinance dataframe.

    Args:
        frame: Raw dataframe returned by ``yf.download``.
        symbol: yfinance symbol to extract.
        total_symbols: Number of unique symbols requested.

    Returns:
        A single-symbol dataframe or an empty dataframe when unavailable.
    """
    if frame.empty:
        return pd.DataFrame()

    if not isinstance(frame.columns, pd.MultiIndex):
        if total_symbols == 1:
            return frame.copy()
        return pd.DataFrame()

    for level in range(frame.columns.nlevels):
        if symbol in frame.columns.get_level_values(level):
            selected = frame.xs(symbol, axis=1, level=level, drop_level=True)
            return _flatten_columns(selected.copy(), symbol)
    return pd.DataFrame()


def _normalize_frame(frame: pd.DataFrame, requested_interval: str) -> pd.DataFrame:
    """Normalize raw yfinance data into the backtest OHLCV schema.

    Args:
        frame: Raw or symbol-scoped yfinance dataframe.
        requested_interval: Original backtest interval.

    Returns:
        Normalized OHLCV dataframe indexed by ``trade_date``.
    """
    if frame.empty:
        return pd.DataFrame(columns=_OHLCV_COLUMNS)

    normalized = _flatten_columns(frame.copy(), "")
    normalized = normalized.rename(columns=_COLUMN_RENAMES)

    for column in _OHLCV_COLUMNS:
        if column not in normalized.columns:
            if column == "volume":
                normalized[column] = 0.0
            else:
                return pd.DataFrame(columns=_OHLCV_COLUMNS)

    normalized = normalized.loc[:, _OHLCV_COLUMNS].copy()
    normalized = normalized.apply(pd.to_numeric, errors="coerce")

    index = pd.DatetimeIndex(pd.to_datetime(normalized.index))
    if getattr(index, "tz", None) is not None:
        index = index.tz_localize(None)
    normalized.index = index
    normalized.index.name = "trade_date"
    normalized = normalized.sort_index()
    normalized["volume"] = normalized["volume"].fillna(0.0)
    normalized = normalized.dropna(subset=["open", "high", "low", "close"])
    normalized = validate_ohlc(normalized)

    # ``requested_interval`` reaches here with whatever case the caller used
    # (``_INTERVAL_MAP`` accepts both ``4H`` and ``4h``). A case-sensitive
    # check here let lowercase ``4h`` fetch hourly data via
    # ``_to_yfinance_interval`` but skip this resample, silently returning
    # native 1h bars mislabeled as 4H.
    if str(requested_interval).strip().upper() == "4H" and not normalized.empty:
        normalized = normalized.resample("4h").agg(
            {
                "open": "first",
                "high": "max",
                "low": "min",
                "close": "last",
                "volume": "sum",
            }
        )
        normalized = normalized.dropna(subset=["open", "high", "low", "close"])
        normalized.index.name = "trade_date"

    return normalized


@register
class DataLoader:
    """Fetch global-equity and crypto bars from Yahoo Finance via yfinance."""

    name = "yfinance"
    markets = {
        "us_equity", "hk_equity", "india_equity", "kr_equity", "ca_equity",
        "vietnam_equity", "uk_equity", "ar_equity", "br_equity", "crypto",
    }
    # yfinance volume is single shares for US/HK equities
    # (HKUDS/Vibe-Trading#1062; HK verified 2026-08-11, 00700.HK ratio 1.00
    # vs tencent/eastmoney). Crypto base-asset units stay undeclared.
    volume_units = {"us_equity": "shares", "hk_equity": "shares", "uk_equity": "shares"}
    requires_auth = False

    def is_available(self) -> bool:
        """Always available (free public data, no auth)."""
        return True

    def __init__(self) -> None:
        """Initialize the loader.

        yfinance is a free public-data wrapper and does not require credentials.
        """
        pass

    def fetch(
        self,
        codes: List[str],
        start_date: str,
        end_date: str,
        *,
        interval: str = "1D",
        fields: Optional[List[str]] = None,
    ) -> Dict[str, pd.DataFrame]:
        """Fetch OHLCV history keyed by the original project symbols.

        Args:
            codes: Project symbols such as ``AAPL.US``, ``700.HK``, and
                ``TD.TO``.
            start_date: Start date in ``YYYY-MM-DD`` format.
            end_date: End date in ``YYYY-MM-DD`` format.
            fields: Ignored for yfinance; included for interface compatibility.
            interval: Backtest interval such as ``1D`` or ``1H``.

        Returns:
            Mapping of input symbol to normalized OHLCV dataframe.
        """
        del fields
        if not codes:
            return {}
        validate_date_range(start_date, end_date)

        requested_interval = str(interval or "1D").strip()
        yf_interval = _to_yfinance_interval(requested_interval)
        yf_end_date = _to_yfinance_exclusive_end(end_date)

        symbol_groups: Dict[str, List[str]] = defaultdict(list)
        for code in codes:
            symbol_groups[_to_yfinance_symbol(code)].append(code)

        unique_symbols = list(symbol_groups.keys())
        results: Dict[str, pd.DataFrame] = {}

        # Serve cached symbols first so a fully-cached request skips the bulk
        # download entirely; only uncached symbols hit the network.
        pending: List[str] = []
        for symbol in unique_symbols:
            cached = loader_cache_get(
                source=self.name,
                symbol=symbol,
                timeframe=requested_interval,
                start_date=start_date,
                end_date=end_date,
                fields=None,
            )
            if cached is not None:
                for original_code in symbol_groups[symbol]:
                    results[original_code] = cached.copy()
            else:
                pending.append(symbol)

        if not pending:
            return results

        try:
            bulk_data = _download_history(pending, start_date, yf_end_date, yf_interval)
        except Exception as exc:
            logger.warning("yfinance bulk download failed for %s: %s", pending, exc)
            bulk_data = pd.DataFrame()

        for symbol in pending:
            try:
                symbol_frame = _extract_symbol_frame(bulk_data, symbol, len(pending))
                if symbol_frame.empty:
                    symbol_frame = _download_history(symbol, start_date, yf_end_date, yf_interval)

                normalized = _normalize_frame(symbol_frame, requested_interval)
                if normalized.empty:
                    logger.warning("yfinance returned no usable data for %s", symbol)
                    continue

                # uk_equity is one static GBP pool and ar_equity one ARS pool,
                # while LSE and BYMA each list lines in other currencies. The
                # suffix identifies the venue, never the currency, so read the
                # declared one (a metadata request, made only for these venues)
                # and reject a line outside the pool's unit.
                if declared_currency_required(symbol):
                    declared = _declared_currency(symbol)
                    normalized = normalize_declared_quote_currency(
                        normalized, symbol, declared
                    )

                loader_cache_put(
                    source=self.name,
                    symbol=symbol,
                    timeframe=requested_interval,
                    start_date=start_date,
                    end_date=end_date,
                    fields=None,
                    frame=normalized,
                )
                for original_code in symbol_groups[symbol]:
                    results[original_code] = normalized.copy()
            except Exception as exc:
                logger.warning("Failed to fetch data for %s: %s", symbol, exc)
                continue

        return results