"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""yfinance-backed loader for global equity and crypto OHLCV data."""
from __future__ import annotations
import logging
from collections import defaultdict
from collections.abc import Mapping
from typing import Dict, List, Optional, Union
import pandas as pd
import yfinance as yf
from backtest.loaders.base import (
declared_currency_required,
loader_cache_get,
loader_cache_put,
normalize_declared_quote_currency,
validate_date_range,
validate_ohlc,
)
from backtest.loaders.registry import register
logger = logging.getLogger(__name__)
_OHLCV_COLUMNS = ["open", "high", "low", "close", "volume"]
_COLUMN_RENAMES = {
"Open": "open",
"High": "high",
"Low": "low",
"Close": "close",
"Volume": "volume",
"open": "open",
"high": "high",
"low": "low",
"close": "close",
"volume": "volume",
}
_INTERVAL_MAP = {
"1D": "1d",
"1H": "1h",
"4H": "1h",
"4h": "1h", # yfinance has no 4h; match project ``4H`` → ``1h``
"1W": "1wk",
"1w": "1wk",
"1M": "1mo",
# Minute tokens stay lowercase; do not fold ``1M`` (month) via ``.lower()``.
"1m": "1m",
"5m": "5m",
"15m": "15m",
"30m": "30m",
}
def _to_yfinance_symbol(code: str) -> str:
"""Convert project symbols into yfinance symbols.
Args:
code: Project symbol, for example ``AAPL.US``, ``700.HK``, or
``TD.TO``.
Returns:
yfinance-compatible symbol.
"""
upper = code.strip().upper()
if upper.endswith(".US"):
# US class shares are hyphenated on Yahoo/yfinance (BRK-B): the dot
# form returns empty data (live-verified), so map BRK.B.US -> BRK-B.
return upper[:-3].replace(".", "-")
if upper.endswith(".HK"):
digits = upper[:-3]
width = max(4, len(digits))
return f"{digits.zfill(width)}.HK"
# Crypto: BTC-USDT -> BTC-USD, ETH-USDT -> ETH-USD, etc.
if upper.endswith("-USDT"):
return upper[:-5] + "-USD"
if upper.endswith("-USDC"):
return upper[:-5] + "-USD"
# India NSE/BSE (RELIANCE.NS, 500325.BO), Korea KRX (005930.KS,
# 247540.KQ), Canada TSX/TSXV (TD.TO, PNG.V), and Vietnam HOSE (VIC.VN):
# yfinance carries these suffixes as-is.
return upper
def _to_yfinance_interval(interval: str) -> str:
"""Map project interval strings to yfinance interval strings.
Args:
interval: Backtest interval such as ``1D`` or ``5m``.
Returns:
yfinance interval string.
"""
normalized = str(interval or "1D").strip()
return _INTERVAL_MAP.get(normalized, normalized.lower())
def _to_yfinance_exclusive_end(end_date: str) -> str:
"""Convert the project-inclusive end date to yfinance's exclusive end."""
return (pd.Timestamp(end_date).normalize() + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
def _declared_currency(symbol: str) -> Optional[str]:
"""Return Yahoo's declared currency for ``symbol``, or ``None`` when absent.
``yf.Ticker(...).history_metadata`` carries the exchange's declared quote
currency. Absence (or any probe failure) MUST NOT be treated as GBp. The
LSE loader contract rejects a missing or non-GBP currency rather than
allowing a USD line into static GBP accounting.
"""
try:
meta = yf.Ticker(symbol).history_metadata
except Exception: # noqa: BLE001 — a metadata probe failure is not data
return None
# yfinance>=1.x returns a lazy HistoryMetadata Mapping, not a dict.
currency = meta.get("currency") if isinstance(meta, Mapping) else None
return currency if isinstance(currency, str) and currency else None
def _download_history(
tickers: Union[List[str], str],
start_date: str,
end_date: str,
interval: str,
) -> pd.DataFrame:
"""Download raw historical data via yfinance.
Args:
tickers: One or more yfinance symbols.
start_date: Inclusive start date string.
end_date: End date string passed directly to yfinance.
interval: yfinance interval string.
Returns:
Raw dataframe from ``yf.download``.
"""
return yf.download(
tickers,
start=start_date,
end=end_date,
interval=interval,
# Adjusted OHLC like every other loader on the chain (qfq caliber);
# volume stays raw on both sides of the comparison.
auto_adjust=True,
progress=False,
)
def _flatten_columns(frame: pd.DataFrame, symbol: str) -> pd.DataFrame:
"""Flatten any leftover multi-index columns after symbol selection.
Args:
frame: Price dataframe.
symbol: yfinance symbol used for column cleanup.
Returns:
Dataframe with flat string columns.
"""
if not isinstance(frame.columns, pd.MultiIndex):
return frame
cleaned_columns = []
for column in frame.columns:
pieces = [str(part) for part in column if str(part) and str(part).upper() != symbol.upper()]
cleaned_columns.append(pieces[-1] if pieces else str(column[-1]))
flattened = frame.copy()
flattened.columns = cleaned_columns
return flattened
def _extract_symbol_frame(frame: pd.DataFrame, symbol: str, total_symbols: int) -> pd.DataFrame:
"""Extract a single symbol slice from a raw yfinance dataframe.
Args:
frame: Raw dataframe returned by ``yf.download``.
symbol: yfinance symbol to extract.
total_symbols: Number of unique symbols requested.
Returns:
A single-symbol dataframe or an empty dataframe when unavailable.
"""
if frame.empty:
return pd.DataFrame()
if not isinstance(frame.columns, pd.MultiIndex):
if total_symbols == 1:
return frame.copy()
return pd.DataFrame()
for level in range(frame.columns.nlevels):
if symbol in frame.columns.get_level_values(level):
selected = frame.xs(symbol, axis=1, level=level, drop_level=True)
return _flatten_columns(selected.copy(), symbol)
return pd.DataFrame()
def _normalize_frame(frame: pd.DataFrame, requested_interval: str) -> pd.DataFrame:
"""Normalize raw yfinance data into the backtest OHLCV schema.
Args:
frame: Raw or symbol-scoped yfinance dataframe.
requested_interval: Original backtest interval.
Returns:
Normalized OHLCV dataframe indexed by ``trade_date``.
"""
if frame.empty:
return pd.DataFrame(columns=_OHLCV_COLUMNS)
normalized = _flatten_columns(frame.copy(), "")
normalized = normalized.rename(columns=_COLUMN_RENAMES)
for column in _OHLCV_COLUMNS:
if column not in normalized.columns:
if column == "volume":
normalized[column] = 0.0
else:
return pd.DataFrame(columns=_OHLCV_COLUMNS)
normalized = normalized.loc[:, _OHLCV_COLUMNS].copy()
normalized = normalized.apply(pd.to_numeric, errors="coerce")
index = pd.DatetimeIndex(pd.to_datetime(normalized.index))
if getattr(index, "tz", None) is not None:
index = index.tz_localize(None)
normalized.index = index
normalized.index.name = "trade_date"
normalized = normalized.sort_index()
normalized["volume"] = normalized["volume"].fillna(0.0)
normalized = normalized.dropna(subset=["open", "high", "low", "close"])
normalized = validate_ohlc(normalized)
# ``requested_interval`` reaches here with whatever case the caller used
# (``_INTERVAL_MAP`` accepts both ``4H`` and ``4h``). A case-sensitive
# check here let lowercase ``4h`` fetch hourly data via
# ``_to_yfinance_interval`` but skip this resample, silently returning
# native 1h bars mislabeled as 4H.
if str(requested_interval).strip().upper() == "4H" and not normalized.empty:
normalized = normalized.resample("4h").agg(
{
"open": "first",
"high": "max",
"low": "min",
"close": "last",
"volume": "sum",
}
)
normalized = normalized.dropna(subset=["open", "high", "low", "close"])
normalized.index.name = "trade_date"
return normalized
@register
class DataLoader:
"""Fetch global-equity and crypto bars from Yahoo Finance via yfinance."""
name = "yfinance"
markets = {
"us_equity", "hk_equity", "india_equity", "kr_equity", "ca_equity",
"vietnam_equity", "uk_equity", "ar_equity", "br_equity", "crypto",
}
# yfinance volume is single shares for US/HK equities
# (HKUDS/Vibe-Trading#1062; HK verified 2026-08-11, 00700.HK ratio 1.00
# vs tencent/eastmoney). Crypto base-asset units stay undeclared.
volume_units = {"us_equity": "shares", "hk_equity": "shares", "uk_equity": "shares"}
requires_auth = False
def is_available(self) -> bool:
"""Always available (free public data, no auth)."""
return True
def __init__(self) -> None:
"""Initialize the loader.
yfinance is a free public-data wrapper and does not require credentials.
"""
pass
def fetch(
self,
codes: List[str],
start_date: str,
end_date: str,
*,
interval: str = "1D",
fields: Optional[List[str]] = None,
) -> Dict[str, pd.DataFrame]:
"""Fetch OHLCV history keyed by the original project symbols.
Args:
codes: Project symbols such as ``AAPL.US``, ``700.HK``, and
``TD.TO``.
start_date: Start date in ``YYYY-MM-DD`` format.
end_date: End date in ``YYYY-MM-DD`` format.
fields: Ignored for yfinance; included for interface compatibility.
interval: Backtest interval such as ``1D`` or ``1H``.
Returns:
Mapping of input symbol to normalized OHLCV dataframe.
"""
del fields
if not codes:
return {}
validate_date_range(start_date, end_date)
requested_interval = str(interval or "1D").strip()
yf_interval = _to_yfinance_interval(requested_interval)
yf_end_date = _to_yfinance_exclusive_end(end_date)
symbol_groups: Dict[str, List[str]] = defaultdict(list)
for code in codes:
symbol_groups[_to_yfinance_symbol(code)].append(code)
unique_symbols = list(symbol_groups.keys())
results: Dict[str, pd.DataFrame] = {}
# Serve cached symbols first so a fully-cached request skips the bulk
# download entirely; only uncached symbols hit the network.
pending: List[str] = []
for symbol in unique_symbols:
cached = loader_cache_get(
source=self.name,
symbol=symbol,
timeframe=requested_interval,
start_date=start_date,
end_date=end_date,
fields=None,
)
if cached is not None:
for original_code in symbol_groups[symbol]:
results[original_code] = cached.copy()
else:
pending.append(symbol)
if not pending:
return results
try:
bulk_data = _download_history(pending, start_date, yf_end_date, yf_interval)
except Exception as exc:
logger.warning("yfinance bulk download failed for %s: %s", pending, exc)
bulk_data = pd.DataFrame()
for symbol in pending:
try:
symbol_frame = _extract_symbol_frame(bulk_data, symbol, len(pending))
if symbol_frame.empty:
symbol_frame = _download_history(symbol, start_date, yf_end_date, yf_interval)
normalized = _normalize_frame(symbol_frame, requested_interval)
if normalized.empty:
logger.warning("yfinance returned no usable data for %s", symbol)
continue
# uk_equity is one static GBP pool and ar_equity one ARS pool,
# while LSE and BYMA each list lines in other currencies. The
# suffix identifies the venue, never the currency, so read the
# declared one (a metadata request, made only for these venues)
# and reject a line outside the pool's unit.
if declared_currency_required(symbol):
declared = _declared_currency(symbol)
normalized = normalize_declared_quote_currency(
normalized, symbol, declared
)
loader_cache_put(
source=self.name,
symbol=symbol,
timeframe=requested_interval,
start_date=start_date,
end_date=end_date,
fields=None,
frame=normalized,
)
for original_code in symbol_groups[symbol]:
results[original_code] = normalized.copy()
except Exception as exc:
logger.warning("Failed to fetch data for %s: %s", symbol, exc)
continue
return results