"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
"""CN industry classification loader from Tushare ``index_member_all`` (SW2021).
Supplies the wide ``sector`` label frame the industry-neutralized CN value
alphas z-score against (issue #1789 follow-up #2): one label per symbol per
trading day, point-in-time via each membership row's ``in_date``/``out_date``.
PIT semantics: symbol ``s`` carries label ``L`` on date ``d`` iff a membership
row exists with ``in_date <= d`` and (``out_date`` is null OR ``d < out_date``)
— ``in_date`` inclusive, ``out_date`` exclusive. If overlapping rows ever claim
the same ``(s, d)``, the LATER ``in_date`` wins; on equal ``in_date`` the row
with the later (or null = current) ``out_date`` wins. Both rules are fixed so
the frame is deterministic regardless of API row order.
Taxonomy caveat: SW2021 classifications are applied retroactively to history —
a name reclassified in 2021 carries its 2021 industry back through the whole
window. This is the same class of caveat the csi300 bench already discloses
for PIT index membership, not a lookahead in the price/valuation data.
Endpoint contract (pinned by tests, verified live 2026-10-12):
``index_member_all`` accepts ONLY the documented ``l1_code``/``l2_code``/
``l3_code``/``ts_code``/``is_new`` filters — passing ``src`` (undocumented)
silently returns an empty 200-row-columns frame, so ``src`` is used for
``index_classify`` in the truncation fallback only. ``is_new`` defaults to
"Y" server-side (current members, no out_date rows), so the historical
half ("N") is fetched separately and concatenated — without it every label
would be today's classification and the PIT claim above false.
Truncation guard: ``index_member_all`` documents a 2000-row cap per call.
A live full-permission token returns the whole snapshot in one call (~5900
current + ~2000 historical rows), but a response of exactly 2000 rows is
treated as truncated and refetched per L1 industry (``index_classify`` then
one ``index_member_all(l1_code=..., is_new=...)`` call per industry).
Degradation: a provider failure after backoff warns and returns an all-None
frame aligned to ``index`` x ``symbols`` — the caller benches without the
sector tag (``requires_sector`` alphas SkipAlpha) instead of crashing. A
missing/placeholder ``TUSHARE_TOKEN`` raises ``RuntimeError`` like every
sibling loader; no label is ever invented.
"""
from __future__ import annotations
import logging
from collections.abc import Sequence
from typing import Any
import numpy as np
import pandas as pd
from backtest.loaders.tushare import (
TUSHARE_TOKEN_PLACEHOLDERS,
_call_with_backoff,
)
logger = logging.getLogger(__name__)
#: Shenwan classification levels ``index_member_all`` serves name columns for.
INDUSTRY_LEVELS: tuple[str, ...] = ("L1", "L2", "L3")
_LEVEL_NAME_COLUMN = {"L1": "l1_name", "L2": "l2_name", "L3": "l3_name"}
#: Documented per-call row cap of ``index_member_all``; a snapshot of exactly
#: this size is treated as truncated and triggers the per-L1 refetch.
_SNAPSHOT_ROW_CAP = 2000
#: Sentinel standing in for a null ``out_date`` (still a member). Inside the
#: ``datetime64[ns]`` range so int64 comparisons against trading days work.
_OPEN_ENDED = pd.Timestamp("2262-01-01")
def validate_industry_params(level: str, src: str) -> str:
"""Validate the requested classification level and source.
Args:
level: Shenwan level, one of :data:`INDUSTRY_LEVELS`.
src: Classification source, e.g. ``"SW2021"``; must be non-empty.
Returns:
The ``index_member_all`` name column serving ``level``.
Raises:
ValueError: If ``level`` is unknown or ``src`` is empty.
"""
if level not in INDUSTRY_LEVELS:
raise ValueError(
f"unsupported industry level {level!r}; expected one of {list(INDUSTRY_LEVELS)}"
)
if not isinstance(src, str) or not src.strip():
raise ValueError(f"src must be a non-empty string, got {src!r}")
return _LEVEL_NAME_COLUMN[level]
def _build_api() -> Any:
"""Build a Tushare pro API client from the configured token.
Raises:
RuntimeError: If ``TUSHARE_TOKEN`` is unset or a placeholder.
"""
import tushare as ts
from src.config.accessor import get_env_config
token = get_env_config().data.tushare_token.strip()
if token in TUSHARE_TOKEN_PLACEHOLDERS:
raise RuntimeError(
"TUSHARE_TOKEN not configured; required for industry classification"
)
return ts.pro_api(token)
def _fetch_membership_part(api: Any, src: str, is_new: str) -> pd.DataFrame | None:
"""Fetch one ``is_new`` half of the snapshot, refetching per L1 if truncated."""
frame = _call_with_backoff(api.index_member_all, is_new=is_new)
if frame is None or len(frame) != _SNAPSHOT_ROW_CAP:
return frame
logger.warning(
"index_member_all(is_new=%s) returned exactly %d rows (documented per-call cap); "
"refetching per L1 industry",
is_new,
_SNAPSHOT_ROW_CAP,
)
classify = _call_with_backoff(api.index_classify, level="L1", src=src)
if classify is None or classify.empty or "index_code" not in classify.columns:
raise RuntimeError(
"index_classify served no L1 codes for the per-industry refetch"
)
parts = []
for code in classify["index_code"].astype(str):
part = _call_with_backoff(api.index_member_all, l1_code=code, is_new=is_new)
if part is not None and not part.empty:
parts.append(part)
if not parts:
return None
return pd.concat(parts, ignore_index=True)
def _fetch_membership(api: Any, src: str) -> pd.DataFrame | None:
"""Fetch the full PIT membership snapshot (current "Y" + historical "N").
Args:
api: Tushare pro API (real or injected fake).
src: Classification source — passed to ``index_classify`` only;
``index_member_all`` rejects an undocumented ``src`` with a
silently empty response (see the module docstring).
Returns:
The concatenated membership rows, or None if nothing was served.
Raises:
Exception: The endpoint's own exception after backoff — the caller
decides whether that degrades or propagates.
"""
parts = []
for is_new in ("Y", "N"):
part = _fetch_membership_part(api, src, is_new)
if part is not None and not part.empty:
parts.append(part)
if not parts:
return None
return pd.concat(parts, ignore_index=True)
def load_industry_frame_with_coverage(
*,
symbols: Sequence[str],
start: str,
end: str,
index: pd.DatetimeIndex,
api: Any | None = None,
level: str = "L1",
src: str = "SW2021",
) -> tuple[pd.DataFrame, float]:
"""Load PIT industry labels as a wide frame plus a coverage fraction.
Args:
symbols: Panel symbols; only names present in the membership data get
a label, the rest stay NaN.
start: Period start (YYYY-MM-DD). Accepted for loader-contract
symmetry with ``valuation_loader``; PIT evaluation uses the actual
``index`` dates, so an index reaching outside the window is still
labelled correctly.
end: Period end (YYYY-MM-DD), same contract note as ``start``.
index: Price-panel trading-day index the frame aligns to.
api: Optional pre-built Tushare pro API (tests inject fakes here).
level: Shenwan level, one of :data:`INDUSTRY_LEVELS`.
src: Classification source, e.g. ``"SW2021"``.
Returns:
``(frame, coverage)`` — ``frame`` is ``dates x symbols`` with the
industry NAME (str) where a membership row covers the date and NaN
where none does (never an invented label); ``coverage`` is the
fraction of ``symbols`` with at least one label anywhere in the frame.
Raises:
ValueError: If ``level`` or ``src`` is invalid.
RuntimeError: If no API is injected and ``TUSHARE_TOKEN`` is missing.
"""
name_column = validate_industry_params(level, src)
columns = list(symbols)
empty = pd.DataFrame(np.nan, index=index, columns=columns, dtype=object)
if not columns or len(index) == 0:
return empty, 0.0
logger.debug("industry frame requested for %s..%s (%d symbols)", start, end, len(columns))
if api is None:
api = _build_api()
try:
membership = _fetch_membership(api, src)
except Exception as exc: # noqa: BLE001 - degrade, never crash the caller
logger.warning(
"index_member_all industry fetch failed (%s); returning unmapped frame", exc
)
return empty, 0.0
if (
membership is None
or membership.empty
or "ts_code" not in membership.columns
or name_column not in membership.columns
# Without the PIT interval columns no membership can be established —
# degrade rather than guess a mapping from names alone.
or "in_date" not in membership.columns
or "out_date" not in membership.columns
):
logger.warning(
"index_member_all served no usable membership rows (src=%s, level=%s)",
src,
level,
)
return empty, 0.0
rows = membership[membership["ts_code"].isin(columns)].copy()
in_dt = pd.to_datetime(rows.get("in_date"), format="%Y%m%d", errors="coerce")
out_dt = pd.to_datetime(rows.get("out_date"), format="%Y%m%d", errors="coerce")
labels = rows[name_column]
# A row with no in_date can never satisfy in_date <= d, and a row with no
# label has nothing to serve — dropped, never defaulted.
usable = in_dt.notna() & labels.notna() & (labels.astype(str).str.strip() != "")
rows, in_dt, labels = rows[usable], in_dt[usable], labels[usable]
out_dt = out_dt[usable].fillna(_OPEN_ENDED)
if rows.empty:
return empty, 0.0
# Later in_date wins, then later/open out_date: a fixed sort makes the
# per-date winner independent of API row order (dict(zip) keeps the last).
order = np.lexsort((out_dt.to_numpy(dtype="int64"), in_dt.to_numpy(dtype="int64")))
codes = rows["ts_code"].to_numpy()[order]
ordered_labels = labels.to_numpy()[order]
in_ns = in_dt.to_numpy(dtype="int64")[order]
out_ns = out_dt.to_numpy(dtype="int64")[order]
position = {symbol: j for j, symbol in enumerate(columns)}
values = np.full((len(index), len(columns)), np.nan, dtype=object)
for i, day_ns in enumerate(index.asi8):
applicable = (in_ns <= day_ns) & (day_ns < out_ns)
if not applicable.any():
continue
for symbol, label in dict(zip(codes[applicable], ordered_labels[applicable])).items():
values[i, position[symbol]] = label
frame = pd.DataFrame(values, index=index, columns=columns)
coverage = float(frame.notna().any(axis=0).mean())
if coverage < 1.0:
logger.warning(
"industry coverage %.1f%% of %d requested symbols (src=%s, level=%s)",
coverage * 100,
len(columns),
src,
level,
)
return frame, coverage
def load_industry_frame(
*,
symbols: Sequence[str],
start: str,
end: str,
index: pd.DatetimeIndex,
api: Any | None = None,
level: str = "L1",
src: str = "SW2021",
) -> pd.DataFrame:
"""Load PIT industry labels as a wide ``dates x symbols`` frame.
Thin wrapper over :func:`load_industry_frame_with_coverage` for callers
that do not need the coverage fraction; same contract, same degradation.
"""
frame, _ = load_industry_frame_with_coverage(
symbols=symbols, start=start, end=end, index=index, api=api, level=level, src=src
)
return frame