Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(loaders): refuse identity relabels and share the frame caliber everywhere

Yufeng He91eeab4c252e
diff --git a/agent/backtest/loaders/additive_conversion.py b/agent/backtest/loaders/additive_conversion.py
index 85d3797b4..716dc7f5c 100644
--- a/agent/backtest/loaders/additive_conversion.py
+++ b/agent/backtest/loaders/additive_conversion.py
@@ -9,9 +9,20 @@ against +100.43% multiplicative), and old levels can even go negative.
The offsets are enough to recover the multiplicative series: at each step the
factor is ``(prev_raw_close - dividend) / prev_raw_close``, and the cumulative
-product scales raw bars onto the 前复权 ratio basis anchored at the last bar,
-the same convention ``cn_adjust.apply_qfq`` uses for Tushare factors. Any
-inconsistency (misaligned calendars, offsets that are not plateaus, a
+product scales raw bars onto the 前复权 ratio basis, the same convention
+``cn_adjust.apply_qfq`` uses for Tushare factors. Two consequences of the
+construction to know before reading levels off the result:
+
+- The series is anchored at the window's END (bars after the last in-window
+ action keep ratio 1), so absolute levels depend on the requested range;
+ returns are unaffected. Windows that cross a 送转 (bonus-share) event have
+ offsets that drift with the price level by construction, and this module
+ refuses them rather than converting on a wrong basis.
+- A window with no corporate action inside it has nothing to convert; the
+ caller keeps today's series and its static stamp instead of an identity
+ relabel.
+
+Any inconsistency (misaligned calendars, offsets that are not plateaus, a
non-positive or oversized dividend) returns ``None`` so the caller keeps the
additive series, stamp and warning instead of shipping a bad conversion.
"""
@@ -25,16 +36,20 @@ import pandas as pd
# one corporate-action segment into two.
_OFFSET_TOL = 1e-6
-# Adjacent plateaus closer than this are the same segment: one bar's rounding
-# noise must not mint a fake dividend (a 2016 first bar wobbles 0.02).
-_PLATEAU_MERGE_TOL = 0.05
+# A one-bar plateau inside (or at the edge of) a longer level is a quote
+# wobble, not a dividend: fold it only when the neighboring level(s) match
+# within this tolerance. Anything else that short cannot be classified and
+# refuses the window (a wrong series with a clean stamp is the worst outcome).
+_WOBBLE_TOL = 0.05
def _plateau_spans(offset: pd.Series) -> list[tuple[int, int, float]] | None:
"""Group the offset series into [start, end) spans of one constant value.
- Returns None when the offsets vary bar to bar (not plateaus), which means
- the series is not dividend-additive and there is nothing to convert.
+ One-bar spans are folded into a matching neighbor within ``_WOBBLE_TOL``
+ (quote wobble); a one-bar span that cannot be classified that way returns
+ None, since dropping a step we cannot classify would mint a wrong series
+ with a clean stamp.
"""
spans: list[tuple[int, int, float]] = []
start = 0
@@ -46,20 +61,50 @@ def _plateau_spans(offset: pd.Series) -> list[tuple[int, int, float]] | None:
start = i
current = value
spans.append((start, len(offset), current))
- # Fold sub-noise steps back into their segment: a boundary whose offset
- # moves by less than a few cents is a rounding artifact, not an action.
- merged: list[tuple[int, int, float]] = []
- for span in spans:
- if merged and abs(span[2] - merged[-1][2]) < _PLATEAU_MERGE_TOL:
- merged[-1] = (merged[-1][0], span[1], merged[-1][2])
- else:
- merged.append(span)
- return merged
-
-
-def convert_additive_to_multiplicative(
- raw: pd.DataFrame, additive: pd.DataFrame
-) -> pd.DataFrame | None:
+
+ # Fold one-bar wobbles into the level they deviate from.
+ while True:
+ foldable_idx = None
+ for i, (span_start, span_end, value) in enumerate(spans):
+ if span_end - span_start != 1:
+ continue
+ prev_value = spans[i - 1][2] if i > 0 else None
+ next_value = spans[i + 1][2] if i + 1 < len(spans) else None
+ # Interior wobble: both neighbors agree with each other.
+ if (
+ prev_value is not None
+ and next_value is not None
+ and abs(prev_value - next_value) <= _WOBBLE_TOL
+ and abs(value - prev_value) <= _WOBBLE_TOL
+ ):
+ foldable_idx = i
+ break
+ # Edge wobble: the lone bar at either end matches its neighbor.
+ if i == 0 and next_value is not None and abs(value - next_value) <= _WOBBLE_TOL:
+ foldable_idx = i
+ break
+ if i == len(spans) - 1 and prev_value is not None and abs(value - prev_value) <= _WOBBLE_TOL:
+ foldable_idx = i
+ break
+ if foldable_idx is None:
+ # Any one-bar plateau left cannot be classified as noise; refuse
+ # rather than guess.
+ if any(end - start == 1 for start, end, _ in spans):
+ return None
+ break
+ del spans[foldable_idx]
+ # Re-join spans split by the wobble (same level now adjacent).
+ joined: list[tuple[int, int, float]] = []
+ for span in spans:
+ if joined and abs(span[2] - joined[-1][2]) <= _OFFSET_TOL:
+ joined[-1] = (joined[-1][0], span[1], joined[-1][2])
+ else:
+ joined.append(span)
+ spans = joined
+ return spans
+
+
+def convert_additive_to_multiplicative(raw: pd.DataFrame, additive: pd.DataFrame) -> pd.DataFrame | None:
"""Convert an additive qfq frame to the multiplicative convention.
Args:
@@ -69,7 +114,8 @@ def convert_additive_to_multiplicative(
Returns:
The converted frame (OHLC scaled by the cumulative ratio, ``volume``
divided by it, other columns untouched), or ``None`` when the inputs
- cannot support a trustworthy conversion.
+ cannot support a trustworthy conversion — including a window with no
+ corporate action in it, which stays as served today.
"""
if raw is None or additive is None or raw.empty or additive.empty:
return None
@@ -84,7 +130,7 @@ def convert_additive_to_multiplicative(
offset = joined["close_adj"] - joined["close_raw"]
spans = _plateau_spans(offset)
- if not spans:
+ if spans is None:
return None
# A genuine additive series steps a handful of times over hundreds of
# bars (the owner measured five offsets over 500 bars). One-bar plateaus
@@ -111,6 +157,11 @@ def convert_additive_to_multiplicative(
return None
factors.append((end_k, (prev_raw_close - dividend) / prev_raw_close))
+ # No in-window corporate action: there is nothing to convert, and an
+ # identity relabel would only misstate the stamp.
+ if not factors:
+ return None
+
# Cumulative ratio per bar, anchored at the last bar (前复权): bars after
# the last corporate action keep ratio 1, and each step multiplies
# everything before it by that action's factor.
diff --git a/agent/backtest/loaders/akshare_loader.py b/agent/backtest/loaders/akshare_loader.py
index df244e61d..56c55a7b2 100644
--- a/agent/backtest/loaders/akshare_loader.py
+++ b/agent/backtest/loaders/akshare_loader.py
@@ -34,9 +34,7 @@ _DAILY_ONLY_ALIASES = frozenset({"1d", "d", "day", "daily"})
def _require_daily_interval(interval: str, market: str) -> None:
if str(interval).strip().lower() not in _DAILY_ONLY_ALIASES:
- raise ValueError(
- f"Unsupported interval {interval!r}; akshare {market} supports daily bars only"
- )
+ raise ValueError(f"Unsupported interval {interval!r}; akshare {market} supports daily bars only")
def _is_a_share(code: str) -> bool:
@@ -67,9 +65,6 @@ def _sina_contract(code: str) -> str:
_CN_FUTURES_MAIN_RE = re.compile(r"^[A-Z]{1,2}0$")
-
-
-
def _is_forex(code: str) -> bool:
"""Detect forex pairs by matching against AKShare's symbol_market_map.
@@ -103,6 +98,7 @@ class DataLoader:
"""Available if akshare is installed."""
try:
import akshare # noqa: F401
+
return True
except ImportError:
return False
@@ -163,7 +159,8 @@ class DataLoader:
except Exception as exc: # noqa: BLE001 - degrade to additive
logger.warning(
"akshare raw companion fetch failed for %s, serving additive: %s",
- code, exc,
+ code,
+ exc,
)
converted = convert_additive_to_multiplicative(raw_df, df)
if converted is not None:
@@ -180,7 +177,11 @@ class DataLoader:
return result
def _fetch_one(
- self, code: str, start_date: str, end_date: str, interval: str,
+ self,
+ code: str,
+ start_date: str,
+ end_date: str,
+ interval: str,
) -> Optional[pd.DataFrame]:
"""Fetch a single symbol."""
import akshare as ak
@@ -208,23 +209,25 @@ class DataLoader:
# Returning None hands the symbol to the next link in the chain;
# letting it reach the A-share default below priced a USD-quoted
# global contract off ``stock_zh_a_hist`` without erroring (#1395).
- logger.warning(
- "akshare serves Chinese futures only; %s has no akshare source", code
- )
+ logger.warning("akshare serves Chinese futures only; %s has no akshare source", code)
return None
# Default: try A-share
return self._fetch_a_share(ak, code, start_date, end_date, interval)
def _fetch_a_share(
- self, ak, code: str, start_date: str, end_date: str, interval: str,
+ self,
+ ak,
+ code: str,
+ start_date: str,
+ end_date: str,
+ interval: str,
) -> Optional[pd.DataFrame]:
"""Fetch A-share via stock_zh_a_hist."""
symbol = code.split(".")[0]
period = _INTERVAL_MAP_DAILY.get(interval)
if period is None:
raise ValueError(
- f"Unsupported interval {interval!r}; akshare a-share supports "
- f"{sorted(_INTERVAL_MAP_DAILY)}"
+ f"Unsupported interval {interval!r}; akshare a-share supports {sorted(_INTERVAL_MAP_DAILY)}"
)
sd = start_date.replace("-", "")
ed = end_date.replace("-", "")
@@ -240,7 +243,11 @@ class DataLoader:
return self._normalize(df, date_col="日期")
def _fetch_a_share_raw(
- self, code: str, start_date: str, end_date: str, interval: str,
+ self,
+ code: str,
+ start_date: str,
+ end_date: str,
+ interval: str,
) -> Optional[pd.DataFrame]:
"""Fetch the raw (unadjusted) companion of ``_fetch_a_share`` (#1541)."""
import akshare as ak
@@ -249,8 +256,7 @@ class DataLoader:
period = _INTERVAL_MAP_DAILY.get(interval)
if period is None:
raise ValueError(
- f"Unsupported interval {interval!r}; akshare a-share supports "
- f"{sorted(_INTERVAL_MAP_DAILY)}"
+ f"Unsupported interval {interval!r}; akshare a-share supports {sorted(_INTERVAL_MAP_DAILY)}"
)
df = ak.stock_zh_a_hist(
symbol=symbol,
@@ -309,21 +315,21 @@ class DataLoader:
df = ak.forex_hist_em(symbol=symbol)
if df is None or df.empty:
return None
- df = df.rename(columns={
- "日期": "trade_date",
- "今开": "open",
- "最新价": "close",
- "最高": "high",
- "最低": "low",
- })
+ df = df.rename(
+ columns={
+ "日期": "trade_date",
+ "今开": "open",
+ "最新价": "close",
+ "最高": "high",
+ "最低": "low",
+ }
+ )
df["trade_date"] = pd.to_datetime(df["trade_date"])
df = df.set_index("trade_date").sort_index()
df["volume"] = 0.0
for col in ("open", "high", "low", "close"):
df[col] = pd.to_numeric(df[col], errors="coerce")
- df = df[["open", "high", "low", "close", "volume"]].dropna(
- subset=["open", "high", "low", "close"]
- )
+ df = df[["open", "high", "low", "close", "volume"]].dropna(subset=["open", "high", "low", "close"])
return df.loc[start_date:end_date]
def _fetch_hk(self, ak, code: str, start_date: str, end_date: str) -> Optional[pd.DataFrame]:
@@ -341,7 +347,11 @@ class DataLoader:
return self._normalize(df, date_col="日期")
def _fetch_china_futures(
- self, ak, code: str, start_date: str, end_date: str,
+ self,
+ ak,
+ code: str,
+ start_date: str,
+ end_date: str,
) -> Optional[pd.DataFrame]:
"""Fetch a Chinese futures contract from Sina's token-free endpoints.
@@ -376,10 +386,14 @@ class DataLoader:
)
if raw is None or raw.empty:
return None
- raw = raw.rename(columns={
- "开盘价": "开盘", "最高价": "最高",
- "最低价": "最低", "收盘价": "收盘",
- })
+ raw = raw.rename(
+ columns={
+ "开盘价": "开盘",
+ "最高价": "最高",
+ "最低价": "最低",
+ "收盘价": "收盘",
+ }
+ )
return self._normalize(raw, date_col="日期")
raw = ak.futures_zh_daily_sina(symbol=symbol)
@@ -395,7 +409,7 @@ class DataLoader:
df = self._normalize(raw, date_col="date")
# The dated endpoint ignores the window, so slice it here; without this
# a one-month request came back with the contract's entire history.
- return df.loc[str(start_date):str(end_date)]
+ return df.loc[str(start_date) : str(end_date)]
@staticmethod
def _normalize(df: pd.DataFrame, date_col: str = "日期") -> pd.DataFrame:
@@ -405,7 +419,14 @@ class DataLoader:
AKShare English column names: date, open, high, low, close, volume
"""
col_map_cn = {"开盘": "open", "最高": "high", "最低": "low", "收盘": "close", "成交量": "volume"}
- col_map_en = {"date": "trade_date", "open": "open", "high": "high", "low": "low", "close": "close", "volume": "volume"}
+ col_map_en = {
+ "date": "trade_date",
+ "open": "open",
+ "high": "high",
+ "low": "low",
+ "close": "close",
+ "volume": "volume",
+ }
if date_col in df.columns:
df = df.rename(columns={date_col: "trade_date"})
diff --git a/agent/backtest/loaders/eastmoney_loader.py b/agent/backtest/loaders/eastmoney_loader.py
index e090227a7..71699e20b 100644
--- a/agent/backtest/loaders/eastmoney_loader.py
+++ b/agent/backtest/loaders/eastmoney_loader.py
@@ -21,6 +21,7 @@ from typing import Dict, List, Optional
import pandas as pd
from backtest.loaders import eastmoney_client
+from backtest.loaders._symbol_utils import _is_etf_listed
from backtest.loaders.additive_conversion import convert_additive_to_multiplicative
from backtest.loaders.base import cached_loader_fetch, validate_date_range
from backtest.loaders.registry import register
@@ -106,11 +107,9 @@ class DataLoader:
start_date=start_date,
end_date=end_date,
fields=None,
- fetch=lambda code=code: self._fetch_one(
- code, start_date, end_date, interval
- ),
+ fetch=lambda code=code: self._fetch_one(code, start_date, end_date, interval),
)
- if df is not None and not df.empty and _is_a_share(code):
+ if df is not None and not df.empty and _is_a_share(code) and not _is_etf_listed(code):
# #1541: fqt=1 is dividend-additive and cannot serve
# returns; convert to the multiplicative convention when
# the fqt=0 companion supports it.
@@ -123,14 +122,13 @@ class DataLoader:
start_date=start_date,
end_date=end_date,
fields=["raw"],
- fetch=lambda code=code: self._fetch_one(
- code, start_date, end_date, interval, fqt=0
- ),
+ fetch=lambda code=code: self._fetch_one(code, start_date, end_date, interval, fqt=0),
)
except Exception as exc: # noqa: BLE001 - degrade to additive
logger.warning(
"eastmoney fqt=0 companion fetch failed for %s, serving additive: %s",
- code, exc,
+ code,
+ exc,
)
converted = convert_additive_to_multiplicative(raw_df, df)
if converted is not None:
@@ -147,7 +145,11 @@ class DataLoader:
return result
def _fetch_one(
- self, code: str, start_date: str, end_date: str, interval: str,
+ self,
+ code: str,
+ start_date: str,
+ end_date: str,
+ interval: str,
fqt: int = 1,
) -> Optional[pd.DataFrame]:
"""Resolve one symbol and build its OHLCV frame, or ``None`` on a miss.
diff --git a/agent/backtest/loaders/registry.py b/agent/backtest/loaders/registry.py
index f0057dda2..f3197e28a 100644
--- a/agent/backtest/loaders/registry.py
+++ b/agent/backtest/loaders/registry.py
@@ -326,9 +326,7 @@ _A_SHARE_INDEX_CODE = re.compile(r"^(?:000\d{3}\.SH|399\d{3}\.SZ|899\d{3}\.BJ)$"
#: "split_dividend_additive" does participate — it is the whole point of the
#: caliber: an additive level and a multiplicative one are two different scales,
#: so a basket holding both must be warned about (#1493).
-_COMPARABLE_CALIBERS = frozenset(
- {"raw", "split", "split_dividend", "split_dividend_additive"}
-)
+_COMPARABLE_CALIBERS = frozenset({"raw", "split", "split_dividend", "split_dividend_additive"})
#: Calibers whose daily moves cannot be read as returns at all. Cash dividends
#: enter the price *level* as an offset rather than scaling the series, so a
@@ -338,6 +336,20 @@ _COMPARABLE_CALIBERS = frozenset(
_ADDITIVE_CALIBERS = frozenset({"split_dividend_additive"})
+def frame_caliber(frame: object, source: str, market: str | None = None, symbol: str | None = None) -> str:
+ """Caliber of one served frame, honoring what the loader actually served.
+
+ A loader that converted its series (e.g. the additive-to-multiplicative
+ path from #1541) reports the caliber on the frame itself; the static
+ (source, market) table is the fallback for unconverted data.
+ """
+ attrs = getattr(frame, "attrs", None)
+ adjustment = attrs.get("adjustment") if isinstance(attrs, dict) else None
+ if isinstance(adjustment, str) and adjustment:
+ return adjustment
+ return price_caliber(source, market, symbol)
+
+
def market_has_corporate_actions(market: str) -> bool:
"""Whether prices in ``market`` depend on a split/dividend adjustment.
@@ -371,9 +383,7 @@ def price_caliber(source: str, market: str | None = None, symbol: str | None = N
market == "a_share" and symbol is not None and _A_SHARE_INDEX_CODE.match(symbol)
):
return "raw"
- return PRICE_CALIBER_BY_SOURCE_MARKET.get(
- (source, market), PRICE_CALIBER_BY_SOURCE.get(source, "unknown")
- )
+ return PRICE_CALIBER_BY_SOURCE_MARKET.get((source, market), PRICE_CALIBER_BY_SOURCE.get(source, "unknown"))
def mixed_caliber_warning(stamps: dict[str, tuple[str, str]]) -> str | None:
@@ -413,16 +423,10 @@ def additive_caliber_warning(stamps: dict[str, tuple[str, str]]) -> str | None:
dividends added back as a flat offset. That is the common A-share case,
since ``tencent`` heads the chain (#1493).
"""
- additive = {
- symbol: source
- for symbol, (source, caliber) in stamps.items()
- if caliber in _ADDITIVE_CALIBERS
- }
+ additive = {symbol: source for symbol, (source, caliber) in stamps.items() if caliber in _ADDITIVE_CALIBERS}
if not additive:
return None
- shown = ", ".join(
- f"{symbol} ({source})" for symbol, source in sorted(additive.items())[:4]
- )
+ shown = ", ".join(f"{symbol} ({source})" for symbol, source in sorted(additive.items())[:4])
if len(additive) > 4:
shown += f", +{len(additive) - 4} more"
return (
@@ -451,9 +455,7 @@ _SOURCE_ORDER_ENV_PREFIX = "MARKET_DATA_ORDER_"
# Snapshot of the chains as written above. refresh_source_order_overrides()
# restores from here; entries must never leak into FALLBACK_CHAINS by
# aliasing (always copy on restore), or a restore would mutate the snapshot.
-_DEFAULT_CHAINS: dict[str, list[str]] = {
- market: chain[:] for market, chain in FALLBACK_CHAINS.items()
-}
+_DEFAULT_CHAINS: dict[str, list[str]] = {market: chain[:] for market, chain in FALLBACK_CHAINS.items()}
# market -> override currently in effect. Populated only by
# refresh_source_order_overrides(); absent key = default chain in effect.
@@ -527,8 +529,7 @@ def refresh_source_order_overrides() -> None:
from src.config.accessor import get_env_value
snapshot = {
- source_order_env_var(market): get_env_value(source_order_env_var(market), "")
- for market in _DEFAULT_CHAINS
+ source_order_env_var(market): get_env_value(source_order_env_var(market), "") for market in _DEFAULT_CHAINS
}
if snapshot == _LAST_ORDER_ENV_SNAPSHOT:
return
@@ -543,8 +544,7 @@ def refresh_source_order_overrides() -> None:
continue
if order: # non-empty but invalid — warn, keep default order
logger.warning(
- "Ignoring invalid %s=%r: value must be a permutation of the"
- " default chain %s; keeping default order",
+ "Ignoring invalid %s=%r: value must be a permutation of the default chain %s; keeping default order",
source_order_env_var(market),
raw,
default,
@@ -591,8 +591,7 @@ def resolve_loader(market: str) -> Any:
if loader.is_available():
return loader
raise NoAvailableSourceError(
- f"No available data source for market '{market}'. "
- f"Tried: {tried or chain}. Check network and API token config."
+ f"No available data source for market '{market}'. Tried: {tried or chain}. Check network and API token config."
)
@@ -641,8 +640,7 @@ def get_loader_cls_with_fallback(source: str) -> Type[Any]:
"to api.wallex.ir.",
}.get(source, "")
raise NoAvailableSourceError(
- f"Data source '{source}' is unavailable and does not fall back to a "
- f"network source. {hint}".rstrip()
+ f"Data source '{source}' is unavailable and does not fall back to a network source. {hint}".rstrip()
)
# Source unavailable — try same-market fallback
@@ -659,6 +657,4 @@ def get_loader_cls_with_fallback(source: str) -> Type[Any]:
except NoAvailableSourceError:
continue
- raise NoAvailableSourceError(
- f"Data source '{source}' is unavailable and no fallback found."
- )
+ raise NoAvailableSourceError(f"Data source '{source}' is unavailable and no fallback found.")
diff --git a/agent/backtest/loaders/tencent_loader.py b/agent/backtest/loaders/tencent_loader.py
index 37a242971..5f8690453 100644
--- a/agent/backtest/loaders/tencent_loader.py
+++ b/agent/backtest/loaders/tencent_loader.py
@@ -20,6 +20,7 @@ import urllib.request
import certifi
import pandas as pd
+from backtest.loaders._symbol_utils import _is_etf_listed
from backtest.loaders.additive_conversion import convert_additive_to_multiplicative
from backtest.loaders.base import cached_loader_fetch, validate_date_range
from backtest.loaders.registry import register
@@ -27,6 +28,10 @@ from backtest.loaders.registry import register
logger = logging.getLogger(__name__)
_BASE_URL = "https://web.ifzq.gtimg.cn/appstock/app/fqkline/get"
+# The raw companion lives on the non-fq endpoint: fqkline without the qfq
+# suffix serves no bars at all, while kline/kline returns the unadjusted
+# "day" series (verified live 2026-09-23).
+_RAW_BASE_URL = "https://web.ifzq.gtimg.cn/appstock/app/kline/kline"
# Tencent fqkline caps a single response at `_PAGE_SIZE` bars and serves the
# LAST bars of the requested window (ending at the `end` parameter), not the
@@ -44,6 +49,7 @@ _PAGE_BACKOFF = 0.6
_SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where())
+
def _is_a_share(code: str) -> bool:
return code.upper().endswith((".SZ", ".SH"))
@@ -103,7 +109,7 @@ class DataLoader:
fields=None,
fetch=lambda code=code: self._fetch_one(code, start_date, end_date),
)
- if df is not None and not df.empty and _is_a_share(code):
+ if df is not None and not df.empty and _is_a_share(code) and not _is_etf_listed(code):
# #1541: the additive qfq cannot serve returns; convert to
# the multiplicative convention when the raw companion
# supports it, keeping the additive series otherwise.
@@ -116,14 +122,13 @@ class DataLoader:
start_date=start_date,
end_date=end_date,
fields=["raw"],
- fetch=lambda code=code: self._fetch_one(
- code, start_date, end_date, forward_adjusted=False
- ),
+ fetch=lambda code=code: self._fetch_one(code, start_date, end_date, forward_adjusted=False),
)
except Exception as exc: # noqa: BLE001 - degrade to additive
logger.warning(
"tencent raw companion fetch failed for %s, serving additive: %s",
- code, exc,
+ code,
+ exc,
)
converted = convert_additive_to_multiplicative(raw_df, df)
if converted is not None:
@@ -140,7 +145,10 @@ class DataLoader:
return result
def _request_page(
- self, code: str, start_date: str, end_date: str,
+ self,
+ code: str,
+ start_date: str,
+ end_date: str,
forward_adjusted: bool = True,
) -> Optional[pd.DataFrame]:
"""Fetch up to `_PAGE_SIZE` bars in [start_date, end_date]."""
@@ -162,15 +170,18 @@ class DataLoader:
return None
url = (
- f"{_BASE_URL}?param={tencent_code},day,"
+ f"{_BASE_URL if forward_adjusted else _RAW_BASE_URL}?param={tencent_code},day,"
f"{start_date},{end_date},{_PAGE_SIZE}"
f"{',qfq' if forward_adjusted else ''}"
)
- req = urllib.request.Request(url, headers={
- "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
- "Referer": "https://web.ifzq.gtimg.cn/",
- })
+ req = urllib.request.Request(
+ url,
+ headers={
+ "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
+ "Referer": "https://web.ifzq.gtimg.cn/",
+ },
+ )
with urllib.request.urlopen(req, timeout=15, context=_SSL_CONTEXT) as resp:
raw = resp.read().decode("utf-8")
@@ -202,14 +213,16 @@ class DataLoader:
rows = []
for k in klines:
if len(k) >= 6:
- rows.append({
- "trade_date": k[0],
- "open": float(k[1]),
- "close": float(k[2]),
- "high": float(k[3]),
- "low": float(k[4]),
- "volume": float(k[5]),
- })
+ rows.append(
+ {
+ "trade_date": k[0],
+ "open": float(k[1]),
+ "close": float(k[2]),
+ "high": float(k[3]),
+ "low": float(k[4]),
+ "volume": float(k[5]),
+ }
+ )
if not rows:
return None
@@ -217,13 +230,14 @@ class DataLoader:
df = pd.DataFrame(rows)
df["trade_date"] = pd.to_datetime(df["trade_date"])
df = df.set_index("trade_date").sort_index()
- df = df[["open", "high", "low", "close", "volume"]].dropna(
- subset=["open", "high", "low", "close"]
- )
+ df = df[["open", "high", "low", "close", "volume"]].dropna(subset=["open", "high", "low", "close"])
return df
def _fetch_one(
- self, code: str, start_date: str, end_date: str,
+ self,
+ code: str,
+ start_date: str,
+ end_date: str,
forward_adjusted: bool = True,
) -> Optional[pd.DataFrame]:
"""Paginate [start_date, end_date] in `_PAGE_SIZE`-bar windows.
@@ -250,8 +264,7 @@ class DataLoader:
for _ in range(_MAX_PAGES):
if cursor_end in seen_ends:
raise ValueError(
- f"incomplete tencent history: {code} stopped advancing at "
- f"{cursor_end} before reaching {start_date}"
+ f"incomplete tencent history: {code} stopped advancing at {cursor_end} before reaching {start_date}"
)
seen_ends.add(cursor_end)
@@ -259,15 +272,13 @@ class DataLoader:
last_error: Optional[Exception] = None
for attempt in range(_PAGE_RETRIES):
try:
- page = self._request_page(
- code, start_date, cursor_end, forward_adjusted
- )
+ page = self._request_page(code, start_date, cursor_end, forward_adjusted)
last_error = None
break
except Exception as exc: # noqa: BLE001 - transient network jitter
last_error = exc
if attempt < _PAGE_RETRIES - 1:
- time.sleep(_PAGE_BACKOFF * (2 ** attempt))
+ time.sleep(_PAGE_BACKOFF * (2**attempt))
if last_error is not None:
# Partial pages already collected would read as a complete
# history downstream; a failed page is an error, not a series.
@@ -286,13 +297,10 @@ class DataLoader:
# empty and terminates the walk.
time.sleep(_PAGE_BACKOFF)
try:
- page = self._request_page(
- code, start_date, cursor_end, forward_adjusted
- )
+ page = self._request_page(code, start_date, cursor_end, forward_adjusted)
except Exception as exc: # noqa: BLE001 - transient network jitter
raise ValueError(
- f"incomplete tencent history: {code} re-request at "
- f"{cursor_end} failed: {exc}"
+ f"incomplete tencent history: {code} re-request at {cursor_end} failed: {exc}"
) from exc
if page is None or page.empty:
break
@@ -307,10 +315,7 @@ class DataLoader:
break
cursor_end = next_end
else:
- raise ValueError(
- f"incomplete tencent history: {code} hit {_MAX_PAGES} pages "
- f"without reaching {start_date}"
- )
+ raise ValueError(f"incomplete tencent history: {code} hit {_MAX_PAGES} pages without reaching {start_date}")
if not chunks:
return None
diff --git a/agent/backtest/runner.py b/agent/backtest/runner.py
index 1d9278f9a..7b3435935 100644
--- a/agent/backtest/runner.py
+++ b/agent/backtest/runner.py
@@ -30,7 +30,7 @@ from backtest.loaders.registry import (
get_loader_cls_with_fallback,
is_no_network_fallback_source,
mixed_caliber_warning,
- price_caliber,
+ frame_caliber,
resolve_loader,
)
from backtest.loaders.base import (
@@ -39,6 +39,7 @@ from backtest.loaders.base import (
source_interval,
validate_ohlc,
)
+
# Symbol classification lives in ``_market_hooks`` so runner.py and
# composite.py share a single source of truth (audit-2026-05-18 B1+C1+C2).
# ``_detect_market`` is also re-exported here for back-compat with
@@ -138,9 +139,7 @@ class BacktestConfigSchema(BaseModel):
try:
pd.Timestamp(v)
except Exception:
- raise ValueError(
- f"invalid evaluation_start_date: {v!r} (expected YYYY-MM-DD)"
- ) from None
+ raise ValueError(f"invalid evaluation_start_date: {v!r} (expected YYYY-MM-DD)") from None
return v
@field_validator("rebalance_mask")
@@ -191,9 +190,7 @@ class BacktestConfigSchema(BaseModel):
return v
for entry in v:
if not isinstance(entry, dict):
- raise ValueError(
- "each event_feeds entry must be an object with name/route_template/event_type"
- )
+ raise ValueError("each event_feeds entry must be an object with name/route_template/event_type")
for key in ("name", "route_template", "event_type"):
if not str(entry.get(key, "")).strip():
raise ValueError(f"event_feeds entry missing required field: {key}")
@@ -202,13 +199,9 @@ class BacktestConfigSchema(BaseModel):
@model_validator(mode="after")
def start_before_end(self) -> "BacktestConfigSchema":
if self.rebalance_mask is not None and self.position_adjustment != "rebalance":
- raise ValueError(
- "rebalance_mask requires position_adjustment='rebalance'"
- )
+ raise ValueError("rebalance_mask requires position_adjustment='rebalance'")
if pd.Timestamp(self.start_date) > pd.Timestamp(self.end_date):
- raise ValueError(
- f"start_date ({self.start_date}) must be <= end_date ({self.end_date})"
- )
+ raise ValueError(f"start_date ({self.start_date}) must be <= end_date ({self.end_date})")
return self
@@ -316,9 +309,7 @@ def _validate_class_body(node: ast.ClassDef) -> None:
continue
if isinstance(child, ast.Pass):
continue
- raise ValueError(
- f"Executable class-level statement {type(child).__name__} is not allowed"
- )
+ raise ValueError(f"Executable class-level statement {type(child).__name__} is not allowed")
# --- Runtime-reachable operation scrubber (VT-001 defense-in-depth) ---
@@ -430,10 +421,9 @@ def _is_forbidden_module_path(name: str) -> bool:
return False
if name.split(".")[0] in _FORBIDDEN_IMPORT_MODULES:
return True
- return any(
- name == prefix or name.startswith(prefix + ".")
- for prefix in _FORBIDDEN_IMPORT_PREFIXES
- )
+ return any(name == prefix or name.startswith(prefix + ".") for prefix in _FORBIDDEN_IMPORT_PREFIXES)
+
+
# ``os`` itself is allowed (os.path etc.), but these attributes shell out, spawn,
# or read the process environment — none has a place in a signal engine.
_FORBIDDEN_OS_ATTRS = frozenset(
@@ -498,9 +488,7 @@ _FORBIDDEN_DUNDER_ATTRS = frozenset(
"__subclasshook__",
}
)
-_FORBIDDEN_BUILTINS = frozenset(
- {"eval", "exec", "compile", "__import__", "globals", "locals", "vars", "breakpoint"}
-)
+_FORBIDDEN_BUILTINS = frozenset({"eval", "exec", "compile", "__import__", "globals", "locals", "vars", "breakpoint"})
# getattr/setattr/delattr can indirect around the attribute scanner
# (``getattr(os, "system")("id")``). We reject them ONLY when the target object
# is ``os`` or a forbidden module — keyed off the target, not the attribute
@@ -755,14 +743,8 @@ def _scan_runtime_reachable(tree: ast.Module) -> None:
if engine_cls is None:
return
- module_funcs = {
- n.name: n
- for n in tree.body
- if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))
- }
- worklist: list[ast.AST] = [
- m for m in engine_cls.body if isinstance(m, (ast.FunctionDef, ast.AsyncFunctionDef))
- ]
+ module_funcs = {n.name: n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))}
+ worklist: list[ast.AST] = [m for m in engine_cls.body if isinstance(m, (ast.FunctionDef, ast.AsyncFunctionDef))]
aliases = _module_level_forbidden_aliases(tree)
visited: set[int] = set()
while worklist:
@@ -773,10 +755,7 @@ def _scan_runtime_reachable(tree: ast.Module) -> None:
for node in ast.walk(fn):
_reject_forbidden_node(node)
if isinstance(node, ast.Name) and node.id in aliases:
- raise ValueError(
- f"Use of {node.id!r}, bound at module level to "
- f"{aliases[node.id]!r}, {_SCRUB_MSG}"
- )
+ raise ValueError(f"Use of {node.id!r}, bound at module level to {aliases[node.id]!r}, {_SCRUB_MSG}")
if isinstance(node, ast.Call) and isinstance(node.func, ast.Name):
target = module_funcs.get(node.func.id)
if target is not None:
@@ -808,9 +787,7 @@ def _validate_signal_engine_source(file_path: Path) -> None:
continue
if _is_safe_constant_assignment(node):
continue
- raise ValueError(
- f"Executable top-level statement {type(node).__name__} is not allowed"
- )
+ raise ValueError(f"Executable top-level statement {type(node).__name__} is not allowed")
# Deep pass: the structural loop above only guards import-time execution;
# this walks the code that runs on SignalEngine().generate() (VT-001).
@@ -821,8 +798,10 @@ def _validate_signal_engine_class(engine_cls) -> None:
"""Pre-flight check: SignalEngine can be instantiated with no args and has generate()."""
sig = inspect.signature(engine_cls.__init__)
required = [
- p.name for p in sig.parameters.values()
- if p.name != "self" and p.default is inspect.Parameter.empty
+ p.name
+ for p in sig.parameters.values()
+ if p.name != "self"
+ and p.default is inspect.Parameter.empty
and p.kind not in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD)
]
if required:
@@ -1050,11 +1029,7 @@ def _build_price_panel(data_map: dict[str, pd.DataFrame]) -> dict[str, pd.DataFr
"""
panel: dict[str, pd.DataFrame] = {}
for column in _PRICE_PANEL_COLUMNS:
- series_by_symbol = {
- symbol: frame[column]
- for symbol, frame in data_map.items()
- if column in frame.columns
- }
+ series_by_symbol = {symbol: frame[column] for symbol, frame in data_map.items() if column in frame.columns}
if series_by_symbol:
panel[column] = pd.DataFrame(series_by_symbol)
return panel
@@ -1096,7 +1071,7 @@ def _inject_fundamental_panel(
Returns:
The same panel dictionary with ``fund:<field>`` frames added.
"""
- fields = [column[len(_FUND_PREFIX):] for column in fund_columns if column.startswith(_FUND_PREFIX)]
+ fields = [column[len(_FUND_PREFIX) :] for column in fund_columns if column.startswith(_FUND_PREFIX)]
fields = list(dict.fromkeys(fields))
if not fields:
return panel
@@ -1188,6 +1163,7 @@ def _maybe_inject_fundamentals_for_factor_panel(
# --- Main entry ---
+
def main(run_dir: Path) -> None:
"""Load config, fetch data, run the selected backtest engine.
@@ -1220,6 +1196,7 @@ def main(run_dir: Path) -> None:
# below blocks executable top-level statements but a method body still
# runs on instantiation. See ``safe_run_dir`` for the policy.
from src.tools.path_utils import safe_run_dir
+
try:
run_dir = safe_run_dir(str(run_dir))
except ValueError as exc:
@@ -1311,6 +1288,7 @@ def main(run_dir: Path) -> None:
if engine_type == "options":
from backtest.engines.options_portfolio import run_options_backtest
+
run_options_backtest(config, loader, signal_engine, run_dir, bars_per_year=bars_per_year)
else:
market_engine = _create_market_engine(effective_source, config, codes)
@@ -1365,9 +1343,7 @@ def _observed_spacing(data_map: dict, codes: List[str]) -> float | None:
fundamental panel cannot decide the annualisation.
"""
indexes = [
- data_map[code].index
- for code in codes
- if code in data_map and len(data_map[code]) >= _MIN_BARS_FOR_SPACING
+ data_map[code].index for code in codes if code in data_map and len(data_map[code]) >= _MIN_BARS_FOR_SPACING
]
if not indexes:
return None
@@ -1442,8 +1418,7 @@ def _annualisation_bars(
matched = min(
_INTERVAL_SECONDS,
- key=lambda name: max(_INTERVAL_SECONDS[name], observed)
- / min(_INTERVAL_SECONDS[name], observed),
+ key=lambda name: max(_INTERVAL_SECONDS[name], observed) / min(_INTERVAL_SECONDS[name], observed),
)
matched_spacing = _INTERVAL_SECONDS[matched]
if max(matched_spacing, observed) / min(matched_spacing, observed) >= _SPACING_MISMATCH_RATIO:
@@ -1513,6 +1488,7 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# Cross-market -> CompositeEngine
if len(markets) > 1:
from backtest.engines.composite import CompositeEngine
+
return CompositeEngine(config, codes)
# Futures routing (Wave 2)
@@ -1520,13 +1496,16 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# Distinguish China vs global futures by exchange suffix
if any(_is_china_futures(c) for c in codes):
from backtest.engines.china_futures import ChinaFuturesEngine
+
return ChinaFuturesEngine(config)
from backtest.engines.global_futures import GlobalFuturesEngine
+
return GlobalFuturesEngine(config)
# Forex routing (Wave 2)
if "forex" in markets:
from backtest.engines.forex import ForexEngine
+
return ForexEngine(config)
# India equity routing — must precede source-based routing because India's
@@ -1534,12 +1513,14 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# otherwise fall through to the crypto default.
if "india_equity" in markets:
from backtest.engines.india_equity import IndiaEquityEngine
+
return IndiaEquityEngine(config)
# Korea equity routing — same reason as India: its effective source
# (``pykrx``) has no Wave-1 branch and would fall through to the default.
if "kr_equity" in markets:
from backtest.engines.korea_equity import KoreaEquityEngine
+
return KoreaEquityEngine(config)
# Vietnam equity routing — same reason as India and Korea: its effective
@@ -1547,6 +1528,7 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# default.
if "vietnam_equity" in markets:
from backtest.engines.vietnam_equity import VietnamEquityEngine
+
return VietnamEquityEngine(config)
# Argentina market-data routing is supported, but BYMA execution rules
# are not modeled yet. Fail closed instead of silently applying US/crypto
@@ -1562,18 +1544,22 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# default the source-based fallback would pick.
if "index" in markets:
from backtest.engines.global_equity import GlobalEquityEngine
+
return GlobalEquityEngine(config, market=_detect_submarket(codes))
# Original routing (Wave 1)
if source in ("okx", "ccxt"):
from backtest.engines.crypto import CryptoEngine
+
return CryptoEngine(config)
elif source in ("tushare", "akshare"):
if markets & {"us_equity", "hk_equity", "ca_equity", "uk_equity"}:
from backtest.engines.global_equity import GlobalEquityEngine
+
market = _detect_submarket(codes)
return GlobalEquityEngine(config, market=market)
from backtest.engines.china_a import ChinaAEngine
+
return ChinaAEngine(config)
elif source == "yfinance":
# yfinance serves crypto pairs (BTC-USDT, BTC-USD) next to equities,
@@ -1583,8 +1569,10 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# in the config, and nothing warns.
if "crypto" in markets:
from backtest.engines.crypto import CryptoEngine
+
return CryptoEngine(config)
from backtest.engines.global_equity import GlobalEquityEngine
+
market = _detect_submarket(codes)
return GlobalEquityEngine(config, market=market)
else:
@@ -1593,6 +1581,7 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# local AAPL.US dataset gets US-equity execution rules instead of crypto.
if markets & {"us_equity", "hk_equity", "ca_equity", "uk_equity"}:
from backtest.engines.global_equity import GlobalEquityEngine
+
market = _detect_submarket(codes)
return GlobalEquityEngine(config, market=market)
# A-shares need the same treatment. Every branchless source that serves
@@ -1603,8 +1592,10 @@ def _create_market_engine(source: str, config: dict, codes: List[str]):
# position. The run still succeeds, which is what makes it dangerous.
if "a_share" in markets:
from backtest.engines.china_a import ChinaAEngine
+
return ChinaAEngine(config)
from backtest.engines.crypto import CryptoEngine
+
return CryptoEngine(config)
@@ -1662,13 +1653,14 @@ def _fetch_auto(codes: List[str], config: dict, interval: str = "1D") -> dict:
local_result = local_loader.fetch(stripped, start_date, end_date, interval=interval)
missing_local = [code for code in local_codes if strip_local_prefix(code) not in local_result]
if missing_local:
- raise NoAvailableSourceError(
- f"incomplete data for source=local; missing symbols: {missing_local}"
- )
+ raise NoAvailableSourceError(f"incomplete data for source=local; missing symbols: {missing_local}")
local_name = str(getattr(local_loader, "name", "local") or "local")
served_by.add(local_name)
for code in local_result:
- caliber_stamps[code] = (local_name, price_caliber(local_name, _detect_market(code), code))
+ caliber_stamps[code] = (
+ local_name,
+ frame_caliber(local_result[code], local_name, _detect_market(code), code),
+ )
merged.update(local_result)
market_groups = _group_codes_by_market([code for code in codes if code not in set(local_codes)])
@@ -1692,13 +1684,11 @@ def _fetch_auto(codes: List[str], config: dict, interval: str = "1D") -> dict:
fields=fields,
interval=interval,
)
- market_result = _restore_original_codes(
- result, market_codes, normalized_codes
- )
+ market_result = _restore_original_codes(result, market_codes, normalized_codes)
if market_result:
served_by.add(src_name)
for code in market_result:
- caliber_stamps[code] = (src_name, price_caliber(src_name, market, code))
+ caliber_stamps[code] = (src_name, frame_caliber(market_result[code], src_name, market, code))
missing = [code for code in market_codes if code not in market_result]
# Retry only missing symbols so a partial primary response does not
@@ -1712,9 +1702,7 @@ def _fetch_auto(codes: List[str], config: dict, interval: str = "1D") -> dict:
if not fb_loader.is_available():
continue
fb_codes = _normalize_codes(missing, fb_name)
- fallback_result = fb_loader.fetch(
- fb_codes, start_date, end_date, interval=interval
- )
+ fallback_result = fb_loader.fetch(fb_codes, start_date, end_date, interval=interval)
mapped = _restore_original_codes(fallback_result, missing, fb_codes)
if mapped:
market_result.update(mapped)
@@ -1722,18 +1710,15 @@ def _fetch_auto(codes: List[str], config: dict, interval: str = "1D") -> dict:
fb_served_by = str(getattr(fb_loader, "name", fb_name) or fb_name)
served_by.add(fb_served_by)
for code in mapped:
- caliber_stamps[code] = (fb_served_by, price_caliber(fb_served_by, market, code))
- logger.info(
- "Runtime fallback: %s -> %s for %s", src_name, fb_name, market
- )
+ caliber_stamps[code] = (fb_served_by, frame_caliber(mapped[code], fb_served_by, market, code))
+ logger.info("Runtime fallback: %s -> %s for %s", src_name, fb_name, market)
if missing:
served = sorted(set(merged) - set(missing))
raise NoAvailableSourceError(
f"incomplete data for {market}; missing symbols: {missing}; "
f"served symbols: {served}"
- )
- merged.update(market_result)
+ ) merged.update(market_result)
config["_actual_sources"] = sorted(served_by)
config["_caliber_stamps"] = caliber_stamps
@@ -1773,9 +1758,7 @@ def fetch_data_map(config: dict) -> DataFetchResult:
# a contradictory request is refused here rather than half-served.
prefixed = [code for code in codes if strip_local_prefix(code) != code]
if prefixed and source not in ("local", "auto"):
- raise ValueError(
- f"local: codes need source='local' or 'auto', not {source!r}: {prefixed}"
- )
+ raise ValueError(f"local: codes need source='local' or 'auto', not {source!r}: {prefixed}")
bare = [strip_local_prefix(code) for code in codes]
repeated = sorted({symbol for symbol in bare if bare.count(symbol) > 1})
if repeated:
@@ -1790,9 +1773,9 @@ def fetch_data_map(config: dict) -> DataFetchResult:
# is only a fallback for a stubbed/patched fetcher that recorded nothing.
recorded = config.pop("_actual_sources", None)
caliber_stamps = config.pop("_caliber_stamps", None) or {}
- used_sources: list[str] = [
- str(name) for name in recorded or [] if str(name).strip()
- ] or sorted(_group_codes_by_source(codes))
+ used_sources: list[str] = [str(name) for name in recorded or [] if str(name).strip()] or sorted(
+ _group_codes_by_source(codes)
+ )
else:
codes = _normalize_codes(codes, source)
primary_source = source
@@ -1823,7 +1806,7 @@ def fetch_data_map(config: dict) -> DataFetchResult:
for code in data_map:
caliber_stamps[code] = (
served_by,
- price_caliber(served_by, _detect_market(code), code),
+ frame_caliber(data_map[code], served_by, _detect_market(code), code),
)
used_sources = [served_by] if data_map else []
missing = [code for code in codes if code not in data_map]
@@ -1831,9 +1814,7 @@ def fetch_data_map(config: dict) -> DataFetchResult:
# as if the dataset held it. A ``local:`` code is the dataset's or nothing.
unserved_local = [code for code in prefixed if strip_local_prefix(code) in missing]
if unserved_local:
- raise NoAvailableSourceError(
- f"incomplete data for source=local; missing symbols: {unserved_local}"
- )
+ raise NoAvailableSourceError(f"incomplete data for source=local; missing symbols: {unserved_local}")
if missing:
logger.warning(
"source=%s returned data for %d/%d symbols; missing: %s",
@@ -1880,8 +1861,7 @@ def fetch_data_map(config: dict) -> DataFetchResult:
market_missing = [code for code in market_missing if code not in mapped]
fb_served_by = str(
getattr(fallback_loader, "name", fallback_source)
- or fallback_source
- )
+ or fallback_source )
for code in mapped:
caliber_stamps[code] = (
fb_served_by,
@@ -1895,20 +1875,13 @@ def fetch_data_map(config: dict) -> DataFetchResult:
"Runtime fallback: %s -> %s", primary_source, fb_served_by
)
missing = [code for code in missing if code not in data_map]
-
if missing:
raise NoAvailableSourceError(
f"incomplete data for source={primary_source}; missing symbols: {missing}; "
f"served symbols: {sorted(data_map)}"
)
-
- data_map = {
- code: resample_bars(frame, requested_interval)
- for code, frame in _sanitize_data_map(data_map).items()
- }
- caliber_stamps = {
- code: stamp for code, stamp in caliber_stamps.items() if code in data_map
- }
+ data_map = {code: resample_bars(frame, requested_interval) for code, frame in _sanitize_data_map(data_map).items()}
+ caliber_stamps = {code: stamp for code, stamp in caliber_stamps.items() if code in data_map}
# Both warnings can apply at once (a tencent+baostock basket mixes calibers
# *and* serves an additive one), and each says something the other does not,
# so they are reported together rather than one shadowing the other.
diff --git a/agent/src/market_data.py b/agent/src/market_data.py
index 6cdd479c0..4ec0f81bf 100644
--- a/agent/src/market_data.py
+++ b/agent/src/market_data.py
@@ -72,7 +72,6 @@ def detect_source(code: str) -> str:
return "tushare"
-
def get_loader(source: str):
"""Get loader class via registry with fallback support."""
from backtest.loaders.registry import get_loader_cls_with_fallback
@@ -89,10 +88,46 @@ def get_loader(source: str):
#: search, fetch and grounding all agree.
FIAT_CODES = frozenset(
{
- "USD", "EUR", "GBP", "JPY", "CHF", "CNY", "CNH", "HKD", "AUD", "NZD",
- "CAD", "KRW", "INR", "SGD", "SEK", "NOK", "DKK", "MXN", "BRL", "ZAR",
- "TRY", "RUB", "PLN", "THB", "MYR", "IDR", "PHP", "VND", "ILS", "AED",
- "SAR", "EGP", "CZK", "HUF", "RON", "CLP", "COP", "PEN", "TWD", "CUP",
+ "USD",
+ "EUR",
+ "GBP",
+ "JPY",
+ "CHF",
+ "CNY",
+ "CNH",
+ "HKD",
+ "AUD",
+ "NZD",
+ "CAD",
+ "KRW",
+ "INR",
+ "SGD",
+ "SEK",
+ "NOK",
+ "DKK",
+ "MXN",
+ "BRL",
+ "ZAR",
+ "TRY",
+ "RUB",
+ "PLN",
+ "THB",
+ "MYR",
+ "IDR",
+ "PHP",
+ "VND",
+ "ILS",
+ "AED",
+ "SAR",
+ "EGP",
+ "CZK",
+ "HUF",
+ "RON",
+ "CLP",
+ "COP",
+ "PEN",
+ "TWD",
+ "CUP",
}
)
@@ -210,6 +245,7 @@ def fetch_market_data(
FALLBACK_CHAINS,
_NO_NETWORK_FALLBACK_SOURCES,
get_source_order_override,
+ frame_caliber,
price_caliber,
refresh_source_order_overrides,
)
@@ -221,12 +257,7 @@ def fetch_market_data(
results: dict[str, Any] = {}
provenance: dict[str, dict[str, Any]] = {}
- result_aliases = {
- code: code.split(":", 1)[1]
- if code.lower().startswith("local:")
- else code
- for code in codes
- }
+ result_aliases = {code: code.split(":", 1)[1] if code.lower().startswith("local:") else code for code in codes}
groups: dict[tuple[str, str], list[str]] = {}
for code in codes:
@@ -282,16 +313,9 @@ def fetch_market_data(
override = None
else:
override = (
- get_source_order_override(market)
- if source == "auto"
- and fallback_chain_provider is None
- else None
- )
- candidates = (
- list(override)
- if override is not None and src in override
- else [src, *chain]
+ get_source_order_override(market) if source == "auto" and fallback_chain_provider is None else None
)
+ candidates = list(override) if override is not None and src in override else [src, *chain]
# Deduplicate (preserving order), then cap the attempt budget.
attempts: list[str] = []
for candidate in candidates:
@@ -318,9 +342,7 @@ def fetch_market_data(
try:
loader = loader_cls()
# Weekly and monthly bars are built from daily ones (#1479).
- partial = loader.fetch(
- remaining, start_date, end_date, interval=source_interval(interval)
- )
+ partial = loader.fetch(remaining, start_date, end_date, interval=source_interval(interval))
partial = {
code: resample_bars(frame, interval) if hasattr(frame, "groupby") else frame
for code, frame in (partial or {}).items()
@@ -328,7 +350,9 @@ def fetch_market_data(
except Exception as exc: # noqa: BLE001 — contained per-symbol fallback
logger.error(
"market-data loader %r failed for %s; trying next source in chain: %s",
- attempt_src, remaining, exc,
+ attempt_src,
+ remaining,
+ exc,
)
continue
if not partial:
@@ -347,15 +371,16 @@ def fetch_market_data(
if used_source and used_source != src:
logger.info(
"market-data source %r unavailable for %s; fell back to %r",
- src, src_codes, used_source,
+ src,
+ src_codes,
+ used_source,
)
- served_elsewhere = sorted(
- {serve_src for serve_src, _ in symbol_sources.values()} - {used_source}
- )
+ served_elsewhere = sorted({serve_src for serve_src, _ in symbol_sources.values()} - {used_source})
if served_elsewhere:
logger.info(
"market-data per-symbol fallback: %s served the remaining symbols %r could not",
- served_elsewhere, src,
+ served_elsewhere,
+ src,
)
return data_map, used_source, provider_cls, symbol_sources
@@ -385,9 +410,7 @@ def fetch_market_data(
# A loader that converted its series reports the caliber it actually
# served on the frame; the static (source, market) table is the
# fallback for unconverted data (#1541).
- adjustment = frame_attrs.get("adjustment")
- if not isinstance(adjustment, str) or not adjustment:
- adjustment = price_caliber(used_source or src, market, symbol)
+ adjustment = frame_caliber(df, used_source or src, market, symbol)
entry: dict[str, Any] = {
"source": used_source or src,
"requested_source": source,
@@ -405,24 +428,19 @@ def fetch_market_data(
provenance[symbol] = entry
for (src, market), src_codes in groups.items():
- data_map, used_source, provider_cls, symbol_sources = _fetch_via_chain(
- src, market, src_codes
- )
+ data_map, used_source, provider_cls, symbol_sources = _fetch_via_chain(src, market, src_codes)
for symbol, df in data_map.items():
- symbol_source, symbol_provider_cls = symbol_sources.get(
- symbol, (used_source, provider_cls)
- )
+ symbol_source, symbol_provider_cls = symbol_sources.get(symbol, (used_source, provider_cls))
_emit(
- symbol, df,
- src=src, used_source=symbol_source, provider_cls=symbol_provider_cls,
+ symbol,
+ df,
+ src=src,
+ used_source=symbol_source,
+ provider_cls=symbol_provider_cls,
market=market,
)
- unresolved = [
- code
- for code in codes
- if code not in results and result_aliases[code] not in results
- ]
+ unresolved = [code for code in codes if code not in results and result_aliases[code] not in results]
# Canadian venue-alias fallback: TSX (.TO) <-> TSX Venture (.V).
#
@@ -461,21 +479,25 @@ def fetch_market_data(
"resolved_symbol": sibling,
}
logger.info(
- "market-data venue alias %s -> %s (source=%s)", code, sibling, src,
+ "market-data venue alias %s -> %s (source=%s)",
+ code,
+ sibling,
+ src,
)
unresolved.remove(code)
continue
# Sibling not already resolved — targeted re-fetch of just that
# symbol through the same market's source chain.
- sibling_data, used_source, provider_cls, _ = _fetch_via_chain(
- src, market, [sibling]
- )
+ sibling_data, used_source, provider_cls, _ = _fetch_via_chain(src, market, [sibling])
if sibling_data:
df = next(iter(sibling_data.values()))
if df is not None and not df.empty:
_emit(
- code, df,
- src=src, used_source=used_source, provider_cls=provider_cls,
+ code,
+ df,
+ src=src,
+ used_source=used_source,
+ provider_cls=provider_cls,
market=market,
extra_provenance={
"venue_fallback": True,
@@ -484,7 +506,9 @@ def fetch_market_data(
)
logger.info(
"market-data venue fallback %s -> %s (source=%s)",
- code, sibling, src,
+ code,
+ sibling,
+ src,
)
unresolved.remove(code)
diff --git a/agent/tests/test_additive_conversion.py b/agent/tests/test_additive_conversion.py
index 857b90772..18d87e3f0 100644
--- a/agent/tests/test_additive_conversion.py
+++ b/agent/tests/test_additive_conversion.py
@@ -52,9 +52,14 @@ class TestConvertAdditiveToMultiplicative:
f1 = 100.0 / 102.0
f2 = 99.0 / 101.0
expected = [
- 100.0 * f1 * f2, 101.0 * f1 * f2, 102.0 * f1 * f2,
- 99.0 * f2, 100.0 * f2, 101.0 * f2,
- 98.0, 99.0,
+ 100.0 * f1 * f2,
+ 101.0 * f1 * f2,
+ 102.0 * f1 * f2,
+ 99.0 * f2,
+ 100.0 * f2,
+ 101.0 * f2,
+ 98.0,
+ 99.0,
]
assert converted["close"].tolist() == pytest.approx(expected, rel=1e-9)
@@ -78,12 +83,35 @@ class TestConvertAdditiveToMultiplicative:
assert converted["volume"].iloc[-1] == 1000.0
assert converted["volume"].iloc[0] == pytest.approx(1000.0 / (f1 * f2), rel=1e-9)
- def test_no_actions_is_identity_on_raw(self):
+ def test_no_actions_keeps_todays_series(self):
raw = _frame([10.0, 11.0, 12.0])
additive = _frame([10.0, 11.0, 12.0])
+ # No in-window corporate action means nothing to convert; the caller
+ # keeps the served series and its static stamp rather than relabeling
+ # an identical frame as multiplicative.
+ assert convert_additive_to_multiplicative(raw, additive) is None
+
+ def test_wobble_bar_folds_into_its_level(self):
+ # One quote-glitch bar sits inside an otherwise constant offset: the
+ # structural rule folds it and the two real dividends still convert.
+ raw, additive = _two_action_series()
+ additive.iloc[4, additive.columns.get_loc("close")] += 0.03
+ for col in ("open", "high", "low"):
+ additive.iloc[4, additive.columns.get_loc(col)] += 0.03
converted = convert_additive_to_multiplicative(raw, additive)
assert converted is not None
- assert converted["close"].tolist() == raw["close"].tolist()
+ f1 = 100.0 / 102.0
+ f2 = 99.0 / 101.0
+ assert converted["close"].iloc[0] == pytest.approx(100.0 * f1 * f2, rel=1e-9)
+ assert converted["close"].iloc[-1] == pytest.approx(99.0, rel=1e-9)
+
+ def test_unclassifiable_one_bar_plateau_refuses(self):
+ # A one-bar plateau whose neighbors disagree with each other is a step
+ # pair we cannot classify as noise or dividend; the window is refused
+ # rather than shipped as a clean multiplicative series.
+ raw = _frame([100.0, 100.0, 100.0, 100.0, 100.0])
+ additive = _frame([95.0, 95.0, 98.0, 96.0, 96.0])
+ assert convert_additive_to_multiplicative(raw, additive) is None
def test_misaligned_calendars_fail_closed(self):
raw, additive = _two_action_series()
@@ -96,12 +124,10 @@ class TestConvertAdditiveToMultiplicative:
assert convert_additive_to_multiplicative(raw, additive) is None
def test_dividend_at_or_above_price_fails_closed(self):
- raw = _frame([2.0, 2.0, 1.0, 1.0])
- additive = _frame([0.5, 0.5, 1.0, 1.0]) # implies a 1.5 dividend on a 2.0 close? actually -0.5->? keep shape
- # offsets: -1.5, -1.5, 0, 0 -> dividend 1.5 < 2.0, valid. Force invalid:
- raw2 = _frame([1.0, 1.0, 0.5, 0.5])
- additive2 = _frame([-0.6, -0.6, 0.5, 0.5]) # dividend 1.1 >= price 1.0
- assert convert_additive_to_multiplicative(raw2, additive2) is None
+ # dividend 1.1 >= price 1.0
+ raw = _frame([1.0, 1.0, 0.5, 0.5])
+ additive = _frame([-0.6, -0.6, 0.5, 0.5])
+ assert convert_additive_to_multiplicative(raw, additive) is None
def test_empty_or_missing_inputs_fail_closed(self):
raw, additive = _two_action_series()
@@ -117,7 +143,8 @@ class TestLoaderWiring:
def _patch_fetch_pair(self, monkeypatch, loader_mod, raw, additive):
monkeypatch.setattr(
- loader_mod, "cached_loader_fetch",
+ loader_mod,
+ "cached_loader_fetch",
lambda source, symbol, timeframe, start_date, end_date, fields, fetch: (
raw if fields == ["raw"] else additive
),
@@ -169,6 +196,22 @@ class TestLoaderWiring:
# ETF path is not converted; the additive frame passes through.
assert out["510300.SH"]["close"].tolist() == additive["close"].tolist()
+ def test_tencent_skips_etf_codes(self, monkeypatch):
+ from backtest.loaders import tencent_loader
+
+ _, additive = _two_action_series()
+ self._patch_fetch_pair(monkeypatch, tencent_loader, None, additive)
+ out = tencent_loader.DataLoader().fetch(["510300.SH"], "2024-01-01", "2024-01-10")
+ assert out["510300.SH"]["close"].tolist() == additive["close"].tolist()
+
+ def test_eastmoney_skips_etf_codes(self, monkeypatch):
+ from backtest.loaders import eastmoney_loader
+
+ _, additive = _two_action_series()
+ self._patch_fetch_pair(monkeypatch, eastmoney_loader, None, additive)
+ out = eastmoney_loader.DataLoader().fetch(["510300.SH"], "2024-01-01", "2024-01-10")
+ assert out["510300.SH"]["close"].tolist() == additive["close"].tolist()
+
class TestProvenancePrefersFrameStamp:
def test_frame_adjustment_wins_over_static_table(self, monkeypatch):