"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(loaders): adjust A-share prices and use point-in-time index membership
Haozhe Wud52199ff6d3a
diff --git a/agent/backtest/loaders/cn_adjust.py b/agent/backtest/loaders/cn_adjust.pynew file mode 100644index 000000000..5f904b3a6--- /dev/null+++ b/agent/backtest/loaders/cn_adjust.py@@ -0,0 +1,77 @@+"""Corporate-action adjustment for Tushare A-share and fund bars.++Tushare's ``daily`` and ``fund_daily`` endpoints return *unadjusted* prices, so+a close-to-close return taken across an ex-date spans the mechanical price drop+of a split, bonus issue or dividend rather than the instrument's actual move.+Measured against Tushare's own ``pct_chg`` over 2020-2024: 300750.SZ on+2023-04-26 reads -41.82% raw against a true +5.40%, 300124.SZ -34.26% against+-1.01%, 601012.SH -24.21% against +6.46%. The error is always negative, because+prices only ever gap down on an ex-date, so it is a systematic contaminant of+returns rather than noise.++The matching factor series comes from ``pro.adj_factor`` for equities and+``pro.fund_adj`` for funds; both carry the same ``(trade_date, adj_factor)``+schema. Factors are cumulative and rise across a corporate action — verified on+600519.SH 2022-06-30, where the factor steps 7.4740 -> 7.5546 as the stock goes+ex-dividend.++This module is the single implementation; the backtest loader and the alpha+bench both route through it rather than re-deriving the convention.+"""++from __future__ import annotations++import pandas as pd+++def apply_qfq(df: pd.DataFrame, factor: pd.DataFrame | None) -> pd.DataFrame | None:+ """Forward-adjust raw Tushare daily bars for corporate actions.++ ``pro.daily`` returns *unadjusted* prices, so a close-to-close return taken+ across an ex-date spans the mechanical price drop of a split, bonus issue or+ dividend rather than the stock's actual move. Measured against Tushare's own+ ``pct_chg`` on CSI300 members: 300750.SZ on 2023-04-26 reads -41.82% raw+ against a true +5.40%, 300124.SZ on 2021-06-04 reads -34.26% against -1.01%,+ 601012.SH on 2022-06-06 reads -24.21% against +6.46%. The error is always+ negative, so it is a systematic contaminant of every cross-sectional IC the+ bench reports, not noise.++ Prices are scaled to the last bar in the window (前复权), matching the+ ``adjust="qfq"`` convention the akshare and tencent loaders already use.+ ``volume`` is divided by the same ratio so share counts stay on one basis+ and the ``amount``-derived VWAP keeps the same relationship to ``close``;+ ``amount`` is a cash figure and is left untouched.++ Args:+ df: Raw daily bars indexed by trade date.+ factor: ``pro.adj_factor`` rows for the same symbol and window.++ Returns:+ The adjusted frame, or ``None`` when the adjustment factors are missing+ or unusable — the caller drops the symbol rather than benching it on+ contaminated prices.+ """+ if factor is None or getattr(factor, "empty", True):+ return None+ if "adj_factor" not in factor.columns or "trade_date" not in factor.columns:+ return None+ series = factor.copy()+ series["trade_date"] = pd.to_datetime(series["trade_date"], errors="coerce")+ series = (+ series.dropna(subset=["trade_date"])+ .set_index("trade_date")["adj_factor"]+ .astype(float)+ .sort_index()+ )+ series = series.reindex(df.index).ffill().bfill()+ if series.isna().any() or (series <= 0).any():+ return None++ ratio = series / series.iloc[-1]+ out = df.copy()+ for col in ("open", "high", "low", "close"):+ if col in out.columns:+ out[col] = out[col] * ratio+ if "volume" in out.columns:+ out["volume"] = out["volume"] / ratio+ return outdiff --git a/agent/backtest/loaders/tushare.py b/agent/backtest/loaders/tushare.pyindex 8390e01bc..22c23dd1e 100644--- a/agent/backtest/loaders/tushare.py+++ b/agent/backtest/loaders/tushare.py@@ -11,6 +11,7 @@ import pandas as pdfrom backtest.loaders._symbol_utils import _is_etf_listedfrom backtest.loaders.base import cached_loader_fetch, validate_date_range+from backtest.loaders.cn_adjust import apply_qfqfrom backtest.loaders.registry import registerlogger = logging.getLogger(__name__)@@ -149,15 +150,21 @@ class DataLoader:if _is_etf_listed(code):endpoint_name = "fund_daily"df = self.api.fund_daily(ts_code=code, start_date=start_date, end_date=end_date)+ adjust = getattr(self.api, "fund_adj", None)elif _is_index(code):endpoint_name = "index_daily"df = self.api.index_daily(ts_code=code, start_date=start_date, end_date=end_date)+ # An index level is already continuous across its members' ex-dates.+ adjust = Noneelif _is_hk_equity(code):endpoint_name = "hk_daily"df = self.api.hk_daily(ts_code=code, start_date=start_date, end_date=end_date)+ # Tushare publishes no HK adjustment-factor series.+ adjust = Noneelse:endpoint_name = "daily"df = self.api.daily(ts_code=code, start_date=start_date, end_date=end_date)+ adjust = getattr(self.api, "adj_factor", None)if df is None or df.empty:logger.warning("tushare returned empty for %s via %s", code, endpoint_name)@@ -172,7 +179,25 @@ class DataLoader:ohlcv = df[["open", "high", "low", "close", "volume"]].dropna(subset=["open", "high", "low", "close"])- return ohlcv+ if adjust is None:+ return ohlcv++ try:+ factor = adjust(ts_code=code, start_date=start_date, end_date=end_date)+ except Exception as exc: # noqa: BLE001 - one bad fetch must not raise+ logger.warning("tushare adjustment-factor fetch failed for %s: %s", code, exc)+ factor = None+ adjusted = apply_qfq(ohlcv, factor)+ if adjusted is None:+ # Returning the raw frame here is what produced the defect: a+ # close-to-close return across an ex-date spans the mechanical gap,+ # measured at -47.2%% on 300750.SZ 2023-04-26 against a true +5.4%%.+ logger.warning(+ "tushare: no usable adjustment factors for %s — dropping the "+ "symbol rather than backtesting it on unadjusted prices",+ code,+ )+ return adjusteddef _merge_basic_fields(self,diff --git a/agent/src/tools/alpha_bench_tool.py b/agent/src/tools/alpha_bench_tool.pyindex d2ea6b867..1653db787 100644--- a/agent/src/tools/alpha_bench_tool.py+++ b/agent/src/tools/alpha_bench_tool.py@@ -41,6 +41,7 @@ from typing import Anyimport pandas as pd+from backtest.loaders.cn_adjust import apply_qfq as _apply_qfqfrom src.agent.tools import BaseToolfrom src.config.accessor import get_env_config@@ -325,19 +326,38 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:codes: list[str] = []constituent_source = "tushare index_weight"constituent_source_date: str | None = None+ membership: pd.DataFrame | None = Nonetry:+ # Reach back before ``start`` so the snapshot that was in force on the+ # first requested day is included; Tushare publishes month-end rosters.+ lookback = (pd.Timestamp(start) - pd.Timedelta(days=60)).strftime("%Y%m%d")weights = pro.index_weight(- index_code="399300.SZ", start_date=sd, end_date=ed+ index_code="399300.SZ", start_date=lookback, end_date=ed)if weights is not None and not weights.empty:- latest_date = weights["trade_date"].max()- constituent_source_date = str(latest_date)- codes = (- weights[weights["trade_date"] == latest_date]["con_code"]- .drop_duplicates()- .tolist()+ frame = weights.copy()+ frame["trade_date"] = pd.to_datetime(frame["trade_date"], errors="coerce")+ frame = frame.dropna(subset=["trade_date", "con_code"])+ constituent_source_date = str(weights["trade_date"].max())+ # Every name that was a member at any point in the window, so the+ # panel can carry a name that later left the index.+ codes = sorted(frame["con_code"].astype(str).unique())+ membership = (+ frame.assign(_member=True)+ .pivot_table(+ index="trade_date",+ columns="con_code",+ values="_member",+ aggfunc="first",+ )+ .notna()+ .sort_index()+ )+ logger.info(+ "csi300: %d names ever a member across %d roster snapshots",+ len(codes),+ len(membership),)- logger.info("csi300: %d constituents from index_weight @ %s", len(codes), latest_date)except Exception as exc: # noqa: BLE001logger.warning("csi300 index_weight failed (%s); using fallback list", exc)@@ -362,7 +382,9 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:if col in df.columns:df[col] = pd.to_numeric(df[col], errors="coerce")keep = [c for c in ("open", "high", "low", "close", "volume", "amount") if c in df.columns]- return code, df[keep].dropna(subset=["open", "high", "low", "close"])+ df = df[keep].dropna(subset=["open", "high", "low", "close"])+ factor = _retry(lambda: pro.adj_factor(ts_code=code, start_date=sd, end_date=ed))+ return code, _apply_qfq(df, factor)fetched: dict[str, pd.DataFrame] = {}with ThreadPoolExecutor(max_workers=_CSI300_FETCH_WORKERS) as pool:@@ -376,6 +398,26 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:if frame is not None and not frame.empty:fetched[code] = frame+ # A name with no usable adjustment factors is dropped rather than benched on+ # raw prices, so the drop has to be visible or it becomes its own silent bias.+ dropped = sorted(set(codes) - set(fetched))+ if not fetched:+ raise RuntimeError(+ "csi300: no symbol survived corporate-action adjustment — "+ "pro.adj_factor returned nothing usable for any of the "+ f"{len(codes)} names, which usually means the Tushare token lacks "+ "adj_factor permission. Benching on unadjusted prices is not an "+ "alternative: an ex-date injects a fabricated cross-sectional "+ "return, measured at -47.2% on 300750.SZ 2023-04-26."+ )+ if dropped:+ logger.warning(+ "csi300: dropped %d/%d name(s) with no usable adjustment factors: %s",+ len(dropped),+ len(codes),+ ", ".join(dropped[:10]) + ("..." if len(dropped) > 10 else ""),+ )+panel = _wide_from_fetched(fetched, include_amount=True)# CN equity vwap: Tushare ``amount`` is in 千元, ``volume`` in 手. True VWAP# = (amount * 1000 CNY) / (volume * 100 shares). Matches@@ -386,15 +428,38 @@ def _load_csi300_panel(start: str, end: str) -> dict[str, pd.DataFrame]:panel["vwap"] = safe_div(panel["amount"] * 1000.0, panel["volume"] * 100.0 + 1.0)++ # Restrict each date's cross-section to the names that were index members on+ # that date. Without this the panel carries today's roster back through the+ # whole window, so a name is only present because it survived to the end —+ # every IC is then measured on a set selected with hindsight.+ if membership is not None:+ mask = (+ membership.reindex(columns=panel["close"].columns)+ .reindex(index=panel["close"].index.union(membership.index))+ .ffill()+ .reindex(panel["close"].index)+ .bfill()+ .fillna(False)+ .astype(bool)+ )+ for key, frame in panel.items():+ if isinstance(frame, pd.DataFrame):+ panel[key] = frame.where(mask)+panel["_meta"] = {"universe": "csi300",- # A terminal snapshot is forward-looking relative to the start of the- # requested interval; the static fallback is survivor-selected too.- "survivorship_bias": True,+ # True only on the degraded path: the hand-picked fallback is a+ # survivor-selected static roster with no point-in-time membership.+ "survivorship_bias": membership is None,+ "pit_membership": membership is not None,"degraded": constituent_source != "tushare index_weight","constituent_source": constituent_source,"constituent_source_date": constituent_source_date,"constituent_count": len(codes),+ # Prices are corporate-action adjusted; raw pro.daily is not.+ "price_adjustment": "qfq",+ "dropped_unadjustable": len(dropped),}return paneldiff --git a/agent/tests/test_alpha_bench_qfq.py b/agent/tests/test_alpha_bench_qfq.pynew file mode 100644index 000000000..9ce33e747--- /dev/null+++ b/agent/tests/test_alpha_bench_qfq.py@@ -0,0 +1,134 @@+"""Regression: CSI300 bench prices must be corporate-action adjusted.++``pro.daily`` returns raw prices, so a close-to-close return taken across an+ex-date spans the mechanical drop of a split or bonus issue. Measured against+Tushare's own ``pct_chg`` over 2020-2024: 300750.SZ on 2023-04-26 reads -41.82%+raw against a true +5.40%, 300124.SZ -34.26% against -1.01%, 601012.SH -24.21%+against +6.46%. The error is always negative, so every cross-sectional IC the+bench reports carried a systematic contaminant, not noise.++These tests are offline; the live validation (adjusted return vs ``pct_chg``+within 1e-5, VWAP/close ratio preserved to 2e-16) is recorded in the audit doc.+"""++from __future__ import annotations++import pandas as pd+import pytest++from src.tools.alpha_bench_tool import _apply_qfq++_DATES = pd.to_datetime(["2023-04-24", "2023-04-25", "2023-04-26", "2023-04-27"])+++def _bars(closes):+ return pd.DataFrame(+ {+ "open": closes,+ "high": [c * 1.01 for c in closes],+ "low": [c * 0.99 for c in closes],+ "close": closes,+ "volume": [1000.0, 1000.0, 2000.0, 2000.0],+ "amount": [100.0, 100.0, 100.0, 100.0],+ },+ index=_DATES,+ )+++def _factors(values):+ return pd.DataFrame(+ {+ "trade_date": ["20230424", "20230425", "20230426", "20230427"],+ "adj_factor": values,+ }+ )+++class TestAdjustmentRemovesTheExDateJump:+ """A 2-for-1 split halves the raw price without changing the return."""++ # Price halves on the third bar. Tushare's adj_factor is cumulative and+ # rises across a corporate action (verified on 600519.SH 2022-06-30:+ # 7.4740 -> 7.5546), so a 2-for-1 split doubles it.+ _SPLIT_CLOSES = [200.0, 200.0, 100.0, 100.0]+ _SPLIT_FACTORS = [1.0, 1.0, 2.0, 2.0]++ def test_raw_prices_show_a_fabricated_fifty_percent_drop(self):+ raw = _bars(self._SPLIT_CLOSES)["close"].pct_change()++ assert raw.iloc[2] == pytest.approx(-0.5)++ def test_adjusted_prices_show_no_return_on_the_ex_date(self):+ adjusted = _apply_qfq(_bars(self._SPLIT_CLOSES), _factors(self._SPLIT_FACTORS))++ assert adjusted["close"].pct_change().iloc[2] == pytest.approx(0.0, abs=1e-12)++ def test_the_latest_bar_keeps_its_traded_price(self):+ # 前复权: the most recent bar is the anchor, so it is untouched.+ adjusted = _apply_qfq(_bars(self._SPLIT_CLOSES), _factors(self._SPLIT_FACTORS))++ assert adjusted["close"].iloc[-1] == pytest.approx(100.0)++ def test_ohlc_are_scaled_by_the_same_ratio(self):+ bars = _bars(self._SPLIT_CLOSES)+ adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))++ for column in ("open", "high", "low"):+ assert adjusted[column].iloc[0] == pytest.approx(bars[column].iloc[0] * 0.5)++ def test_volume_is_put_on_the_same_basis_as_price(self):+ # Pre-split bars are restated into post-split shares, so the count doubles.+ bars = _bars(self._SPLIT_CLOSES)+ adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))++ assert adjusted["volume"].iloc[0] == pytest.approx(bars["volume"].iloc[0] * 2.0)++ def test_amount_is_cash_and_is_left_alone(self):+ bars = _bars(self._SPLIT_CLOSES)+ adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))++ assert adjusted["amount"].tolist() == bars["amount"].tolist()++ def test_vwap_keeps_its_relationship_to_close(self):+ # The bench derives vwap from amount/volume, so an adjustment that moved+ # close without moving volume would silently decouple the two.+ bars = _bars(self._SPLIT_CLOSES)+ adjusted = _apply_qfq(bars, _factors(self._SPLIT_FACTORS))++ raw_ratio = (bars["amount"] * 1000 / (bars["volume"] * 100)) / bars["close"]+ adj_ratio = (+ adjusted["amount"] * 1000 / (adjusted["volume"] * 100)+ ) / adjusted["close"]+ pd.testing.assert_series_equal(raw_ratio, adj_ratio)+++class TestUnusableFactorsAreRefused:+ """Missing factors must drop the symbol, never fall back to raw prices."""++ _CLOSES = [100.0, 100.0, 100.0, 100.0]++ def test_none_is_refused(self):+ assert _apply_qfq(_bars(self._CLOSES), None) is None++ def test_an_empty_frame_is_refused(self):+ assert _apply_qfq(_bars(self._CLOSES), pd.DataFrame()) is None++ def test_a_frame_without_the_factor_column_is_refused(self):+ bad = pd.DataFrame({"trade_date": ["20230424"], "something_else": [1.0]})++ assert _apply_qfq(_bars(self._CLOSES), bad) is None++ def test_a_non_positive_factor_is_refused(self):+ assert _apply_qfq(_bars(self._CLOSES), _factors([1.0, 0.0, 1.0, 1.0])) is None++ def test_a_short_factor_series_is_forward_filled_not_refused(self):+ # Tushare returns one row per trading day; a trailing gap is normal and+ # must not cost the symbol its whole history.+ partial = pd.DataFrame(+ {"trade_date": ["20230424", "20230425"], "adj_factor": [2.0, 2.0]}+ )++ adjusted = _apply_qfq(_bars(self._CLOSES), partial)++ assert adjusted is not None+ assert adjusted["close"].notna().all()diff --git a/agent/tests/test_alpha_bench_universe_metadata.py b/agent/tests/test_alpha_bench_universe_metadata.pyindex 85f28c788..1b1605d2b 100644--- a/agent/tests/test_alpha_bench_universe_metadata.py+++ b/agent/tests/test_alpha_bench_universe_metadata.py@@ -42,6 +42,10 @@ class _FakeTusharePro:"""Return deterministic OHLCV data for every requested code."""return _daily_frame()+ def adj_factor(self, **_kwargs) -> pd.DataFrame:+ """Return a flat adjustment factor — no corporate action in the window."""+ return pd.DataFrame({"trade_date": ["20240102"], "adj_factor": [1.0]})+def _install_fake_tushare(monkeypatch: pytest.MonkeyPatch,@@ -60,10 +64,10 @@ def _install_fake_tushare()-def test_csi300_reports_terminal_index_weight_snapshot(+def test_csi300_reports_point_in_time_membership(monkeypatch: pytest.MonkeyPatch,) -> None:- """Healthy CSI300 loads disclose their terminal Tushare snapshot."""+ """A healthy CSI300 load carries every name that was ever a member."""weights = pd.DataFrame({"trade_date": ["20240102", "20240131", "20240131"],@@ -76,14 +80,42 @@ def test_csi300_reports_terminal_index_weight_snapshot(assert panel["_meta"] == {"universe": "csi300",- "survivorship_bias": True,+ # Membership is per date, so a name is no longer present merely because+ # it survived to the end of the window.+ "survivorship_bias": False,+ "pit_membership": True,"degraded": False,"constituent_source": "tushare index_weight","constituent_source_date": "20240131",- "constituent_count": 2,+ # The union of both snapshots, not the terminal roster.+ "constituent_count": 3,+ "price_adjustment": "qfq",+ "dropped_unadjustable": 0,}+def test_csi300_masks_a_name_outside_its_membership_window(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A name that left the index must be absent from later cross-sections."""+ weights = pd.DataFrame(+ {+ "trade_date": ["20240102", "20240102", "20240131"],+ # 600000.SH is a member on the first snapshot only.+ "con_code": ["600000.SH", "000001.SZ", "000001.SZ"],+ }+ )+ _install_fake_tushare(monkeypatch, _FakeTusharePro(weights))++ panel = tool._load_csi300_panel("2024-01-01", "2024-01-31")++ close = panel["close"]+ assert set(close.columns) == {"600000.SH", "000001.SZ"}+ # The fake loader returns a single bar dated 2024-01-02, when both were in.+ assert close.loc["2024-01-02", "600000.SH"] == 10.5+ assert close.loc["2024-01-02", "000001.SZ"] == 10.5++def test_csi300_reports_hand_picked_fallback(monkeypatch: pytest.MonkeyPatch,) -> None:@@ -99,11 +131,16 @@ def test_csi300_reports_hand_picked_fallback(assert panel["_meta"] == {"universe": "csi300",+ # The fallback is a static survivor-selected roster with no membership+ # history, so the bias disclosure stays on."survivorship_bias": True,+ "pit_membership": False,"degraded": True,"constituent_source": "hand-picked fallback","constituent_source_date": None,"constituent_count": 2,+ "price_adjustment": "qfq",+ "dropped_unadjustable": 0,}diff --git a/agent/tests/test_tushare_loader.py b/agent/tests/test_tushare_loader.pyindex 34db26d50..3b4718814 100644--- a/agent/tests/test_tushare_loader.py+++ b/agent/tests/test_tushare_loader.py@@ -135,12 +135,29 @@ def _make_ohlcv_df() -> pd.DataFrame:})+def _make_adj_df(factors=(1.0, 1.0, 2.0)) -> pd.DataFrame:+ """Build the adjustment-factor frame tushare pairs with the bars above.++ The default doubles on the last bar, i.e. a 2-for-1 split, so a test can+ tell an adjusted frame from a raw one.+ """+ return pd.DataFrame({+ "ts_code": ["X"] * 3,+ "trade_date": ["20250102", "20250103", "20250106"],+ "adj_factor": list(factors),+ })++class TestFetchDailyFrameRouting:"""Verify _fetch_daily_frame calls the correct tushare endpoint per symbol type."""def _make_loader(self) -> DataLoader:loader = object.__new__(DataLoader)loader.api = MagicMock()+ # Equities and funds are corporate-action adjusted before they are+ # returned, so the factor endpoints must answer with a real frame.+ loader.api.adj_factor.return_value = _make_adj_df((1.0, 1.0, 1.0))+ loader.api.fund_adj.return_value = _make_adj_df((1.0, 1.0, 1.0))return loaderdef test_stock_routes_to_daily(self) -> None:@@ -191,6 +208,45 @@ class TestFetchDailyFrameRouting:assert result is Noneloader.api.daily.assert_not_called()+ def test_stock_prices_are_corporate_action_adjusted(self) -> None:+ loader = self._make_loader()+ loader.api.daily.return_value = _make_ohlcv_df()+ loader.api.adj_factor.return_value = _make_adj_df((1.0, 1.0, 2.0))++ result = loader._fetch_daily_frame("000001.SZ", "20250102", "20250110")++ loader.api.adj_factor.assert_called_once()+ # Forward-adjusted to the last bar: earlier closes are halved, the last+ # keeps its traded price.+ assert result["close"].tolist() == [5.25, 5.5, 11.5]++ def test_etf_prices_are_corporate_action_adjusted(self) -> None:+ loader = self._make_loader()+ loader.api.fund_daily.return_value = _make_ohlcv_df()+ loader.api.fund_adj.return_value = _make_adj_df((1.0, 1.0, 2.0))++ result = loader._fetch_daily_frame("510050.SH", "20250102", "20250110")++ loader.api.fund_adj.assert_called_once()+ assert result["close"].tolist() == [5.25, 5.5, 11.5]++ def test_a_symbol_with_no_factors_is_dropped_not_returned_raw(self) -> None:+ # Falling back to raw prices is the defect this guard exists to stop.+ loader = self._make_loader()+ loader.api.daily.return_value = _make_ohlcv_df()+ loader.api.adj_factor.return_value = pd.DataFrame()++ assert loader._fetch_daily_frame("000001.SZ", "20250102", "20250110") is None++ def test_an_index_is_not_adjusted(self) -> None:+ loader = self._make_loader()+ loader.api.index_daily.return_value = _make_ohlcv_df()++ result = loader._fetch_daily_frame("000001.SH", "20250102", "20250110")++ loader.api.adj_factor.assert_not_called()+ assert result["close"].tolist() == [10.5, 11.0, 11.5]+def test_empty_result_warns(self) -> None:loader = self._make_loader()loader.api.daily.return_value = pd.DataFrame()