Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(backtest): warn about settings unused by selected engine (#1947)

Rudy Mizrahi Celekli6b98caa86fe5
diff --git a/README.md b/README.md
index f93ff36f3..35692df02 100644
--- a/README.md
+++ b/README.md
@@ -1391,6 +1391,14 @@ Saves write the `channels.<name>` section of `~/.vibe-trading/agent.json` atomic
### Strategy & Backtesting
+Daily-engine backtests report engine-specific unused settings in the run-card
+warnings. For example, crypto reads `maker_rate` and `taker_rate`, so supplying
+the A-share `commission_rate` setting warns instead of silently implying that
+it configured crypto fees. Shared runner/loader settings and keys read by a
+composite engine's selected sub-engines remain accepted. This conservative
+source-reader check does not trace conditional branches inside an engine and
+does not change options-engine configuration handling.
+
Average holding period includes completed same-bar trades (zero bars). The
existing `avg_holding_days` compatibility key reports the engine's holding-bar
measure; it is not a conversion to calendar days.
diff --git a/agent/backtest/runner.py b/agent/backtest/runner.py
index 9b29ac0bc..414435398 100644
--- a/agent/backtest/runner.py
+++ b/agent/backtest/runner.py
@@ -16,6 +16,7 @@ import json
import logging
import sys
from dataclasses import dataclass
+from functools import lru_cache
from pathlib import Path
from typing import Any, Dict, Iterable, List, Literal, Optional
@@ -1385,6 +1386,47 @@ def _unrecognized_config_warning(config: dict) -> str | None:
)
+@lru_cache(maxsize=1)
+def _engine_specific_config_keys() -> frozenset[str]:
+ """Keys read by engines, excluding keys with shared runner/loader readers."""
+ root = Path(__file__).parent
+ shared = set(BacktestConfigSchema.model_fields)
+ for path in [*root.glob("*.py"), *(root / "loaders").glob("*.py")]:
+ shared.update(_ast_config_keys(path))
+ # These helpers support several market engines, including delegated rules.
+ shared.update(_ast_config_keys(root / "engines" / "_market_hooks.py"))
+ engine_keys = set()
+ for path in (root / "engines").glob("*.py"):
+ engine_keys.update(_ast_config_keys(path))
+ return frozenset(engine_keys - shared)
+
+
+def _unused_engine_config_warning(config: dict, engine: Any) -> str | None:
+ """Warn about known engine-only keys with no reader in the selected rules."""
+ selected = set()
+ pending = [engine]
+ seen = set()
+ while pending:
+ current = pending.pop()
+ if id(current) in seen:
+ continue
+ seen.add(id(current))
+ pending.extend(getattr(current, "_rule_engines", {}).values())
+ for cls in type(current).__mro__:
+ if cls.__module__.startswith("backtest.engines."):
+ path = inspect.getsourcefile(cls)
+ if path:
+ selected.update(_ast_config_keys(Path(path)))
+ unused = sorted(set(config) & (_engine_specific_config_keys() - selected))
+ if not unused:
+ return None
+ return (
+ f"Config keys unused by selected {type(engine).__name__}: "
+ + ", ".join(repr(key) for key in unused)
+ + ". These settings do not affect this engine; its defaults still apply."
+ )
+
+
def main(run_dir: Path) -> None:
"""Load config, fetch data, run the selected backtest engine.
@@ -1687,6 +1729,18 @@ def _annualisation_bars(
def _create_market_engine(source: str, config: dict, codes: List[str]):
+ """Select real market rules and surface engine-specific unused settings."""
+ engine = _select_market_engine(source, config, codes)
+ warning = _unused_engine_config_warning(config, engine)
+ if warning:
+ previous = config.get("_run_card_unrecognized_config_warning")
+ config["_run_card_unrecognized_config_warning"] = (
+ f"{previous} {warning}" if previous else warning
+ )
+ return engine
+
+
+def _select_market_engine(source: str, config: dict, codes: List[str]):
"""Create the appropriate market engine based on data source and market type.
Routing priority:
diff --git a/agent/tests/test_backtest_engine_unused_config.py b/agent/tests/test_backtest_engine_unused_config.py
new file mode 100644
index 000000000..183526395
--- /dev/null
+++ b/agent/tests/test_backtest_engine_unused_config.py
@@ -0,0 +1,71 @@
+"""Engine-specific configuration warnings use the actual selected rule books."""
+
+import json
+from pathlib import Path
+
+import pandas as pd
+
+from backtest.engines.crypto import CryptoEngine
+from backtest.runner import _AutoLoader, _create_market_engine
+
+
+def test_crypto_unused_commission_is_visible_in_real_run_card(tmp_path: Path) -> None:
+ config = {
+ "codes": ["BTC-USDT"],
+ "source": "okx",
+ "start_date": "2024-01-01",
+ "end_date": "2024-01-06",
+ "initial_cash": 1000,
+ "commission_rate": 0.001,
+ }
+ engine = _create_market_engine("okx", config, config["codes"])
+ assert isinstance(engine, CryptoEngine)
+ # This key does not configure crypto: the genuine fee remains the default.
+ assert engine.taker_rate == 0.0005
+ dates = pd.date_range("2024-01-01", periods=6)
+ bars = pd.DataFrame(
+ {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 1000.0},
+ index=dates,
+ )
+
+ class SignalEngine:
+ def generate(self, data_map: dict[str, pd.DataFrame]) -> dict[str, pd.Series]:
+ return {"BTC-USDT": pd.Series(0.0, index=dates)}
+
+ engine.run_backtest(
+ config, _AutoLoader({"BTC-USDT": bars}), SignalEngine(), tmp_path
+ )
+ card = json.loads((tmp_path / "run_card.json").read_text())
+ assert any(
+ "commission_rate" in warning and "unused" in warning
+ for warning in card["warnings"]
+ )
+
+
+def test_active_crypto_and_shared_settings_are_not_reported_unused() -> None:
+ config = {
+ "codes": ["BTC-USDT"],
+ "taker_rate": 0.001,
+ "maker_rate": 0.002,
+ "constraints": [],
+ "model_training_cutoff": "2023-12-31",
+ }
+ engine = _create_market_engine("okx", config, config["codes"])
+ assert engine.taker_rate == 0.001
+ assert engine.maker_rate == 0.002
+ assert not config.get("_run_card_unrecognized_config_warning")
+
+
+def test_equity_commission_is_read_by_the_selected_engine() -> None:
+ config = {"codes": ["000001.SZ"], "commission_rate": 0.001}
+ engine = _create_market_engine("tushare", config, config["codes"])
+ assert engine.commission_rate == 0.001
+ assert not config.get("_run_card_unrecognized_config_warning")
+
+
+def test_composite_includes_config_read_by_its_actual_subengines() -> None:
+ config = {"codes": ["AAPL", "BTC-USDT"], "slippage_us": 0.001, "maker_rate": 0.002}
+ engine = _create_market_engine("auto", config, config["codes"])
+ assert engine._rule_engines["crypto"].maker_rate == 0.002
+ assert engine._rule_engines["us_equity"].slippage_us == 0.001
+ assert not config.get("_run_card_unrecognized_config_warning")