"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(backtest): warn about settings unused by selected engine (#1947)
Rudy Mizrahi Celekli6b98caa86fe5
diff --git a/README.md b/README.mdindex f93ff36f3..35692df02 100644--- a/README.md+++ b/README.md@@ -1391,6 +1391,14 @@ Saves write the `channels.<name>` section of `~/.vibe-trading/agent.json` atomic### Strategy & Backtesting+Daily-engine backtests report engine-specific unused settings in the run-card+warnings. For example, crypto reads `maker_rate` and `taker_rate`, so supplying+the A-share `commission_rate` setting warns instead of silently implying that+it configured crypto fees. Shared runner/loader settings and keys read by a+composite engine's selected sub-engines remain accepted. This conservative+source-reader check does not trace conditional branches inside an engine and+does not change options-engine configuration handling.+Average holding period includes completed same-bar trades (zero bars). Theexisting `avg_holding_days` compatibility key reports the engine's holding-barmeasure; it is not a conversion to calendar days.diff --git a/agent/backtest/runner.py b/agent/backtest/runner.pyindex 9b29ac0bc..414435398 100644--- a/agent/backtest/runner.py+++ b/agent/backtest/runner.py@@ -16,6 +16,7 @@ import jsonimport loggingimport sysfrom dataclasses import dataclass+from functools import lru_cachefrom pathlib import Pathfrom typing import Any, Dict, Iterable, List, Literal, Optional@@ -1385,6 +1386,47 @@ def _unrecognized_config_warning(config: dict) -> str | None:)+@lru_cache(maxsize=1)+def _engine_specific_config_keys() -> frozenset[str]:+ """Keys read by engines, excluding keys with shared runner/loader readers."""+ root = Path(__file__).parent+ shared = set(BacktestConfigSchema.model_fields)+ for path in [*root.glob("*.py"), *(root / "loaders").glob("*.py")]:+ shared.update(_ast_config_keys(path))+ # These helpers support several market engines, including delegated rules.+ shared.update(_ast_config_keys(root / "engines" / "_market_hooks.py"))+ engine_keys = set()+ for path in (root / "engines").glob("*.py"):+ engine_keys.update(_ast_config_keys(path))+ return frozenset(engine_keys - shared)+++def _unused_engine_config_warning(config: dict, engine: Any) -> str | None:+ """Warn about known engine-only keys with no reader in the selected rules."""+ selected = set()+ pending = [engine]+ seen = set()+ while pending:+ current = pending.pop()+ if id(current) in seen:+ continue+ seen.add(id(current))+ pending.extend(getattr(current, "_rule_engines", {}).values())+ for cls in type(current).__mro__:+ if cls.__module__.startswith("backtest.engines."):+ path = inspect.getsourcefile(cls)+ if path:+ selected.update(_ast_config_keys(Path(path)))+ unused = sorted(set(config) & (_engine_specific_config_keys() - selected))+ if not unused:+ return None+ return (+ f"Config keys unused by selected {type(engine).__name__}: "+ + ", ".join(repr(key) for key in unused)+ + ". These settings do not affect this engine; its defaults still apply."+ )++def main(run_dir: Path) -> None:"""Load config, fetch data, run the selected backtest engine.@@ -1687,6 +1729,18 @@ def _annualisation_bars(def _create_market_engine(source: str, config: dict, codes: List[str]):+ """Select real market rules and surface engine-specific unused settings."""+ engine = _select_market_engine(source, config, codes)+ warning = _unused_engine_config_warning(config, engine)+ if warning:+ previous = config.get("_run_card_unrecognized_config_warning")+ config["_run_card_unrecognized_config_warning"] = (+ f"{previous} {warning}" if previous else warning+ )+ return engine+++def _select_market_engine(source: str, config: dict, codes: List[str]):"""Create the appropriate market engine based on data source and market type.Routing priority:diff --git a/agent/tests/test_backtest_engine_unused_config.py b/agent/tests/test_backtest_engine_unused_config.pynew file mode 100644index 000000000..183526395--- /dev/null+++ b/agent/tests/test_backtest_engine_unused_config.py@@ -0,0 +1,71 @@+"""Engine-specific configuration warnings use the actual selected rule books."""++import json+from pathlib import Path++import pandas as pd++from backtest.engines.crypto import CryptoEngine+from backtest.runner import _AutoLoader, _create_market_engine+++def test_crypto_unused_commission_is_visible_in_real_run_card(tmp_path: Path) -> None:+ config = {+ "codes": ["BTC-USDT"],+ "source": "okx",+ "start_date": "2024-01-01",+ "end_date": "2024-01-06",+ "initial_cash": 1000,+ "commission_rate": 0.001,+ }+ engine = _create_market_engine("okx", config, config["codes"])+ assert isinstance(engine, CryptoEngine)+ # This key does not configure crypto: the genuine fee remains the default.+ assert engine.taker_rate == 0.0005+ dates = pd.date_range("2024-01-01", periods=6)+ bars = pd.DataFrame(+ {"open": 100.0, "high": 101.0, "low": 99.0, "close": 100.0, "volume": 1000.0},+ index=dates,+ )++ class SignalEngine:+ def generate(self, data_map: dict[str, pd.DataFrame]) -> dict[str, pd.Series]:+ return {"BTC-USDT": pd.Series(0.0, index=dates)}++ engine.run_backtest(+ config, _AutoLoader({"BTC-USDT": bars}), SignalEngine(), tmp_path+ )+ card = json.loads((tmp_path / "run_card.json").read_text())+ assert any(+ "commission_rate" in warning and "unused" in warning+ for warning in card["warnings"]+ )+++def test_active_crypto_and_shared_settings_are_not_reported_unused() -> None:+ config = {+ "codes": ["BTC-USDT"],+ "taker_rate": 0.001,+ "maker_rate": 0.002,+ "constraints": [],+ "model_training_cutoff": "2023-12-31",+ }+ engine = _create_market_engine("okx", config, config["codes"])+ assert engine.taker_rate == 0.001+ assert engine.maker_rate == 0.002+ assert not config.get("_run_card_unrecognized_config_warning")+++def test_equity_commission_is_read_by_the_selected_engine() -> None:+ config = {"codes": ["000001.SZ"], "commission_rate": 0.001}+ engine = _create_market_engine("tushare", config, config["codes"])+ assert engine.commission_rate == 0.001+ assert not config.get("_run_card_unrecognized_config_warning")+++def test_composite_includes_config_read_by_its_actual_subengines() -> None:+ config = {"codes": ["AAPL", "BTC-USDT"], "slippage_us": 0.001, "maker_rate": 0.002}+ engine = _create_market_engine("auto", config, config["codes"])+ assert engine._rule_engines["crypto"].maker_rate == 0.002+ assert engine._rule_engines["us_equity"].slippage_us == 0.001+ assert not config.get("_run_card_unrecognized_config_warning")