"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(backtest): warn on config keys nothing on the path reads (#1935)
Yufeng He7b3db6b2adea
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.pyindex 010bc64a9..7c13b709c 100644--- a/agent/backtest/engines/base.py+++ b/agent/backtest/engines/base.py@@ -1174,6 +1174,8 @@ class BaseEngine(ABC):card_warnings.append(config["_run_card_caliber_warning"])if config.get("_run_card_annualisation_warning"):card_warnings.append(config["_run_card_annualisation_warning"])+ if config.get("_run_card_unrecognized_config_warning"):+ card_warnings.append(config["_run_card_unrecognized_config_warning"])if benchmark_warning:card_warnings.append(benchmark_warning)from backtest.run_card import write_run_carddiff --git a/agent/backtest/runner.py b/agent/backtest/runner.pyindex 614e20106..98c79d5c9 100644--- a/agent/backtest/runner.py+++ b/agent/backtest/runner.py@@ -1302,6 +1302,89 @@ def _maybe_inject_fundamentals_for_factor_panel(# --- Main entry ---+_KNOWN_CONFIG_KEYS: frozenset | None = None+++def _ast_config_keys(path: Path) -> set:+ """Collect string literals read off the backtest config dict in a module.++ Matches ``config.get("key")``, ``config["key"]`` and the same two forms on+ ``self.config``. Anything more dynamic is out of scope for this lint: it+ only feeds a warning, never a rejection.+ """+ keys = set()+ tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))+ for node in ast.walk(tree):+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):+ if node.func.attr != "get" or not node.args:+ continue+ owner = node.func.value+ owner_is_config = (isinstance(owner, ast.Name) and owner.id == "config") or (+ isinstance(owner, ast.Attribute) and owner.attr == "config"+ )+ if owner_is_config and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):+ keys.add(node.args[0].value)+ elif isinstance(node, ast.Subscript):+ owner = node.value+ owner_is_config = (isinstance(owner, ast.Name) and owner.id == "config") or (+ isinstance(owner, ast.Attribute) and owner.attr == "config"+ )+ sub = node.slice+ if owner_is_config and isinstance(sub, ast.Constant) and isinstance(sub.value, str):+ keys.add(sub.value)+ return keys+++def _known_config_keys() -> frozenset:+ """Config keys understood anywhere on the backtest path: schema fields plus+ whatever the engines, market hooks and loaders actually read."""+ global _KNOWN_CONFIG_KEYS+ if _KNOWN_CONFIG_KEYS is None:+ known = set(BacktestConfigSchema.model_fields)+ engines_dir = Path(__file__).parent / "engines"+ loaders_dir = Path(__file__).parent / "loaders"+ modules = (+ sorted(Path(__file__).parent.glob("*.py"))+ + sorted(engines_dir.glob("*.py"))+ + sorted(loaders_dir.glob("*.py"))+ )+ for module in modules:+ known |= _ast_config_keys(module)+ _KNOWN_CONFIG_KEYS = frozenset(known)+ return _KNOWN_CONFIG_KEYS+++def _unrecognized_config_keys(config: dict) -> list[str]:+ """Config keys nothing on the path reads (misspellings included).++ ``BacktestConfigSchema`` allows extras so engine-specific keys flow+ through; the cost is that a misspelled key (``initial_capital`` for+ ``initial_cash``) is silently dropped and the run proceeds on defaults.+ Underscore-prefixed keys are runner internals, not user input.+ """+ known = _known_config_keys()+ return sorted(k for k in config if not k.startswith("_") and k not in known)+++def _unrecognized_config_warning(config: dict) -> str | None:+ import difflib++ unknown = _unrecognized_config_keys(config)+ if not unknown:+ return None+ known = sorted(_known_config_keys())+ parts = []+ for key in unknown:+ guesses = difflib.get_close_matches(key, known, n=2, cutoff=0.6)+ hint = f" (did you mean {', or '.join(repr(g) for g in guesses)}?)" if guesses else ""+ parts.append(f"{key!r}{hint}")+ return (+ "Unrecognized config keys were silently ignored: "+ + "; ".join(parts)+ + ". The run proceeded on defaults for anything they were meant to set."+ )++def main(run_dir: Path) -> None:"""Load config, fetch data, run the selected backtest engine.@@ -1356,6 +1439,8 @@ def main(run_dir: Path) -> None:sys.exit(1)config = raw_config+ if keys_warning := _unrecognized_config_warning(config):+ config["_run_card_unrecognized_config_warning"] = keys_warningsource = config.get("source", "tushare")codes = config.get("codes", [])diff --git a/agent/tests/test_backtest_unrecognized_config_keys.py b/agent/tests/test_backtest_unrecognized_config_keys.pynew file mode 100644index 000000000..2860448aa--- /dev/null+++ b/agent/tests/test_backtest_unrecognized_config_keys.py@@ -0,0 +1,67 @@+"""Regression tests for unrecognized backtest config keys (#1934).++An agent-written ``initial_capital`` used to validate cleanly (extras are+allowed so engine keys flow through) and then be silently dropped by the+engine, so the run proceeded at 10x the intended capital while reporting+confident metrics. The lint is deliberately path-wide: a key is flagged only+when nothing on the backtest path reads it, so engine-namespaced keys like+``commission_rate`` (read by china_a, not crypto) are not false-flagged.+"""++from __future__ import annotations++from backtest.runner import _known_config_keys, _unrecognized_config_warning+++def test_misspelled_capital_and_commission_keys_are_flagged() -> None:+ config = {+ "codes": ["BTC-USDT"],+ "start_date": "2024-01-01",+ "end_date": "2024-12-31",+ "initial_capital": 100000,+ "commission_rate": 0.001,+ "source": "okx",+ "interval": "1D",+ }+ warning = _unrecognized_config_warning(config)+ assert warning is not None+ assert "'initial_capital'" in warning+ # The misspelling is pointed at the key the engine actually reads.+ assert "'initial_cash'" in warning+ # commission_rate is a real china_a engine key, so the path-wide lint must+ # not flag it on a crypto run (per-engine namespacing is a separate, wider+ # change); this case pins that conservative contract down.+ assert "'commission_rate'" not in warning+++def test_engine_keys_and_runner_internals_are_not_flagged() -> None:+ config = {+ "codes": ["BTC-USDT"],+ "start_date": "2024-01-01",+ "end_date": "2024-12-31",+ "initial_cash": 100000,+ "maker_rate": 0.001,+ "taker_rate": 0.002,+ "leverage": 2.0,+ "slippage": 0.0005,+ "funding_mode": "data",+ "source": "okx",+ "engine": "crypto",+ "interval": "1D",+ "_run_card_effective_sources": {"BTC-USDT": "okx"},+ }+ assert _unrecognized_config_warning(config) is None+++def test_known_keys_cover_engine_and_loader_reads() -> None:+ known = _known_config_keys()+ for key in ("initial_cash", "maker_rate", "taker_rate", "slippage", "funding_rate"):+ assert key in known+++def test_shared_path_config_is_not_reported_as_ignored() -> None:+ """Constraints and provenance have real readers outside engine modules."""+ assert _unrecognized_config_warning({+ "constraints": [{"type": "turnover_aware", "max_turnover": 0.1}],+ "model_training_cutoff": "2023-12-31",+ }) is None