Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(backtest): warn on config keys nothing on the path reads (#1935)

Yufeng He7b3db6b2adea
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.py
index 010bc64a9..7c13b709c 100644
--- a/agent/backtest/engines/base.py
+++ b/agent/backtest/engines/base.py
@@ -1174,6 +1174,8 @@ class BaseEngine(ABC):
card_warnings.append(config["_run_card_caliber_warning"])
if config.get("_run_card_annualisation_warning"):
card_warnings.append(config["_run_card_annualisation_warning"])
+ if config.get("_run_card_unrecognized_config_warning"):
+ card_warnings.append(config["_run_card_unrecognized_config_warning"])
if benchmark_warning:
card_warnings.append(benchmark_warning)
from backtest.run_card import write_run_card
diff --git a/agent/backtest/runner.py b/agent/backtest/runner.py
index 614e20106..98c79d5c9 100644
--- a/agent/backtest/runner.py
+++ b/agent/backtest/runner.py
@@ -1302,6 +1302,89 @@ def _maybe_inject_fundamentals_for_factor_panel(
# --- Main entry ---
+_KNOWN_CONFIG_KEYS: frozenset | None = None
+
+
+def _ast_config_keys(path: Path) -> set:
+ """Collect string literals read off the backtest config dict in a module.
+
+ Matches ``config.get("key")``, ``config["key"]`` and the same two forms on
+ ``self.config``. Anything more dynamic is out of scope for this lint: it
+ only feeds a warning, never a rejection.
+ """
+ keys = set()
+ tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
+ for node in ast.walk(tree):
+ if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
+ if node.func.attr != "get" or not node.args:
+ continue
+ owner = node.func.value
+ owner_is_config = (isinstance(owner, ast.Name) and owner.id == "config") or (
+ isinstance(owner, ast.Attribute) and owner.attr == "config"
+ )
+ if owner_is_config and isinstance(node.args[0], ast.Constant) and isinstance(node.args[0].value, str):
+ keys.add(node.args[0].value)
+ elif isinstance(node, ast.Subscript):
+ owner = node.value
+ owner_is_config = (isinstance(owner, ast.Name) and owner.id == "config") or (
+ isinstance(owner, ast.Attribute) and owner.attr == "config"
+ )
+ sub = node.slice
+ if owner_is_config and isinstance(sub, ast.Constant) and isinstance(sub.value, str):
+ keys.add(sub.value)
+ return keys
+
+
+def _known_config_keys() -> frozenset:
+ """Config keys understood anywhere on the backtest path: schema fields plus
+ whatever the engines, market hooks and loaders actually read."""
+ global _KNOWN_CONFIG_KEYS
+ if _KNOWN_CONFIG_KEYS is None:
+ known = set(BacktestConfigSchema.model_fields)
+ engines_dir = Path(__file__).parent / "engines"
+ loaders_dir = Path(__file__).parent / "loaders"
+ modules = (
+ sorted(Path(__file__).parent.glob("*.py"))
+ + sorted(engines_dir.glob("*.py"))
+ + sorted(loaders_dir.glob("*.py"))
+ )
+ for module in modules:
+ known |= _ast_config_keys(module)
+ _KNOWN_CONFIG_KEYS = frozenset(known)
+ return _KNOWN_CONFIG_KEYS
+
+
+def _unrecognized_config_keys(config: dict) -> list[str]:
+ """Config keys nothing on the path reads (misspellings included).
+
+ ``BacktestConfigSchema`` allows extras so engine-specific keys flow
+ through; the cost is that a misspelled key (``initial_capital`` for
+ ``initial_cash``) is silently dropped and the run proceeds on defaults.
+ Underscore-prefixed keys are runner internals, not user input.
+ """
+ known = _known_config_keys()
+ return sorted(k for k in config if not k.startswith("_") and k not in known)
+
+
+def _unrecognized_config_warning(config: dict) -> str | None:
+ import difflib
+
+ unknown = _unrecognized_config_keys(config)
+ if not unknown:
+ return None
+ known = sorted(_known_config_keys())
+ parts = []
+ for key in unknown:
+ guesses = difflib.get_close_matches(key, known, n=2, cutoff=0.6)
+ hint = f" (did you mean {', or '.join(repr(g) for g in guesses)}?)" if guesses else ""
+ parts.append(f"{key!r}{hint}")
+ return (
+ "Unrecognized config keys were silently ignored: "
+ + "; ".join(parts)
+ + ". The run proceeded on defaults for anything they were meant to set."
+ )
+
+
def main(run_dir: Path) -> None:
"""Load config, fetch data, run the selected backtest engine.
@@ -1356,6 +1439,8 @@ def main(run_dir: Path) -> None:
sys.exit(1)
config = raw_config
+ if keys_warning := _unrecognized_config_warning(config):
+ config["_run_card_unrecognized_config_warning"] = keys_warning
source = config.get("source", "tushare")
codes = config.get("codes", [])
diff --git a/agent/tests/test_backtest_unrecognized_config_keys.py b/agent/tests/test_backtest_unrecognized_config_keys.py
new file mode 100644
index 000000000..2860448aa
--- /dev/null
+++ b/agent/tests/test_backtest_unrecognized_config_keys.py
@@ -0,0 +1,67 @@
+"""Regression tests for unrecognized backtest config keys (#1934).
+
+An agent-written ``initial_capital`` used to validate cleanly (extras are
+allowed so engine keys flow through) and then be silently dropped by the
+engine, so the run proceeded at 10x the intended capital while reporting
+confident metrics. The lint is deliberately path-wide: a key is flagged only
+when nothing on the backtest path reads it, so engine-namespaced keys like
+``commission_rate`` (read by china_a, not crypto) are not false-flagged.
+"""
+
+from __future__ import annotations
+
+from backtest.runner import _known_config_keys, _unrecognized_config_warning
+
+
+def test_misspelled_capital_and_commission_keys_are_flagged() -> None:
+ config = {
+ "codes": ["BTC-USDT"],
+ "start_date": "2024-01-01",
+ "end_date": "2024-12-31",
+ "initial_capital": 100000,
+ "commission_rate": 0.001,
+ "source": "okx",
+ "interval": "1D",
+ }
+ warning = _unrecognized_config_warning(config)
+ assert warning is not None
+ assert "'initial_capital'" in warning
+ # The misspelling is pointed at the key the engine actually reads.
+ assert "'initial_cash'" in warning
+ # commission_rate is a real china_a engine key, so the path-wide lint must
+ # not flag it on a crypto run (per-engine namespacing is a separate, wider
+ # change); this case pins that conservative contract down.
+ assert "'commission_rate'" not in warning
+
+
+def test_engine_keys_and_runner_internals_are_not_flagged() -> None:
+ config = {
+ "codes": ["BTC-USDT"],
+ "start_date": "2024-01-01",
+ "end_date": "2024-12-31",
+ "initial_cash": 100000,
+ "maker_rate": 0.001,
+ "taker_rate": 0.002,
+ "leverage": 2.0,
+ "slippage": 0.0005,
+ "funding_mode": "data",
+ "source": "okx",
+ "engine": "crypto",
+ "interval": "1D",
+ "_run_card_effective_sources": {"BTC-USDT": "okx"},
+ }
+ assert _unrecognized_config_warning(config) is None
+
+
+def test_known_keys_cover_engine_and_loader_reads() -> None:
+ known = _known_config_keys()
+ for key in ("initial_cash", "maker_rate", "taker_rate", "slippage", "funding_rate"):
+ assert key in known
+
+
+def test_shared_path_config_is_not_reported_as_ignored() -> None:
+ """Constraints and provenance have real readers outside engine modules."""
+ assert _unrecognized_config_warning({
+ "constraints": [{"type": "turnover_aware", "max_turnover": 0.1}],
+ "model_training_cutoff": "2023-12-31",
+ }) is None