"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(autopilot): surface run_card validation evidence in link_autopilot_backtest
Lanre Shittuf70bb7706b2c
diff --git a/agent/src/hypotheses/registry.py b/agent/src/hypotheses/registry.pyindex 0c9b1f0ad..1d72aa88f 100644--- a/agent/src/hypotheses/registry.py+++ b/agent/src/hypotheses/registry.py@@ -286,6 +286,7 @@ class HypothesisRegistry:run_card_path: str = "",backtest_run_dir: str = "",metrics: dict[str, Any] | None = None,+ validation: dict[str, Any] | None = None,notes: str = "",) -> Hypothesis:"""Link a run card or backtest artifact to a hypothesis.@@ -295,6 +296,9 @@ class HypothesisRegistry:run_card_path: Optional path to a run_card.json.backtest_run_dir: Optional backtest run directory.metrics: Optional metrics summary.+ validation: Optional walk-forward/Monte-Carlo/bootstrap robustness+ results, as written by write_run_card's top-level+ "validation" key (a sibling of "metrics", not nested in it).notes: Optional human note about the link.Returns:@@ -308,13 +312,16 @@ class HypothesisRegistry:raise ValueError("run_card_path or backtest_run_dir is required")records = self.list()hyp = self._find_required(records, hypothesis_id)- hyp.run_cards.append({+ run_card_record: dict[str, Any] = {"run_card_path": run_card_path,"backtest_run_dir": backtest_run_dir,"metrics": metrics or {},"notes": notes,"linked_at": _utc_now(),- })+ }+ if validation is not None:+ run_card_record["validation"] = validation+ hyp.run_cards.append(run_card_record)hyp.updated_at = _utc_now()self._save(records)return hypdiff --git a/agent/src/tools/autopilot_tool.py b/agent/src/tools/autopilot_tool.pyindex b0a489dc6..575bd9ed4 100644--- a/agent/src/tools/autopilot_tool.py+++ b/agent/src/tools/autopilot_tool.py@@ -627,11 +627,16 @@ class LinkAutopilotBacktestTool(BaseTool):metrics = {}warning = "run_card.json had no 'metrics' object; linked with empty metrics"+ validation = card.get("validation") if isinstance(card, dict) else None+ if not isinstance(validation, dict):+ validation = None+try:hypothesis = HypothesisRegistry().link_backtest(hypothesis_id,backtest_run_dir=str(run_path),metrics=metrics,+ validation=validation,notes=str(kwargs.get("notes", "")),)except KeyError:@@ -658,6 +663,8 @@ class LinkAutopilotBacktestTool(BaseTool):"then record_evidence / add_goal_evidence to close the loop."),}+ if validation is not None:+ payload["validation"] = validationif warning:payload["warning"] = warningreturn _ok(payload)diff --git a/agent/tests/test_autopilot_backtest_validation_evidence.py b/agent/tests/test_autopilot_backtest_validation_evidence.pynew file mode 100644index 000000000..b978d7fdc--- /dev/null+++ b/agent/tests/test_autopilot_backtest_validation_evidence.py@@ -0,0 +1,82 @@+"""link_autopilot_backtest must surface run_card.json's validation section.++write_run_card writes walk-forward / Monte-Carlo / bootstrap robustness+results as "validation", a top-level sibling of "metrics" (see+backtest/run_card.py). LinkAutopilotBacktestTool only ever read "metrics",+so an agent linking a backtest to a hypothesis received the performance+numbers while silently losing the evidence needed to judge whether that+performance is real or overfit.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from src.hypotheses import HypothesisRegistry+from src.tools.autopilot_tool import LinkAutopilotBacktestTool, _get_hypothesis+++def isolated_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setenv("VIBE_TRADING_HYPOTHESES_PATH", str(tmp_path / "hypotheses.json"))+ monkeypatch.setenv("VIBE_TRADING_ALLOWED_RUN_ROOTS", str(tmp_path))+ return tmp_path+++def _make_hypothesis() -> str:+ registry = HypothesisRegistry()+ return registry.create(+ title="A-share momentum",+ thesis="Cross-sectional momentum persists in large caps.",+ universe="CSI 300",+ signal_definition="rank(returns_20d) top decile",+ data_sources=["akshare"],+ ).hypothesis_id+++def _run_dir(base: Path) -> Path:+ d = base / "runs" / "autopilot_test"+ d.mkdir(parents=True, exist_ok=True)+ return d+++def _write_run_card_with_validation(run_dir: Path) -> dict:+ # Mirrors write_run_card's real shape: validation is a sibling of+ # metrics, not nested inside it.+ validation = {+ "walk_forward": {+ "consistency_rate": 0.2,+ "n_windows": 5,+ "profitable_windows": 1,+ }+ }+ card = {+ "schema_version": 1,+ "run_dir": str(run_dir),+ "metrics": {"sharpe": 1.9, "total_return": 0.42, "max_drawdown": -0.11},+ "validation": validation,+ }+ (run_dir / "run_card.json").write_text(json.dumps(card, ensure_ascii=False), encoding="utf-8")+ return validation+++def test_link_autopilot_backtest_surfaces_validation_section(isolated_env: Path) -> None:+ hyp_id = _make_hypothesis()+ run_dir = _run_dir(isolated_env)+ validation = _write_run_card_with_validation(run_dir)++ result = json.loads(LinkAutopilotBacktestTool().execute(hypothesis_id=hyp_id, run_dir=str(run_dir)))++ assert result["status"] == "ok"+ assert (+ "validation" in result+ ), f"link_autopilot_backtest dropped run_card.json's validation section from its result: {result}"+ assert result["validation"] == validation++ reloaded = _get_hypothesis(hyp_id)+ card = reloaded.run_cards[0]+ assert "validation" in card, "the persisted hypothesis run_card also lost the validation section"+ assert card["validation"] == validation