Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(autopilot): surface run_card validation evidence in link_autopilot_backtest

Lanre Shittuf70bb7706b2c
diff --git a/agent/src/hypotheses/registry.py b/agent/src/hypotheses/registry.py
index 0c9b1f0ad..1d72aa88f 100644
--- a/agent/src/hypotheses/registry.py
+++ b/agent/src/hypotheses/registry.py
@@ -286,6 +286,7 @@ class HypothesisRegistry:
run_card_path: str = "",
backtest_run_dir: str = "",
metrics: dict[str, Any] | None = None,
+ validation: dict[str, Any] | None = None,
notes: str = "",
) -> Hypothesis:
"""Link a run card or backtest artifact to a hypothesis.
@@ -295,6 +296,9 @@ class HypothesisRegistry:
run_card_path: Optional path to a run_card.json.
backtest_run_dir: Optional backtest run directory.
metrics: Optional metrics summary.
+ validation: Optional walk-forward/Monte-Carlo/bootstrap robustness
+ results, as written by write_run_card's top-level
+ "validation" key (a sibling of "metrics", not nested in it).
notes: Optional human note about the link.
Returns:
@@ -308,13 +312,16 @@ class HypothesisRegistry:
raise ValueError("run_card_path or backtest_run_dir is required")
records = self.list()
hyp = self._find_required(records, hypothesis_id)
- hyp.run_cards.append({
+ run_card_record: dict[str, Any] = {
"run_card_path": run_card_path,
"backtest_run_dir": backtest_run_dir,
"metrics": metrics or {},
"notes": notes,
"linked_at": _utc_now(),
- })
+ }
+ if validation is not None:
+ run_card_record["validation"] = validation
+ hyp.run_cards.append(run_card_record)
hyp.updated_at = _utc_now()
self._save(records)
return hyp
diff --git a/agent/src/tools/autopilot_tool.py b/agent/src/tools/autopilot_tool.py
index b0a489dc6..575bd9ed4 100644
--- a/agent/src/tools/autopilot_tool.py
+++ b/agent/src/tools/autopilot_tool.py
@@ -627,11 +627,16 @@ class LinkAutopilotBacktestTool(BaseTool):
metrics = {}
warning = "run_card.json had no 'metrics' object; linked with empty metrics"
+ validation = card.get("validation") if isinstance(card, dict) else None
+ if not isinstance(validation, dict):
+ validation = None
+
try:
hypothesis = HypothesisRegistry().link_backtest(
hypothesis_id,
backtest_run_dir=str(run_path),
metrics=metrics,
+ validation=validation,
notes=str(kwargs.get("notes", "")),
)
except KeyError:
@@ -658,6 +663,8 @@ class LinkAutopilotBacktestTool(BaseTool):
"then record_evidence / add_goal_evidence to close the loop."
),
}
+ if validation is not None:
+ payload["validation"] = validation
if warning:
payload["warning"] = warning
return _ok(payload)
diff --git a/agent/tests/test_autopilot_backtest_validation_evidence.py b/agent/tests/test_autopilot_backtest_validation_evidence.py
new file mode 100644
index 000000000..b978d7fdc
--- /dev/null
+++ b/agent/tests/test_autopilot_backtest_validation_evidence.py
@@ -0,0 +1,82 @@
+"""link_autopilot_backtest must surface run_card.json's validation section.
+
+write_run_card writes walk-forward / Monte-Carlo / bootstrap robustness
+results as "validation", a top-level sibling of "metrics" (see
+backtest/run_card.py). LinkAutopilotBacktestTool only ever read "metrics",
+so an agent linking a backtest to a hypothesis received the performance
+numbers while silently losing the evidence needed to judge whether that
+performance is real or overfit.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from src.hypotheses import HypothesisRegistry
+from src.tools.autopilot_tool import LinkAutopilotBacktestTool, _get_hypothesis
+
+
+def isolated_env(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+ monkeypatch.setenv("VIBE_TRADING_HYPOTHESES_PATH", str(tmp_path / "hypotheses.json"))
+ monkeypatch.setenv("VIBE_TRADING_ALLOWED_RUN_ROOTS", str(tmp_path))
+ return tmp_path
+
+
+def _make_hypothesis() -> str:
+ registry = HypothesisRegistry()
+ return registry.create(
+ title="A-share momentum",
+ thesis="Cross-sectional momentum persists in large caps.",
+ universe="CSI 300",
+ signal_definition="rank(returns_20d) top decile",
+ data_sources=["akshare"],
+ ).hypothesis_id
+
+
+def _run_dir(base: Path) -> Path:
+ d = base / "runs" / "autopilot_test"
+ d.mkdir(parents=True, exist_ok=True)
+ return d
+
+
+def _write_run_card_with_validation(run_dir: Path) -> dict:
+ # Mirrors write_run_card's real shape: validation is a sibling of
+ # metrics, not nested inside it.
+ validation = {
+ "walk_forward": {
+ "consistency_rate": 0.2,
+ "n_windows": 5,
+ "profitable_windows": 1,
+ }
+ }
+ card = {
+ "schema_version": 1,
+ "run_dir": str(run_dir),
+ "metrics": {"sharpe": 1.9, "total_return": 0.42, "max_drawdown": -0.11},
+ "validation": validation,
+ }
+ (run_dir / "run_card.json").write_text(json.dumps(card, ensure_ascii=False), encoding="utf-8")
+ return validation
+
+
+def test_link_autopilot_backtest_surfaces_validation_section(isolated_env: Path) -> None:
+ hyp_id = _make_hypothesis()
+ run_dir = _run_dir(isolated_env)
+ validation = _write_run_card_with_validation(run_dir)
+
+ result = json.loads(LinkAutopilotBacktestTool().execute(hypothesis_id=hyp_id, run_dir=str(run_dir)))
+
+ assert result["status"] == "ok"
+ assert (
+ "validation" in result
+ ), f"link_autopilot_backtest dropped run_card.json's validation section from its result: {result}"
+ assert result["validation"] == validation
+
+ reloaded = _get_hypothesis(hyp_id)
+ card = reloaded.run_cards[0]
+ assert "validation" in card, "the persisted hypothesis run_card also lost the validation section"
+ assert card["validation"] == validation