"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(backtest): add hash-only run card traces
lorenzozaneed25d9f93c6d1
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.pyindex a7b9e8c4d..2f8b8a2fe 100644--- a/agent/backtest/engines/base.py+++ b/agent/backtest/engines/base.py@@ -18,6 +18,7 @@ from abc import ABC, abstractmethodfrom collections import Counterfrom collections.abc import Iterablefrom dataclasses import dataclass, replace+from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any, Callable, Dict, List, Optional@@ -897,6 +898,7 @@ class BaseEngine(ABC):Returns:Metrics dictionary."""+ trace_started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")codes = config.get("codes", [])interval = config.get("interval", "1D")extra_fields = config.get("extra_fields") or None@@ -1157,6 +1159,16 @@ class BaseEngine(ABC):data_sources=_run_card_data_sources(config, loader),strategy_path=run_dir / "code" / "signal_engine.py",warnings=card_warnings or None,+ tool_traces=[+ {+ "tool": "backtest",+ "args": config,+ "started_at": trace_started_at,+ "ended_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),+ "status": "ok",+ "result": m,+ }+ ],)# Print scalar metrics (skip nested dicts for JSON compat).diff --git a/agent/backtest/engines/options_portfolio.py b/agent/backtest/engines/options_portfolio.pyindex cdff18f23..e30438b02 100644--- a/agent/backtest/engines/options_portfolio.py+++ b/agent/backtest/engines/options_portfolio.py@@ -21,6 +21,7 @@ the per-leg vol every pricing site must agree on.import jsonimport mathimport sys+from datetime import datetime, timezonefrom pathlib import Pathfrom typing import Any, Dict, List, Optional@@ -209,6 +210,7 @@ def run_options_backtest(Raises:SystemExit: When no data is fetched."""+ trace_started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")codes = config.get("codes", [])start_date = config.get("start_date", "")end_date = config.get("end_date", "")@@ -628,6 +630,16 @@ def run_options_backtest(data_sources=[str(getattr(loader, "name", config.get("source", "")))],strategy_path=run_dir / "code" / "signal_engine.py",warnings=config.get("content_filter_warnings") or None,+ tool_traces=[+ {+ "tool": "backtest",+ "args": config,+ "started_at": trace_started_at,+ "ended_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),+ "status": "ok",+ "result": metrics,+ }+ ],)print(json.dumps(metrics, indent=2, allow_nan=False))diff --git a/agent/backtest/run_card.py b/agent/backtest/run_card.pyindex 7d48e9dc7..5170b2868 100644--- a/agent/backtest/run_card.py+++ b/agent/backtest/run_card.py@@ -10,7 +10,7 @@ from pathlib import Pathfrom typing import Any, Mapping, Sequence-SCHEMA_VERSION = "0.1"+SCHEMA_VERSION = "1.0"# Largest single structured metric the card will carry, counted in BYTES of the# JSON the card actually writes (indented, sorted, non-ASCII kept literal). The# card is an at-a-glance artefact read on every run, so a structured metric that@@ -50,6 +50,7 @@ def write_run_card(strategy_path: Path | None = None,warnings: Sequence[str] | None = None,artifact_refs: Sequence[Mapping[str, Any]] | None = None,+ tool_traces: Sequence[Mapping[str, Any]] | None = None,) -> dict[str, Any]:"""Write JSON and Markdown run cards for a backtest run.@@ -63,6 +64,8 @@ def write_run_card(strategy_path: Optional strategy source file to hash for reproducibility.warnings: Optional warnings to include in the card.artifact_refs: Optional IRR-AGL artifact references.+ tool_traces: Optional tool events. Arguments and results are hashed+ before serialization.Returns:The run card payload written to ``run_card.json``.@@ -93,6 +96,12 @@ def write_run_card(normalized_refs = _normalize_artifact_refs(artifact_refs)if normalized_refs:card["artifact_refs"] = normalized_refs+ normalized_traces = _normalize_tool_traces(tool_traces)+ if normalized_traces:+ card["tool_traces"] = normalized_traces+ citations = _metric_citations(metrics, card["artifacts"])+ if citations:+ card["citations"] = citationsstructured = _structured_metrics(metrics)if structured:card["structured_metrics"] = structured@@ -272,6 +281,53 @@ def _normalize_artifact_refs(artifact_refs: Sequence[Mapping[str, Any]] | None)return refs+def _normalize_tool_traces(tool_traces: Sequence[Mapping[str, Any]] | None) -> list[dict[str, Any]]:+ traces = []+ for trace in tool_traces or []:+ args = trace["args"]+ result = trace["result"]+ if not isinstance(args, Mapping) or not isinstance(result, Mapping):+ raise TypeError("tool trace args and result must be mappings")+ traces.append(+ {+ "tool": str(trace["tool"]),+ "args_hash": _tool_payload_hash(args),+ "started_at": str(trace["started_at"]),+ "ended_at": str(trace["ended_at"]),+ "status": str(trace["status"]),+ "result_hash": _tool_payload_hash(result),+ }+ )+ return traces+++def _tool_payload_hash(value: Mapping[str, Any]) -> str:+ payload = json.dumps(+ value,+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ )+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()+++def _metric_citations(+ metrics: Mapping[str, Any], artifacts: Sequence[Mapping[str, Any]]+) -> list[dict[str, str]]:+ metrics_artifact = next(+ (artifact for artifact in artifacts if artifact.get("path") == "artifacts/metrics.csv"),+ None,+ )+ if metrics_artifact is None:+ return []+ return [+ {"metric": key, "artifact_id": "artifacts/metrics.csv"}+ for key, value in metrics.items()+ if key != "validation" and _is_scalar(value)+ ]++def _render_markdown(card: Mapping[str, Any]) -> str:lines = ["# Backtest Run Card",diff --git a/agent/tests/test_engine_robustness.py b/agent/tests/test_engine_robustness.pyindex 4bc82225c..8d7befce6 100644--- a/agent/tests/test_engine_robustness.py+++ b/agent/tests/test_engine_robustness.py@@ -9,9 +9,10 @@from __future__ import annotations+import hashlibimport json+from datetime import datetime, timezonefrom pathlib import Path-from types import SimpleNamespacefrom typing import Dictfrom unittest.mock import patch@@ -237,15 +238,16 @@ class TestSymbolIsolation:monkeypatch.setattr("backtest.benchmark.resolve_benchmark", fake_resolve_benchmark)engine = ChinaAEngine({"initial_cash": 1_000_000})+ config = {+ "codes": ["000001.SZ"],+ "start_date": "2024-04-01",+ "end_date": "2024-04-30",+ "source": "tushare",+ "benchmark": "000300.SH",+ "initial_cash": 1_000_000,+ }metrics = engine.run_backtest(- {- "codes": ["000001.SZ"],- "start_date": "2024-04-01",- "end_date": "2024-04-30",- "source": "tushare",- "benchmark": "000300.SH",- "initial_cash": 1_000_000,- },+ config,FakeLoader(),SignalEngine(),tmp_path,@@ -257,7 +259,44 @@ class TestSymbolIsolation:run_card_path = tmp_path / "run_card.json"assert run_card_path.exists()run_card = json.loads(run_card_path.read_text(encoding="utf-8"))- assert run_card["schema_version"] == "0.1"+ assert run_card["schema_version"] == "1.0"+ assert len(run_card["tool_traces"]) == 1+ assert run_card["tool_traces"][0]["tool"] == "backtest"+ assert run_card["tool_traces"][0]["status"] == "ok"+ started_at = datetime.fromisoformat(+ run_card["tool_traces"][0]["started_at"].replace("Z", "+00:00")+ )+ ended_at = datetime.fromisoformat(+ run_card["tool_traces"][0]["ended_at"].replace("Z", "+00:00")+ )+ assert started_at.tzinfo == timezone.utc+ assert ended_at.tzinfo == timezone.utc+ assert started_at <= ended_at+ assert (+ run_card["tool_traces"][0]["args_hash"]+ == hashlib.sha256(+ json.dumps(+ config,+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ ).encode("utf-8")+ ).hexdigest()+ )+ assert (+ run_card["tool_traces"][0]["result_hash"]+ == hashlib.sha256(+ json.dumps(+ metrics,+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ ).encode("utf-8")+ ).hexdigest()+ )+ assert run_card["citations"]assert run_card["backtest"]["codes"] == ["000001.SZ"]assert run_card["data_sources"] == ["tushare"]assert run_card["metrics"]["benchmark_return"] == 0.00495diff --git a/agent/tests/test_run_card.py b/agent/tests/test_run_card.pyindex 4125091ae..8c8f90259 100644--- a/agent/tests/test_run_card.py+++ b/agent/tests/test_run_card.py@@ -4,6 +4,7 @@ from __future__ import annotationsimport hashlibimport json+from datetime import datetime, timezonefrom pathlib import Pathimport pandas as pd@@ -115,7 +116,7 @@ def test_json_and_markdown_files_are_written(tmp_path: Path) -> None:markdown = md_path.read_text(encoding="utf-8")assert loaded == card- assert loaded["schema_version"] == "0.1"+ assert loaded["schema_version"] == "1.0"assert loaded["generated_at"].endswith("Z")assert loaded["metrics"] == {"max_drawdown": -0.08, "sharpe": 1.23}assert loaded["validation"] == {"consistency_rate": 0.8, "n_windows": 5}@@ -296,15 +297,16 @@ def test_options_backtest_writes_run_card(tmp_path: Path) -> None:},]- run_options_backtest(- {- "codes": ["SPY"],- "start_date": "2025-01-01",- "end_date": "2025-01-06",- "source": "yfinance",- "engine": "options",- "initial_cash": 100_000,- },+ config = {+ "codes": ["SPY"],+ "start_date": "2025-01-01",+ "end_date": "2025-01-06",+ "source": "yfinance",+ "engine": "options",+ "initial_cash": 100_000,+ }+ metrics = run_options_backtest(+ config,FakeLoader(),SignalEngine(),tmp_path,@@ -313,6 +315,47 @@ def test_options_backtest_writes_run_card(tmp_path: Path) -> None:card = json.loads((tmp_path / "run_card.json").read_text(encoding="utf-8"))assert card["backtest"]["engine"] == "options"assert card["data_sources"] == ["yfinance"]+ assert len(card["tool_traces"]) == 1+ assert card["tool_traces"][0]["tool"] == "backtest"+ assert card["tool_traces"][0]["status"] == "ok"+ started_at = datetime.fromisoformat(+ card["tool_traces"][0]["started_at"].replace("Z", "+00:00")+ )+ ended_at = datetime.fromisoformat(+ card["tool_traces"][0]["ended_at"].replace("Z", "+00:00")+ )+ assert started_at.tzinfo == timezone.utc+ assert ended_at.tzinfo == timezone.utc+ assert started_at <= ended_at+ assert (+ card["tool_traces"][0]["args_hash"]+ == hashlib.sha256(+ json.dumps(+ config,+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ ).encode("utf-8")+ ).hexdigest()+ )+ assert (+ card["tool_traces"][0]["result_hash"]+ == hashlib.sha256(+ json.dumps(+ metrics,+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ ).encode("utf-8")+ ).hexdigest()+ )+ assert card["citations"]+ assert all(+ citation["artifact_id"] == "artifacts/metrics.csv"+ for citation in card["citations"]+ )assert "greeks.csv" in {Path(artifact["path"]).name for artifact in card["artifacts"]}assert (tmp_path / "run_card.md").exists()diff --git a/agent/tests/test_run_card_trust_layer_v1.py b/agent/tests/test_run_card_trust_layer_v1.pynew file mode 100644index 000000000..9e6eb740d--- /dev/null+++ b/agent/tests/test_run_card_trust_layer_v1.py@@ -0,0 +1,111 @@+import hashlib+import json++from backtest.run_card import write_run_card+++def test_run_card_v1_hashes_tool_inputs_and_cites_metric_artifact(tmp_path) -> None:+ run_dir = tmp_path / "run"+ metrics_dir = run_dir / "artifacts"+ metrics_dir.mkdir(parents=True)+ (metrics_dir / "metrics.csv").write_text("sharpe\n1.23\n", encoding="utf-8")+ trace = {+ "tool": "backtest",+ "args": {"run_dir": "/private/local/run", "api_key": "trace-secret"},+ "started_at": "2026-09-27T00:00:00Z",+ "ended_at": "2026-09-27T00:00:01Z",+ "status": "ok",+ "result": {"sharpe": 1.23, "payload": "raw-result"},+ }++ card = write_run_card(+ run_dir,+ {"codes": ["BIL"]},+ {"sharpe": 1.23},+ tool_traces=[trace],+ )++ trace_record = card["tool_traces"][0]+ assert card["schema_version"] == "1.0"+ assert set(trace_record) == {+ "tool",+ "args_hash",+ "started_at",+ "ended_at",+ "status",+ "result_hash",+ }+ assert trace_record["tool"] == "backtest"+ assert trace_record["started_at"] == trace["started_at"]+ assert trace_record["ended_at"] == trace["ended_at"]+ assert trace_record["status"] == "ok"+ assert len(trace_record["args_hash"]) == 64+ assert len(trace_record["result_hash"]) == 64+ for key in ("args", "result"):+ canonical = json.dumps(+ trace[key],+ sort_keys=True,+ default=str,+ separators=(",", ":"),+ ensure_ascii=False,+ )+ expected_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()+ assert trace_record[f"{key}_hash"] == expected_hash+ assert card["citations"] == [+ {"metric": "sharpe", "artifact_id": "artifacts/metrics.csv"}+ ]++ serialized = json.dumps(card)+ on_disk = json.loads((run_dir / "run_card.json").read_text(encoding="utf-8"))+ assert on_disk == card+ assert "/private/local/run" not in serialized+ assert "trace-secret" not in serialized+ assert "raw-result" not in serialized++ same_payload_different_key_order = {+ **trace,+ "args": {"api_key": "trace-secret", "run_dir": "/private/local/run"},+ "result": {"payload": "raw-result", "sharpe": 1.23},+ }+ reordered_card = write_run_card(+ tmp_path / "reordered",+ {"codes": ["BIL"]},+ {"sharpe": 1.23},+ tool_traces=[same_payload_different_key_order],+ )+ assert reordered_card["tool_traces"][0]["args_hash"] == trace_record["args_hash"]+ assert (+ reordered_card["tool_traces"][0]["result_hash"] == trace_record["result_hash"]+ )++ same_shape_trace = {+ **trace,+ "args": {**trace["args"], "_private_state": "first"},+ "result": {**trace["result"], "_private_state": "first"},+ }+ same_shape_card = write_run_card(+ tmp_path / "same-shape",+ {"codes": ["BIL"]},+ {"sharpe": 1.23},+ tool_traces=[same_shape_trace],+ )+ changed_private_trace = {+ **same_shape_trace,+ "args": {**same_shape_trace["args"], "_private_state": "different"},+ "result": {**same_shape_trace["result"], "_private_state": "different"},+ }+ changed_card = write_run_card(+ tmp_path / "other",+ {"codes": ["BIL"]},+ {"sharpe": 1.23},+ tool_traces=[changed_private_trace],+ )+ assert "citations" not in changed_card+ assert (+ changed_card["tool_traces"][0]["args_hash"]+ != same_shape_card["tool_traces"][0]["args_hash"]+ )+ assert (+ changed_card["tool_traces"][0]["result_hash"]+ != same_shape_card["tool_traces"][0]["result_hash"]+ )