Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(backtest): add hash-only run card traces

lorenzozaneed25d9f93c6d1
diff --git a/agent/backtest/engines/base.py b/agent/backtest/engines/base.py
index a7b9e8c4d..2f8b8a2fe 100644
--- a/agent/backtest/engines/base.py
+++ b/agent/backtest/engines/base.py
@@ -18,6 +18,7 @@ from abc import ABC, abstractmethod
from collections import Counter
from collections.abc import Iterable
from dataclasses import dataclass, replace
+from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
@@ -897,6 +898,7 @@ class BaseEngine(ABC):
Returns:
Metrics dictionary.
"""
+ trace_started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
codes = config.get("codes", [])
interval = config.get("interval", "1D")
extra_fields = config.get("extra_fields") or None
@@ -1157,6 +1159,16 @@ class BaseEngine(ABC):
data_sources=_run_card_data_sources(config, loader),
strategy_path=run_dir / "code" / "signal_engine.py",
warnings=card_warnings or None,
+ tool_traces=[
+ {
+ "tool": "backtest",
+ "args": config,
+ "started_at": trace_started_at,
+ "ended_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+ "status": "ok",
+ "result": m,
+ }
+ ],
)
# Print scalar metrics (skip nested dicts for JSON compat).
diff --git a/agent/backtest/engines/options_portfolio.py b/agent/backtest/engines/options_portfolio.py
index cdff18f23..e30438b02 100644
--- a/agent/backtest/engines/options_portfolio.py
+++ b/agent/backtest/engines/options_portfolio.py
@@ -21,6 +21,7 @@ the per-leg vol every pricing site must agree on.
import json
import math
import sys
+from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
@@ -209,6 +210,7 @@ def run_options_backtest(
Raises:
SystemExit: When no data is fetched.
"""
+ trace_started_at = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
codes = config.get("codes", [])
start_date = config.get("start_date", "")
end_date = config.get("end_date", "")
@@ -628,6 +630,16 @@ def run_options_backtest(
data_sources=[str(getattr(loader, "name", config.get("source", "")))],
strategy_path=run_dir / "code" / "signal_engine.py",
warnings=config.get("content_filter_warnings") or None,
+ tool_traces=[
+ {
+ "tool": "backtest",
+ "args": config,
+ "started_at": trace_started_at,
+ "ended_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+ "status": "ok",
+ "result": metrics,
+ }
+ ],
)
print(json.dumps(metrics, indent=2, allow_nan=False))
diff --git a/agent/backtest/run_card.py b/agent/backtest/run_card.py
index 7d48e9dc7..5170b2868 100644
--- a/agent/backtest/run_card.py
+++ b/agent/backtest/run_card.py
@@ -10,7 +10,7 @@ from pathlib import Path
from typing import Any, Mapping, Sequence
-SCHEMA_VERSION = "0.1"
+SCHEMA_VERSION = "1.0"
# Largest single structured metric the card will carry, counted in BYTES of the
# JSON the card actually writes (indented, sorted, non-ASCII kept literal). The
# card is an at-a-glance artefact read on every run, so a structured metric that
@@ -50,6 +50,7 @@ def write_run_card(
strategy_path: Path | None = None,
warnings: Sequence[str] | None = None,
artifact_refs: Sequence[Mapping[str, Any]] | None = None,
+ tool_traces: Sequence[Mapping[str, Any]] | None = None,
) -> dict[str, Any]:
"""Write JSON and Markdown run cards for a backtest run.
@@ -63,6 +64,8 @@ def write_run_card(
strategy_path: Optional strategy source file to hash for reproducibility.
warnings: Optional warnings to include in the card.
artifact_refs: Optional IRR-AGL artifact references.
+ tool_traces: Optional tool events. Arguments and results are hashed
+ before serialization.
Returns:
The run card payload written to ``run_card.json``.
@@ -93,6 +96,12 @@ def write_run_card(
normalized_refs = _normalize_artifact_refs(artifact_refs)
if normalized_refs:
card["artifact_refs"] = normalized_refs
+ normalized_traces = _normalize_tool_traces(tool_traces)
+ if normalized_traces:
+ card["tool_traces"] = normalized_traces
+ citations = _metric_citations(metrics, card["artifacts"])
+ if citations:
+ card["citations"] = citations
structured = _structured_metrics(metrics)
if structured:
card["structured_metrics"] = structured
@@ -272,6 +281,53 @@ def _normalize_artifact_refs(artifact_refs: Sequence[Mapping[str, Any]] | None)
return refs
+def _normalize_tool_traces(tool_traces: Sequence[Mapping[str, Any]] | None) -> list[dict[str, Any]]:
+ traces = []
+ for trace in tool_traces or []:
+ args = trace["args"]
+ result = trace["result"]
+ if not isinstance(args, Mapping) or not isinstance(result, Mapping):
+ raise TypeError("tool trace args and result must be mappings")
+ traces.append(
+ {
+ "tool": str(trace["tool"]),
+ "args_hash": _tool_payload_hash(args),
+ "started_at": str(trace["started_at"]),
+ "ended_at": str(trace["ended_at"]),
+ "status": str(trace["status"]),
+ "result_hash": _tool_payload_hash(result),
+ }
+ )
+ return traces
+
+
+def _tool_payload_hash(value: Mapping[str, Any]) -> str:
+ payload = json.dumps(
+ value,
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ )
+ return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def _metric_citations(
+ metrics: Mapping[str, Any], artifacts: Sequence[Mapping[str, Any]]
+) -> list[dict[str, str]]:
+ metrics_artifact = next(
+ (artifact for artifact in artifacts if artifact.get("path") == "artifacts/metrics.csv"),
+ None,
+ )
+ if metrics_artifact is None:
+ return []
+ return [
+ {"metric": key, "artifact_id": "artifacts/metrics.csv"}
+ for key, value in metrics.items()
+ if key != "validation" and _is_scalar(value)
+ ]
+
+
def _render_markdown(card: Mapping[str, Any]) -> str:
lines = [
"# Backtest Run Card",
diff --git a/agent/tests/test_engine_robustness.py b/agent/tests/test_engine_robustness.py
index 4bc82225c..8d7befce6 100644
--- a/agent/tests/test_engine_robustness.py
+++ b/agent/tests/test_engine_robustness.py
@@ -9,9 +9,10 @@
from __future__ import annotations
+import hashlib
import json
+from datetime import datetime, timezone
from pathlib import Path
-from types import SimpleNamespace
from typing import Dict
from unittest.mock import patch
@@ -237,15 +238,16 @@ class TestSymbolIsolation:
monkeypatch.setattr("backtest.benchmark.resolve_benchmark", fake_resolve_benchmark)
engine = ChinaAEngine({"initial_cash": 1_000_000})
+ config = {
+ "codes": ["000001.SZ"],
+ "start_date": "2024-04-01",
+ "end_date": "2024-04-30",
+ "source": "tushare",
+ "benchmark": "000300.SH",
+ "initial_cash": 1_000_000,
+ }
metrics = engine.run_backtest(
- {
- "codes": ["000001.SZ"],
- "start_date": "2024-04-01",
- "end_date": "2024-04-30",
- "source": "tushare",
- "benchmark": "000300.SH",
- "initial_cash": 1_000_000,
- },
+ config,
FakeLoader(),
SignalEngine(),
tmp_path,
@@ -257,7 +259,44 @@ class TestSymbolIsolation:
run_card_path = tmp_path / "run_card.json"
assert run_card_path.exists()
run_card = json.loads(run_card_path.read_text(encoding="utf-8"))
- assert run_card["schema_version"] == "0.1"
+ assert run_card["schema_version"] == "1.0"
+ assert len(run_card["tool_traces"]) == 1
+ assert run_card["tool_traces"][0]["tool"] == "backtest"
+ assert run_card["tool_traces"][0]["status"] == "ok"
+ started_at = datetime.fromisoformat(
+ run_card["tool_traces"][0]["started_at"].replace("Z", "+00:00")
+ )
+ ended_at = datetime.fromisoformat(
+ run_card["tool_traces"][0]["ended_at"].replace("Z", "+00:00")
+ )
+ assert started_at.tzinfo == timezone.utc
+ assert ended_at.tzinfo == timezone.utc
+ assert started_at <= ended_at
+ assert (
+ run_card["tool_traces"][0]["args_hash"]
+ == hashlib.sha256(
+ json.dumps(
+ config,
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ ).encode("utf-8")
+ ).hexdigest()
+ )
+ assert (
+ run_card["tool_traces"][0]["result_hash"]
+ == hashlib.sha256(
+ json.dumps(
+ metrics,
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ ).encode("utf-8")
+ ).hexdigest()
+ )
+ assert run_card["citations"]
assert run_card["backtest"]["codes"] == ["000001.SZ"]
assert run_card["data_sources"] == ["tushare"]
assert run_card["metrics"]["benchmark_return"] == 0.00495
diff --git a/agent/tests/test_run_card.py b/agent/tests/test_run_card.py
index 4125091ae..8c8f90259 100644
--- a/agent/tests/test_run_card.py
+++ b/agent/tests/test_run_card.py
@@ -4,6 +4,7 @@ from __future__ import annotations
import hashlib
import json
+from datetime import datetime, timezone
from pathlib import Path
import pandas as pd
@@ -115,7 +116,7 @@ def test_json_and_markdown_files_are_written(tmp_path: Path) -> None:
markdown = md_path.read_text(encoding="utf-8")
assert loaded == card
- assert loaded["schema_version"] == "0.1"
+ assert loaded["schema_version"] == "1.0"
assert loaded["generated_at"].endswith("Z")
assert loaded["metrics"] == {"max_drawdown": -0.08, "sharpe": 1.23}
assert loaded["validation"] == {"consistency_rate": 0.8, "n_windows": 5}
@@ -296,15 +297,16 @@ def test_options_backtest_writes_run_card(tmp_path: Path) -> None:
},
]
- run_options_backtest(
- {
- "codes": ["SPY"],
- "start_date": "2025-01-01",
- "end_date": "2025-01-06",
- "source": "yfinance",
- "engine": "options",
- "initial_cash": 100_000,
- },
+ config = {
+ "codes": ["SPY"],
+ "start_date": "2025-01-01",
+ "end_date": "2025-01-06",
+ "source": "yfinance",
+ "engine": "options",
+ "initial_cash": 100_000,
+ }
+ metrics = run_options_backtest(
+ config,
FakeLoader(),
SignalEngine(),
tmp_path,
@@ -313,6 +315,47 @@ def test_options_backtest_writes_run_card(tmp_path: Path) -> None:
card = json.loads((tmp_path / "run_card.json").read_text(encoding="utf-8"))
assert card["backtest"]["engine"] == "options"
assert card["data_sources"] == ["yfinance"]
+ assert len(card["tool_traces"]) == 1
+ assert card["tool_traces"][0]["tool"] == "backtest"
+ assert card["tool_traces"][0]["status"] == "ok"
+ started_at = datetime.fromisoformat(
+ card["tool_traces"][0]["started_at"].replace("Z", "+00:00")
+ )
+ ended_at = datetime.fromisoformat(
+ card["tool_traces"][0]["ended_at"].replace("Z", "+00:00")
+ )
+ assert started_at.tzinfo == timezone.utc
+ assert ended_at.tzinfo == timezone.utc
+ assert started_at <= ended_at
+ assert (
+ card["tool_traces"][0]["args_hash"]
+ == hashlib.sha256(
+ json.dumps(
+ config,
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ ).encode("utf-8")
+ ).hexdigest()
+ )
+ assert (
+ card["tool_traces"][0]["result_hash"]
+ == hashlib.sha256(
+ json.dumps(
+ metrics,
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ ).encode("utf-8")
+ ).hexdigest()
+ )
+ assert card["citations"]
+ assert all(
+ citation["artifact_id"] == "artifacts/metrics.csv"
+ for citation in card["citations"]
+ )
assert "greeks.csv" in {Path(artifact["path"]).name for artifact in card["artifacts"]}
assert (tmp_path / "run_card.md").exists()
diff --git a/agent/tests/test_run_card_trust_layer_v1.py b/agent/tests/test_run_card_trust_layer_v1.py
new file mode 100644
index 000000000..9e6eb740d
--- /dev/null
+++ b/agent/tests/test_run_card_trust_layer_v1.py
@@ -0,0 +1,111 @@
+import hashlib
+import json
+
+from backtest.run_card import write_run_card
+
+
+def test_run_card_v1_hashes_tool_inputs_and_cites_metric_artifact(tmp_path) -> None:
+ run_dir = tmp_path / "run"
+ metrics_dir = run_dir / "artifacts"
+ metrics_dir.mkdir(parents=True)
+ (metrics_dir / "metrics.csv").write_text("sharpe\n1.23\n", encoding="utf-8")
+ trace = {
+ "tool": "backtest",
+ "args": {"run_dir": "/private/local/run", "api_key": "trace-secret"},
+ "started_at": "2026-09-27T00:00:00Z",
+ "ended_at": "2026-09-27T00:00:01Z",
+ "status": "ok",
+ "result": {"sharpe": 1.23, "payload": "raw-result"},
+ }
+
+ card = write_run_card(
+ run_dir,
+ {"codes": ["BIL"]},
+ {"sharpe": 1.23},
+ tool_traces=[trace],
+ )
+
+ trace_record = card["tool_traces"][0]
+ assert card["schema_version"] == "1.0"
+ assert set(trace_record) == {
+ "tool",
+ "args_hash",
+ "started_at",
+ "ended_at",
+ "status",
+ "result_hash",
+ }
+ assert trace_record["tool"] == "backtest"
+ assert trace_record["started_at"] == trace["started_at"]
+ assert trace_record["ended_at"] == trace["ended_at"]
+ assert trace_record["status"] == "ok"
+ assert len(trace_record["args_hash"]) == 64
+ assert len(trace_record["result_hash"]) == 64
+ for key in ("args", "result"):
+ canonical = json.dumps(
+ trace[key],
+ sort_keys=True,
+ default=str,
+ separators=(",", ":"),
+ ensure_ascii=False,
+ )
+ expected_hash = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+ assert trace_record[f"{key}_hash"] == expected_hash
+ assert card["citations"] == [
+ {"metric": "sharpe", "artifact_id": "artifacts/metrics.csv"}
+ ]
+
+ serialized = json.dumps(card)
+ on_disk = json.loads((run_dir / "run_card.json").read_text(encoding="utf-8"))
+ assert on_disk == card
+ assert "/private/local/run" not in serialized
+ assert "trace-secret" not in serialized
+ assert "raw-result" not in serialized
+
+ same_payload_different_key_order = {
+ **trace,
+ "args": {"api_key": "trace-secret", "run_dir": "/private/local/run"},
+ "result": {"payload": "raw-result", "sharpe": 1.23},
+ }
+ reordered_card = write_run_card(
+ tmp_path / "reordered",
+ {"codes": ["BIL"]},
+ {"sharpe": 1.23},
+ tool_traces=[same_payload_different_key_order],
+ )
+ assert reordered_card["tool_traces"][0]["args_hash"] == trace_record["args_hash"]
+ assert (
+ reordered_card["tool_traces"][0]["result_hash"] == trace_record["result_hash"]
+ )
+
+ same_shape_trace = {
+ **trace,
+ "args": {**trace["args"], "_private_state": "first"},
+ "result": {**trace["result"], "_private_state": "first"},
+ }
+ same_shape_card = write_run_card(
+ tmp_path / "same-shape",
+ {"codes": ["BIL"]},
+ {"sharpe": 1.23},
+ tool_traces=[same_shape_trace],
+ )
+ changed_private_trace = {
+ **same_shape_trace,
+ "args": {**same_shape_trace["args"], "_private_state": "different"},
+ "result": {**same_shape_trace["result"], "_private_state": "different"},
+ }
+ changed_card = write_run_card(
+ tmp_path / "other",
+ {"codes": ["BIL"]},
+ {"sharpe": 1.23},
+ tool_traces=[changed_private_trace],
+ )
+ assert "citations" not in changed_card
+ assert (
+ changed_card["tool_traces"][0]["args_hash"]
+ != same_shape_card["tool_traces"][0]["args_hash"]
+ )
+ assert (
+ changed_card["tool_traces"][0]["result_hash"]
+ != same_shape_card["tool_traces"][0]["result_hash"]
+ )