Skip to content
Python

feat(backtest): summarize a backtest result, or a decision log by its path

Yijia-Xiaodcbff5ea428b
diff --git a/README.md b/README.md
index 6cd125f3e..54213bb4e 100644
--- a/README.md
+++ b/README.md
@@ -325,11 +325,10 @@ One run gives one decision, which cannot tell you whether the system decides wel
```python
from tradingagents.backtest import iter_grid, run_backtest, summarize
-from tradingagents.agents.utils.memory import TradingMemoryLog
dates = iter_grid("2026-06-01", "2026-08-01", every_n_days=7)
result = run_backtest(["NVDA", "AAPL"], dates, config, selected_analysts=["market", "news"])
-print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())
+print(summarize(result).render())
```
From the CLI:
diff --git a/cli/main.py b/cli/main.py
index 4c792928a..db3dea86d 100644
--- a/cli/main.py
+++ b/cli/main.py
@@ -1372,7 +1372,6 @@ def backtest(
),
):
"""Score past decisions over a grid of tickers and dates."""
- from tradingagents.agents.utils.memory import TradingMemoryLog
try:
dates = iter_grid(start, end, every)
@@ -1395,7 +1394,7 @@ def backtest(
except Exception as exc: # a missing key or an unknown analyst is a setup error
console.print(f"[red]{exc}[/red]")
raise typer.Exit(code=1) from None
- console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())
+ console.print(summarize(result).render())
console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")
for ticker, date, reason in result.failures:
console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")
diff --git a/tests/test_backtest.py b/tests/test_backtest.py
index cebb63134..fb32e66ec 100644
--- a/tests/test_backtest.py
+++ b/tests/test_backtest.py
@@ -120,7 +120,7 @@ def _log_with(tmp_path, rows):
log.store_decision(ticker, date, decision)
if outcome is not None:
log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01")
- return log
+ return tmp_path / "m.md"
@pytest.mark.unit
@@ -245,4 +245,32 @@ def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path):
log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")
log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01")
- assert "21 trading days" in summarize(log).render()
+ assert "21 trading days" in summarize(tmp_path / "m.md").render()
+
+
+def test_a_backtest_result_is_summarized_directly(tmp_path):
+ """The result names its own log, so a caller never builds the log to score it."""
+ from tradingagents.backtest import BacktestResult
+
+ path = _log_with(tmp_path, [("NVDA", "2026-01-05", "Rating: Buy\n\nx", (0.10, 0.04))])
+
+ assert summarize(BacktestResult(run_id="r", log_path=path)).resolved == 1
+
+
+def test_a_log_path_that_does_not_exist_is_an_error_not_an_empty_summary(tmp_path):
+ missing = tmp_path / "no-such-dir" / "m.md"
+
+ with pytest.raises(FileNotFoundError):
+ summarize(missing)
+ assert not missing.parent.exists()
+
+
+def test_a_result_whose_cells_all_failed_summarizes_as_empty(tmp_path):
+ from tradingagents.backtest import BacktestResult
+
+ result = BacktestResult(run_id="r", log_path=tmp_path / "never-written.md")
+
+ assert summarize(result).resolved == 0
diff --git a/tests/test_rating_integrity.py b/tests/test_rating_integrity.py
index 55a5e5627..77e8f14b9 100644
--- a/tests/test_rating_integrity.py
+++ b/tests/test_rating_integrity.py
@@ -98,7 +98,7 @@ def test_an_unscored_decision_is_left_out_of_the_backtest_figures(tmp_path):
log.store_decision("AAPL", "2026-01-05", REFUSAL)
log.update_with_outcome("AAPL", "2026-01-05", 0.1, 0.04, 5, "note", "2026-02-01")
- summary = summarize(log)
+ summary = summarize(tmp_path / "m.md")
assert set(summary.by_rating) == {"Buy"}
diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py
index c88844035..afa0b409b 100644
--- a/tradingagents/backtest.py
+++ b/tradingagents/backtest.py
@@ -175,9 +175,15 @@ def run_backtest(
return result
-def summarize(memory_log: TradingMemoryLog) -> BacktestSummary:
- """Score the settled decisions in a log, by rating."""
- entries = memory_log.load_entries()
+def summarize(source: BacktestResult | str | Path) -> BacktestSummary:
+ """Score the settled decisions of a backtest, or of a decision log at a path, by rating."""
+ if isinstance(source, BacktestResult):
+ path = source.log_path # a run whose cells all failed wrote no log: nothing to score
+ elif Path(source).is_file():
+ path = Path(source)
+ else:
+ raise FileNotFoundError(f"no decision log at {source}")
+ entries = TradingMemoryLog({"memory_log_path": str(path)}).load_entries()
# A decision with no readable rating has no direction, so it can neither
# count for nor against the system; it is reported as unscored instead.
resolved = [(e, _alpha(e)) for e in entries