Python
feat(backtest): summarize a backtest result, or a decision log by its path
Yijia-Xiaodcbff5ea428b
diff --git a/README.md b/README.mdindex 6cd125f3e..54213bb4e 100644--- a/README.md+++ b/README.md@@ -325,11 +325,10 @@ One run gives one decision, which cannot tell you whether the system decides wel```pythonfrom tradingagents.backtest import iter_grid, run_backtest, summarize-from tradingagents.agents.utils.memory import TradingMemoryLogdates = iter_grid("2026-06-01", "2026-08-01", every_n_days=7)result = run_backtest(["NVDA", "AAPL"], dates, config, selected_analysts=["market", "news"])-print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())+print(summarize(result).render())```From the CLI:diff --git a/cli/main.py b/cli/main.pyindex 4c792928a..db3dea86d 100644--- a/cli/main.py+++ b/cli/main.py@@ -1372,7 +1372,6 @@ def backtest(),):"""Score past decisions over a grid of tickers and dates."""- from tradingagents.agents.utils.memory import TradingMemoryLogtry:dates = iter_grid(start, end, every)@@ -1395,7 +1394,7 @@ def backtest(except Exception as exc: # a missing key or an unknown analyst is a setup errorconsole.print(f"[red]{exc}[/red]")raise typer.Exit(code=1) from None- console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())+ console.print(summarize(result).render())console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")for ticker, date, reason in result.failures:console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")diff --git a/tests/test_backtest.py b/tests/test_backtest.pyindex cebb63134..fb32e66ec 100644--- a/tests/test_backtest.py+++ b/tests/test_backtest.py@@ -120,7 +120,7 @@ def _log_with(tmp_path, rows):log.store_decision(ticker, date, decision)if outcome is not None:log.update_with_outcome(ticker, date, outcome[0], outcome[1], 5, "note", "2026-02-01")- return log+ return tmp_path / "m.md"@pytest.mark.unit@@ -245,4 +245,32 @@ def test_the_window_reported_is_the_one_the_outcomes_used(tmp_path):log.store_decision("NVDA", "2026-01-05", "**Rating**: Buy\n\nx")log.update_with_outcome("NVDA", "2026-01-05", 0.1, 0.04, 21, "note", "2026-02-01")- assert "21 trading days" in summarize(log).render()+ assert "21 trading days" in summarize(tmp_path / "m.md").render()+++def test_a_backtest_result_is_summarized_directly(tmp_path):+ """The result names its own log, so a caller never builds the log to score it."""+ from tradingagents.backtest import BacktestResult++ path = _log_with(tmp_path, [("NVDA", "2026-01-05", "Rating: Buy\n\nx", (0.10, 0.04))])++ assert summarize(BacktestResult(run_id="r", log_path=path)).resolved == 1+++def test_a_log_path_that_does_not_exist_is_an_error_not_an_empty_summary(tmp_path):+ missing = tmp_path / "no-such-dir" / "m.md"++ with pytest.raises(FileNotFoundError):+ summarize(missing)+ assert not missing.parent.exists()+++def test_a_result_whose_cells_all_failed_summarizes_as_empty(tmp_path):+ from tradingagents.backtest import BacktestResult++ result = BacktestResult(run_id="r", log_path=tmp_path / "never-written.md")++ assert summarize(result).resolved == 0diff --git a/tests/test_rating_integrity.py b/tests/test_rating_integrity.pyindex 55a5e5627..77e8f14b9 100644--- a/tests/test_rating_integrity.py+++ b/tests/test_rating_integrity.py@@ -98,7 +98,7 @@ def test_an_unscored_decision_is_left_out_of_the_backtest_figures(tmp_path):log.store_decision("AAPL", "2026-01-05", REFUSAL)log.update_with_outcome("AAPL", "2026-01-05", 0.1, 0.04, 5, "note", "2026-02-01")- summary = summarize(log)+ summary = summarize(tmp_path / "m.md")assert set(summary.by_rating) == {"Buy"}diff --git a/tradingagents/backtest.py b/tradingagents/backtest.pyindex c88844035..afa0b409b 100644--- a/tradingagents/backtest.py+++ b/tradingagents/backtest.py@@ -175,9 +175,15 @@ def run_backtest(return result-def summarize(memory_log: TradingMemoryLog) -> BacktestSummary:- """Score the settled decisions in a log, by rating."""- entries = memory_log.load_entries()+def summarize(source: BacktestResult | str | Path) -> BacktestSummary:+ """Score the settled decisions of a backtest, or of a decision log at a path, by rating."""+ if isinstance(source, BacktestResult):+ path = source.log_path # a run whose cells all failed wrote no log: nothing to score+ elif Path(source).is_file():+ path = Path(source)+ else:+ raise FileNotFoundError(f"no decision log at {source}")+ entries = TradingMemoryLog({"memory_log_path": str(path)}).load_entries()# A decision with no readable rating has no direction, so it can neither# count for nor against the system; it is reported as unscored instead.resolved = [(e, _alpha(e)) for e in entries