TradingAgents: Multi-Agents LLM Financial Trading Framework
feat(cli): add a backtest command (#1234)
Yijia-Xiaod8eceb6571d2
diff --git a/cli/main.py b/cli/main.pyindex 348246d8c..00a233426 100644--- a/cli/main.py+++ b/cli/main.py@@ -20,6 +20,7 @@ from rich.table import Tablefrom rich.text import Textfrom cli.announcements import display_announcements, fetch_announcements+from cli.prefs import load_last_run, sanitize, save_last_runfrom cli.stats_handler import StatsCallbackHandlerfrom cli.utils import (ask_anthropic_effort,@@ -41,6 +42,7 @@ from cli.utils import (select_research_depth,select_shallow_thinking_agent,)+from tradingagents.backtest import iter_grid, run_backtest, summarizefrom tradingagents.default_config import DEFAULT_CONFIGfrom tradingagents.graph.analyst_execution import (AnalystWallTimeTracker,@@ -49,6 +51,7 @@ from tradingagents.graph.analyst_execution import (sync_analyst_tracker_from_chunk,)from tradingagents.graph.trading_graph import TradingAgentsGraph+from tradingagents.portfolio import load_portfoliofrom tradingagents.reporting import write_report_treeconsole = Console()@@ -493,7 +496,14 @@ def update_display(layout, spinner_text=None, stats_handler=None, start_time=Nondef get_user_selections():- """Get all user selections before starting the analysis display."""+ """Ask for the run's settings, offering the previous run's answers."""+ selections = _prompt_selections(load_last_run())+ save_last_run(selections)+ return selections+++def _prompt_selections(prefs):+ """Walk the selection steps. ``prefs`` prefills, the environment skips."""# Display ASCII art welcome messagewith open(Path(__file__).parent / "static" / "welcome.txt", encoding="utf-8") as f:welcome_ascii = f.read()@@ -586,7 +596,7 @@ def get_user_selections():"Select the language for analyst reports and final decision"))- output_language = ask_output_language()+ output_language = ask_output_language(prefs.get("output_language"))# Step 4: Select analystsconsole.print(@@ -594,7 +604,8 @@ def get_user_selections():"Step 4: Analysts Team", "Select your LLM analyst agents for the analysis"))- selected_analysts = select_analysts(asset_type)+ prefs = sanitize(prefs, asset_type.value)+ selected_analysts = select_analysts(asset_type, prefs.get("analysts"))console.print(f"[green]Selected analysts:[/green] {', '.join(analyst.value for analyst in selected_analysts)}")@@ -619,7 +630,7 @@ def get_user_selections():"Step 5: Research Depth", "Select your research depth level"))- selected_research_depth = select_research_depth()+ selected_research_depth = select_research_depth(prefs.get("research_depth"))# Step 6: LLM Provider (skipped when set via TRADINGAGENTS_LLM_PROVIDER).# The backend URL comes from TRADINGAGENTS_LLM_BACKEND_URL when set,@@ -641,7 +652,7 @@ def get_user_selections():"Step 6: LLM Provider", "Select your LLM provider"))- selected_llm_provider, backend_url = select_llm_provider()+ selected_llm_provider, backend_url = select_llm_provider(prefs.get("llm_provider"))# Providers with regional endpoints prompt for the region as a secondary# step so the main dropdown stays clean (mainland China and international@@ -688,8 +699,13 @@ def get_user_selections():"Step 7: Thinking Agents", "Select your thinking agents for analysis"))- selected_shallow_thinker = select_shallow_thinking_agent(selected_llm_provider)- selected_deep_thinker = select_deep_thinking_agent(selected_llm_provider)+ remembered = prefs if prefs.get("llm_provider") == selected_llm_provider else {}+ selected_shallow_thinker = select_shallow_thinking_agent(+ selected_llm_provider, remembered.get("quick_think_llm")+ )+ selected_deep_thinker = select_deep_thinking_agent(+ selected_llm_provider, remembered.get("deep_think_llm")+ )# Step 8: Provider-specific reasoning/thinking configuration. Each knob is# settable via its TRADINGAGENTS_* env var; when that var is set (or the@@ -732,8 +748,8 @@ def get_user_selections():"research_depth": selected_research_depth,"llm_provider": selected_llm_provider.lower(),"backend_url": backend_url,- "shallow_thinker": selected_shallow_thinker,- "deep_thinker": selected_deep_thinker,+ "quick_think_llm": selected_shallow_thinker,+ "deep_think_llm": selected_deep_thinker,"google_thinking_level": thinking_level,"openai_reasoning_effort": reasoning_effort,"anthropic_effort": anthropic_effort,@@ -985,8 +1001,8 @@ def _build_run_config(selections: dict, checkpoint: bool | None) -> dict:config["max_debate_rounds"] = selections["research_depth"]if not os.environ.get("TRADINGAGENTS_MAX_RISK_ROUNDS"):config["max_risk_discuss_rounds"] = selections["research_depth"]- config["quick_think_llm"] = selections["shallow_thinker"]- config["deep_think_llm"] = selections["deep_thinker"]+ config["quick_think_llm"] = selections["quick_think_llm"]+ config["deep_think_llm"] = selections["deep_think_llm"]config["backend_url"] = selections["backend_url"]config["llm_provider"] = selections["llm_provider"].lower()# Provider-specific thinking configuration@@ -1295,8 +1311,9 @@ def run_analysis(checkpoint: bool | None = None, portfolio=None):display_complete_report(final_state)[email protected](invoke_without_command=True)def analyze(+ ctx: typer.Context,checkpoint: bool | None = typer.Option(None,"--checkpoint/--no-checkpoint",@@ -1315,6 +1332,9 @@ def analyze("portfolio agents size against your actual position.",),):+ """Run an analysis. This is what a bare `tradingagents` does."""+ if ctx.invoked_subcommand is not None:+ returnif clear_checkpoints:from tradingagents.graph.checkpointer import clear_all_checkpointsn = clear_all_checkpoints(DEFAULT_CONFIG["data_cache_dir"])@@ -1343,5 +1363,43 @@ def analyze(raise typer.Exit(code=1) from None+def backtest(+ tickers: str = typer.Argument(..., help="Comma-separated tickers, e.g. NVDA,AAPL"),+ start: str = typer.Option(..., "--start", help="First analysis date, YYYY-MM-DD"),+ end: str = typer.Option(..., "--end", help="Last analysis date, YYYY-MM-DD"),+ every: int = typer.Option(7, "--every", help="Days between analysis dates"),+ analysts: str = typer.Option(+ None, "--analysts", help="Comma-separated analysts to run; omit for all four"+ ),+ asset_type: str = typer.Option("stock", "--asset-type", help="stock or crypto"),+ portfolio: str = typer.Option(+ None, "--portfolio", help="JSON file with holdings and cash, held constant across the grid"+ ),+):+ """Score past decisions over a grid of tickers and dates."""+ from tradingagents.agents.utils.memory import TradingMemoryLog++ try:+ dates = iter_grid(start, end, every)+ book = load_portfolio(portfolio) if portfolio else None+ except ValueError as exc:+ console.print(f"[red]{exc}[/red]")+ raise typer.Exit(code=1) from None++ names = [t.strip() for t in tickers.split(",") if t.strip()]+ kwargs = {"asset_type": asset_type, "portfolio": book}+ if analysts:+ kwargs["selected_analysts"] = [a.strip().lower() for a in analysts.split(",") if a.strip()]++ result = run_backtest(names, dates, DEFAULT_CONFIG, **kwargs)+ console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())+ console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")+ for ticker, date, reason in result.failures:+ console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")+ for ticker, reason in result.settlement_failures:+ console.print(f"[yellow]unsettled:[/yellow] {ticker}: {reason}")++if __name__ == "__main__":app()diff --git a/tests/test_backtest.py b/tests/test_backtest.pyindex a81ce7778..08aff1e08 100644--- a/tests/test_backtest.py+++ b/tests/test_backtest.py@@ -179,3 +179,10 @@ def test_a_failed_settlement_does_not_lose_the_remaining_tickers(tmp_path, monkeassert result.cells_run == 2assert settled == ["AAPL"]assert result.settlement_failures == [("NVDA", "reflector timed out")]+++def test_pending_note_appears_only_when_something_is_pending(tmp_path):+ settled = [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))]+ assert "Pending" not in summarize(_log_with(tmp_path, settled)).render()+ assert "Pending" in summarize(_log_with(tmp_path, settled + [("AAPL", "2026-01-05", DECISION, None)])).render()diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.pynew file mode 100644index 000000000..5b104a9b7--- /dev/null+++ b/tests/test_cli_commands.py@@ -0,0 +1,91 @@+"""The CLI keeps running an analysis with no arguments, and gains `backtest`.++Every documented invocation is bare (`tradingagents --checkpoint`), so analysis+has to stay the default action while a second command exists alongside it.+"""++from __future__ import annotations++import pytest+from typer.testing import CliRunner++import cli.main as m+++def runner(monkeypatch):+ monkeypatch.setattr(m, "run_analysis", lambda **kw: calls.append(("analysis", kw)))+ calls.clear()+ return CliRunner()+++calls: list = []+++def test_no_arguments_still_runs_an_analysis(runner):+ assert runner.invoke(m.app, []).exit_code == 0+ assert calls == [("analysis", {"checkpoint": None, "portfolio": None})]+++def test_options_still_parse_without_a_subcommand(runner):+ assert runner.invoke(m.app, ["--checkpoint"]).exit_code == 0+ assert calls[0][1]["checkpoint"] is True+++def test_backtest_does_not_also_run_an_analysis(runner, monkeypatch, tmp_path):+ swept = []+ monkeypatch.setattr(m, "run_backtest", lambda *a, **kw: swept.append((a, kw)) or _Result(tmp_path))+ monkeypatch.setattr(m, "summarize", lambda log: _Summary())++ result = runner.invoke(m.app, ["backtest", "NVDA,AAPL", "--start", "2026-06-01",+ "--end", "2026-06-15", "--every", "7"])++ assert result.exit_code == 0, result.output+ assert calls == [] # the interactive analysis must not run+ (tickers, dates, _config), kwargs = swept[0]+ assert tickers == ["NVDA", "AAPL"]+ assert dates == ["2026-06-01", "2026-06-08", "2026-06-15"]+ assert "scored" in result.output+++def test_backtest_reports_a_bad_date_instead_of_a_traceback(runner):+ result = runner.invoke(m.app, ["backtest", "NVDA", "--start", "June", "--end", "2026-06-15"])+ assert result.exit_code == 1+ assert "YYYY-MM-DD" in result.output+++def test_help_lists_the_backtest_command(runner):+ assert "backtest" in runner.invoke(m.app, ["--help"]).output+++class _Result:+ def __init__(self, tmp_path):+ self.run_id = "20260916_000000"+ self.log_path = tmp_path / "trading_memory.md"+ self.cells_run = 2+ self.skipped = 0+ self.failures = []+ self.settlement_failures = []+++class _Summary:+ def render(self):+ return "scored 2 cells"+++def test_every_command_is_registered_when_run_as_a_module():+ """README documents `python -m cli.main`, which executes the file top to+ bottom, so a command defined after the __main__ block would not exist."""+ import subprocess+ import sys++ out = subprocess.run([sys.executable, "-m", "cli.main", "backtest", "--help"],+ capture_output=True, text=True, timeout=120)+ assert out.returncode == 0, out.stderr[-400:]+ assert "--start" in out.stdoutdiff --git a/tests/test_cli_config_precedence.py b/tests/test_cli_config_precedence.pyindex e2ba90d14..b2f1f0cc8 100644--- a/tests/test_cli_config_precedence.py+++ b/tests/test_cli_config_precedence.py@@ -14,8 +14,8 @@ import cli.main as m# Minimal selections dict shaped like get_user_selections()'s return value.SELECTIONS = {"research_depth": 5,- "shallow_thinker": "gpt-5.4-mini",- "deep_thinker": "gpt-5.5",+ "quick_think_llm": "gpt-5.4-mini",+ "deep_think_llm": "gpt-5.5","backend_url": None,"llm_provider": "openai","google_thinking_level": None,diff --git a/tests/test_cli_env_skip.py b/tests/test_cli_env_skip.pyindex c98c24549..b3d50d956 100644--- a/tests/test_cli_env_skip.py+++ b/tests/test_cli_env_skip.py@@ -77,8 +77,8 @@ class TestCliSkipsPromptsFromEnv(unittest.TestCase):# The env values flow into the returned selections.self.assertEqual(sel["llm_provider"], "openai")self.assertEqual(sel["backend_url"], "https://opencode.ai/zen/go/v1")- self.assertEqual(sel["shallow_thinker"], "deepseek-v4-pro")- self.assertEqual(sel["deep_thinker"], "kimi-k2.5")+ self.assertEqual(sel["quick_think_llm"], "deepseek-v4-pro")+ self.assertEqual(sel["deep_think_llm"], "kimi-k2.5")self.assertEqual(sel["output_language"], "Japanese")diff --git a/tradingagents/backtest.py b/tradingagents/backtest.pyindex 6557f3f96..f6d9a4e23 100644--- a/tradingagents/backtest.py+++ b/tradingagents/backtest.py@@ -100,12 +100,13 @@ class BacktestSummary:f"- {rating}: n={score.count}, beat the benchmark "f"{score.hit_rate:.0%}, mean alpha {score.mean_alpha:+.2%}")- lines += [- "",- "Pending cells are not scored above; re-run to settle them.",+ lines.append("")+ if self.pending:+ lines.append("Pending cells are not scored above; re-run to settle them.")+ lines.append("One model sampling per cell, and text feeds are not archived, so "- "these figures are indicative rather than repeatable.",- ]+ "these figures are indicative rather than repeatable."+ )return "\n".join(lines)