Skip to content

TradingAgents: Multi-Agents LLM Financial Trading Framework

feat(cli): add a backtest command (#1234)

Yijia-Xiaod8eceb6571d2
diff --git a/cli/main.py b/cli/main.py
index 348246d8c..00a233426 100644
--- a/cli/main.py
+++ b/cli/main.py
@@ -20,6 +20,7 @@ from rich.table import Table
from rich.text import Text
from cli.announcements import display_announcements, fetch_announcements
+from cli.prefs import load_last_run, sanitize, save_last_run
from cli.stats_handler import StatsCallbackHandler
from cli.utils import (
ask_anthropic_effort,
@@ -41,6 +42,7 @@ from cli.utils import (
select_research_depth,
select_shallow_thinking_agent,
)
+from tradingagents.backtest import iter_grid, run_backtest, summarize
from tradingagents.default_config import DEFAULT_CONFIG
from tradingagents.graph.analyst_execution import (
AnalystWallTimeTracker,
@@ -49,6 +51,7 @@ from tradingagents.graph.analyst_execution import (
sync_analyst_tracker_from_chunk,
)
from tradingagents.graph.trading_graph import TradingAgentsGraph
+from tradingagents.portfolio import load_portfolio
from tradingagents.reporting import write_report_tree
console = Console()
@@ -493,7 +496,14 @@ def update_display(layout, spinner_text=None, stats_handler=None, start_time=Non
def get_user_selections():
- """Get all user selections before starting the analysis display."""
+ """Ask for the run's settings, offering the previous run's answers."""
+ selections = _prompt_selections(load_last_run())
+ save_last_run(selections)
+ return selections
+
+
+def _prompt_selections(prefs):
+ """Walk the selection steps. ``prefs`` prefills, the environment skips."""
# Display ASCII art welcome message
with open(Path(__file__).parent / "static" / "welcome.txt", encoding="utf-8") as f:
welcome_ascii = f.read()
@@ -586,7 +596,7 @@ def get_user_selections():
"Select the language for analyst reports and final decision"
)
)
- output_language = ask_output_language()
+ output_language = ask_output_language(prefs.get("output_language"))
# Step 4: Select analysts
console.print(
@@ -594,7 +604,8 @@ def get_user_selections():
"Step 4: Analysts Team", "Select your LLM analyst agents for the analysis"
)
)
- selected_analysts = select_analysts(asset_type)
+ prefs = sanitize(prefs, asset_type.value)
+ selected_analysts = select_analysts(asset_type, prefs.get("analysts"))
console.print(
f"[green]Selected analysts:[/green] {', '.join(analyst.value for analyst in selected_analysts)}"
)
@@ -619,7 +630,7 @@ def get_user_selections():
"Step 5: Research Depth", "Select your research depth level"
)
)
- selected_research_depth = select_research_depth()
+ selected_research_depth = select_research_depth(prefs.get("research_depth"))
# Step 6: LLM Provider (skipped when set via TRADINGAGENTS_LLM_PROVIDER).
# The backend URL comes from TRADINGAGENTS_LLM_BACKEND_URL when set,
@@ -641,7 +652,7 @@ def get_user_selections():
"Step 6: LLM Provider", "Select your LLM provider"
)
)
- selected_llm_provider, backend_url = select_llm_provider()
+ selected_llm_provider, backend_url = select_llm_provider(prefs.get("llm_provider"))
# Providers with regional endpoints prompt for the region as a secondary
# step so the main dropdown stays clean (mainland China and international
@@ -688,8 +699,13 @@ def get_user_selections():
"Step 7: Thinking Agents", "Select your thinking agents for analysis"
)
)
- selected_shallow_thinker = select_shallow_thinking_agent(selected_llm_provider)
- selected_deep_thinker = select_deep_thinking_agent(selected_llm_provider)
+ remembered = prefs if prefs.get("llm_provider") == selected_llm_provider else {}
+ selected_shallow_thinker = select_shallow_thinking_agent(
+ selected_llm_provider, remembered.get("quick_think_llm")
+ )
+ selected_deep_thinker = select_deep_thinking_agent(
+ selected_llm_provider, remembered.get("deep_think_llm")
+ )
# Step 8: Provider-specific reasoning/thinking configuration. Each knob is
# settable via its TRADINGAGENTS_* env var; when that var is set (or the
@@ -732,8 +748,8 @@ def get_user_selections():
"research_depth": selected_research_depth,
"llm_provider": selected_llm_provider.lower(),
"backend_url": backend_url,
- "shallow_thinker": selected_shallow_thinker,
- "deep_thinker": selected_deep_thinker,
+ "quick_think_llm": selected_shallow_thinker,
+ "deep_think_llm": selected_deep_thinker,
"google_thinking_level": thinking_level,
"openai_reasoning_effort": reasoning_effort,
"anthropic_effort": anthropic_effort,
@@ -985,8 +1001,8 @@ def _build_run_config(selections: dict, checkpoint: bool | None) -> dict:
config["max_debate_rounds"] = selections["research_depth"]
if not os.environ.get("TRADINGAGENTS_MAX_RISK_ROUNDS"):
config["max_risk_discuss_rounds"] = selections["research_depth"]
- config["quick_think_llm"] = selections["shallow_thinker"]
- config["deep_think_llm"] = selections["deep_thinker"]
+ config["quick_think_llm"] = selections["quick_think_llm"]
+ config["deep_think_llm"] = selections["deep_think_llm"]
config["backend_url"] = selections["backend_url"]
config["llm_provider"] = selections["llm_provider"].lower()
# Provider-specific thinking configuration
@@ -1295,8 +1311,9 @@ def run_analysis(checkpoint: bool | None = None, portfolio=None):
display_complete_report(final_state)
[email protected](invoke_without_command=True)
def analyze(
+ ctx: typer.Context,
checkpoint: bool | None = typer.Option(
None,
"--checkpoint/--no-checkpoint",
@@ -1315,6 +1332,9 @@ def analyze(
"portfolio agents size against your actual position.",
),
):
+ """Run an analysis. This is what a bare `tradingagents` does."""
+ if ctx.invoked_subcommand is not None:
+ return
if clear_checkpoints:
from tradingagents.graph.checkpointer import clear_all_checkpoints
n = clear_all_checkpoints(DEFAULT_CONFIG["data_cache_dir"])
@@ -1343,5 +1363,43 @@ def analyze(
raise typer.Exit(code=1) from None
+def backtest(
+ tickers: str = typer.Argument(..., help="Comma-separated tickers, e.g. NVDA,AAPL"),
+ start: str = typer.Option(..., "--start", help="First analysis date, YYYY-MM-DD"),
+ end: str = typer.Option(..., "--end", help="Last analysis date, YYYY-MM-DD"),
+ every: int = typer.Option(7, "--every", help="Days between analysis dates"),
+ analysts: str = typer.Option(
+ None, "--analysts", help="Comma-separated analysts to run; omit for all four"
+ ),
+ asset_type: str = typer.Option("stock", "--asset-type", help="stock or crypto"),
+ portfolio: str = typer.Option(
+ None, "--portfolio", help="JSON file with holdings and cash, held constant across the grid"
+ ),
+):
+ """Score past decisions over a grid of tickers and dates."""
+ from tradingagents.agents.utils.memory import TradingMemoryLog
+
+ try:
+ dates = iter_grid(start, end, every)
+ book = load_portfolio(portfolio) if portfolio else None
+ except ValueError as exc:
+ console.print(f"[red]{exc}[/red]")
+ raise typer.Exit(code=1) from None
+
+ names = [t.strip() for t in tickers.split(",") if t.strip()]
+ kwargs = {"asset_type": asset_type, "portfolio": book}
+ if analysts:
+ kwargs["selected_analysts"] = [a.strip().lower() for a in analysts.split(",") if a.strip()]
+
+ result = run_backtest(names, dates, DEFAULT_CONFIG, **kwargs)
+ console.print(summarize(TradingMemoryLog({"memory_log_path": str(result.log_path)})).render())
+ console.print(f"\nRan {result.cells_run} cells, skipped {result.skipped}. Log: {result.log_path}")
+ for ticker, date, reason in result.failures:
+ console.print(f"[yellow]failed:[/yellow] {ticker} {date}: {reason}")
+ for ticker, reason in result.settlement_failures:
+ console.print(f"[yellow]unsettled:[/yellow] {ticker}: {reason}")
+
+
if __name__ == "__main__":
app()
diff --git a/tests/test_backtest.py b/tests/test_backtest.py
index a81ce7778..08aff1e08 100644
--- a/tests/test_backtest.py
+++ b/tests/test_backtest.py
@@ -179,3 +179,10 @@ def test_a_failed_settlement_does_not_lose_the_remaining_tickers(tmp_path, monke
assert result.cells_run == 2
assert settled == ["AAPL"]
assert result.settlement_failures == [("NVDA", "reflector timed out")]
+
+
+def test_pending_note_appears_only_when_something_is_pending(tmp_path):
+ settled = [("NVDA", "2026-01-05", DECISION, (0.1, 0.05))]
+ assert "Pending" not in summarize(_log_with(tmp_path, settled)).render()
+ assert "Pending" in summarize(_log_with(tmp_path, settled + [("AAPL", "2026-01-05", DECISION, None)])).render()
diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py
new file mode 100644
index 000000000..5b104a9b7
--- /dev/null
+++ b/tests/test_cli_commands.py
@@ -0,0 +1,91 @@
+"""The CLI keeps running an analysis with no arguments, and gains `backtest`.
+
+Every documented invocation is bare (`tradingagents --checkpoint`), so analysis
+has to stay the default action while a second command exists alongside it.
+"""
+
+from __future__ import annotations
+
+import pytest
+from typer.testing import CliRunner
+
+import cli.main as m
+
+
+def runner(monkeypatch):
+ monkeypatch.setattr(m, "run_analysis", lambda **kw: calls.append(("analysis", kw)))
+ calls.clear()
+ return CliRunner()
+
+
+calls: list = []
+
+
+def test_no_arguments_still_runs_an_analysis(runner):
+ assert runner.invoke(m.app, []).exit_code == 0
+ assert calls == [("analysis", {"checkpoint": None, "portfolio": None})]
+
+
+def test_options_still_parse_without_a_subcommand(runner):
+ assert runner.invoke(m.app, ["--checkpoint"]).exit_code == 0
+ assert calls[0][1]["checkpoint"] is True
+
+
+def test_backtest_does_not_also_run_an_analysis(runner, monkeypatch, tmp_path):
+ swept = []
+ monkeypatch.setattr(m, "run_backtest", lambda *a, **kw: swept.append((a, kw)) or _Result(tmp_path))
+ monkeypatch.setattr(m, "summarize", lambda log: _Summary())
+
+ result = runner.invoke(m.app, ["backtest", "NVDA,AAPL", "--start", "2026-06-01",
+ "--end", "2026-06-15", "--every", "7"])
+
+ assert result.exit_code == 0, result.output
+ assert calls == [] # the interactive analysis must not run
+ (tickers, dates, _config), kwargs = swept[0]
+ assert tickers == ["NVDA", "AAPL"]
+ assert dates == ["2026-06-01", "2026-06-08", "2026-06-15"]
+ assert "scored" in result.output
+
+
+def test_backtest_reports_a_bad_date_instead_of_a_traceback(runner):
+ result = runner.invoke(m.app, ["backtest", "NVDA", "--start", "June", "--end", "2026-06-15"])
+ assert result.exit_code == 1
+ assert "YYYY-MM-DD" in result.output
+
+
+def test_help_lists_the_backtest_command(runner):
+ assert "backtest" in runner.invoke(m.app, ["--help"]).output
+
+
+class _Result:
+ def __init__(self, tmp_path):
+ self.run_id = "20260916_000000"
+ self.log_path = tmp_path / "trading_memory.md"
+ self.cells_run = 2
+ self.skipped = 0
+ self.failures = []
+ self.settlement_failures = []
+
+
+class _Summary:
+ def render(self):
+ return "scored 2 cells"
+
+
+def test_every_command_is_registered_when_run_as_a_module():
+ """README documents `python -m cli.main`, which executes the file top to
+ bottom, so a command defined after the __main__ block would not exist."""
+ import subprocess
+ import sys
+
+ out = subprocess.run([sys.executable, "-m", "cli.main", "backtest", "--help"],
+ capture_output=True, text=True, timeout=120)
+ assert out.returncode == 0, out.stderr[-400:]
+ assert "--start" in out.stdout
diff --git a/tests/test_cli_config_precedence.py b/tests/test_cli_config_precedence.py
index e2ba90d14..b2f1f0cc8 100644
--- a/tests/test_cli_config_precedence.py
+++ b/tests/test_cli_config_precedence.py
@@ -14,8 +14,8 @@ import cli.main as m
# Minimal selections dict shaped like get_user_selections()'s return value.
SELECTIONS = {
"research_depth": 5,
- "shallow_thinker": "gpt-5.4-mini",
- "deep_thinker": "gpt-5.5",
+ "quick_think_llm": "gpt-5.4-mini",
+ "deep_think_llm": "gpt-5.5",
"backend_url": None,
"llm_provider": "openai",
"google_thinking_level": None,
diff --git a/tests/test_cli_env_skip.py b/tests/test_cli_env_skip.py
index c98c24549..b3d50d956 100644
--- a/tests/test_cli_env_skip.py
+++ b/tests/test_cli_env_skip.py
@@ -77,8 +77,8 @@ class TestCliSkipsPromptsFromEnv(unittest.TestCase):
# The env values flow into the returned selections.
self.assertEqual(sel["llm_provider"], "openai")
self.assertEqual(sel["backend_url"], "https://opencode.ai/zen/go/v1")
- self.assertEqual(sel["shallow_thinker"], "deepseek-v4-pro")
- self.assertEqual(sel["deep_thinker"], "kimi-k2.5")
+ self.assertEqual(sel["quick_think_llm"], "deepseek-v4-pro")
+ self.assertEqual(sel["deep_think_llm"], "kimi-k2.5")
self.assertEqual(sel["output_language"], "Japanese")
diff --git a/tradingagents/backtest.py b/tradingagents/backtest.py
index 6557f3f96..f6d9a4e23 100644
--- a/tradingagents/backtest.py
+++ b/tradingagents/backtest.py
@@ -100,12 +100,13 @@ class BacktestSummary:
f"- {rating}: n={score.count}, beat the benchmark "
f"{score.hit_rate:.0%}, mean alpha {score.mean_alpha:+.2%}"
)
- lines += [
- "",
- "Pending cells are not scored above; re-run to settle them.",
+ lines.append("")
+ if self.pending:
+ lines.append("Pending cells are not scored above; re-run to settle them.")
+ lines.append(
"One model sampling per cell, and text feeds are not archived, so "
- "these figures are indicative rather than repeatable.",
- ]
+ "these figures are indicative rather than repeatable."
+ )
return "\n".join(lines)