TradingAgents: Multi-Agents LLM Financial Trading Framework
fix(memory): score a decision from the first close after its date (#1518)
Yijia-Xiaof5e3eae3a888
diff --git a/README.md b/README.mdindex 38cc4f3cf..03f19b039 100644--- a/README.md+++ b/README.md@@ -374,7 +374,7 @@ From the CLI:tradingagents backtest NVDA,AAPL --start 2026-06-01 --end 2026-08-01 --every 7```-Each cell is scored on realized alpha against the instrument's regional benchmark, grouped by rating. Your own memory log is never written to, and re-running the same grid with `run_id=result.run_id` skips the cells that already ran, so an interrupted sweep continues where it stopped.+Each cell is scored on realized alpha against the instrument's regional benchmark, grouped by rating. A decision is entered at the first close after its date, because a run reads news published after that day's close. Your own memory log is never written to, and re-running the same grid with `run_id=result.run_id` skips the cells that already ran, so an interrupted sweep continues where it stopped.## Reproducibilitydiff --git a/tests/test_graph_end_to_end.py b/tests/test_graph_end_to_end.pyindex 2f5b55e86..b55562e0d 100644--- a/tests/test_graph_end_to_end.py+++ b/tests/test_graph_end_to_end.py@@ -312,7 +312,7 @@ def test_past_decisions_settle_alongside_the_analysts(tmp_path, monkeypatch, offmodel = TimedModel()graph = _graph(tmp_path, monkeypatch, model)- graph.memory_log.store_decision("NVDA", "2026-01-02", "Rating: Buy\n\nBuy NVDA.")+ graph.memory_log.store_decision("NVDA", "2025-12-31", "Rating: Buy\n\nBuy NVDA.")closes = pd.Series([100.0 + i for i in range(10)], index=pd.bdate_range("2026-01-02", periods=10))monkeypatch.setattr(settlement, "get_closes", lambda *a, **k: closes)@@ -322,7 +322,7 @@ def test_past_decisions_settle_alongside_the_analysts(tmp_path, monkeypatch, offassert len(reflections) == 1 # settled once, inside the runanalyst_turns = [c for c in model.seen if c[0] == "agent"]assert reflections[0][2] < max(c[3] for c in analyst_turns), "settled before the analysts, not alongside"- settled = next(e for e in graph.memory_log.load_entries() if e["date"] == "2026-01-02")+ settled = next(e for e in graph.memory_log.load_entries() if e["date"] == "2025-12-31")assert not settled["pending"]lessons = graph.memory_log.get_past_context("NVDA", as_of=TRADE_DATE)assert lessons@@ -372,7 +372,7 @@ def test_the_memory_step_reads_lessons_as_of_the_trade_date_and_settles_once(tmpfrom tradingagents.memory import settlementgraph = _graph(tmp_path, monkeypatch, ScriptedModel(structured=True))- graph.memory_log.store_decision("NVDA", "2026-01-02", "Rating: Buy\n\nBuy NVDA.")+ graph.memory_log.store_decision("NVDA", "2025-12-31", "Rating: Buy\n\nBuy NVDA.")closes = pd.Series([100.0 + i for i in range(10)], index=pd.bdate_range("2026-01-02", periods=10))monkeypatch.setattr(settlement, "get_closes", lambda *a, **k: closes)asked = []diff --git a/tests/test_memory_log.py b/tests/test_memory_log.pyindex 375be979e..b7206afae 100644--- a/tests/test_memory_log.py+++ b/tests/test_memory_log.py@@ -512,8 +512,8 @@ class TestDeferredReflection:# settlement.fetch_returnsdef test_fetch_returns_valid_ticker(self):- stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0]- spy_prices = [400.0, 402.0, 404.0, 403.0, 405.0, 406.0]+ stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0, 107.0]+ spy_prices = [400.0, 402.0, 404.0, 403.0, 405.0, 406.0, 407.0]with patch("yfinance.Ticker") as mock_ticker_cls:def _make_ticker(sym):m = MagicMock()@@ -524,8 +524,8 @@ class TestDeferredReflection:assert raw is not None and alpha is not None and days is not Noneassert isinstance(raw, float) and isinstance(alpha, float) and isinstance(days, int)assert days == 5- # resolution date = the bar `days` sessions after the trade date (#1251)- assert resolved == "2026-01-10"+ # resolution date = the bar `days` sessions after entry, the first close after the decision (#1251)+ assert resolved == "2026-01-11"def test_fetch_returns_too_recent(self):"""Only 1 data point available → returns all-None, no crash."""@@ -549,7 +549,7 @@ class TestDeferredReflection:"""SPY having fewer rows than the stock (but still a full window) mustnot raise IndexError."""stock_prices = [100.0, 102.0, 104.0, 103.0, 105.0, 106.0, 107.0, 108.0] # 8 rows- spy_prices = [400.0, 402.0, 403.0, 405.0, 406.0, 407.0] # 6 rows+ spy_prices = [400.0, 402.0, 403.0, 405.0, 406.0, 407.0, 408.0] # 7 rowswith patch("yfinance.Ticker") as mock_ticker_cls:def _make_ticker(sym):m = MagicMock()@@ -559,7 +559,22 @@ class TestDeferredReflection:raw, alpha, days, resolved = settlement.fetch_returns("NVDA", "2026-01-05")assert raw is not None and alpha is not Noneassert days == 5 # full holding window used for both series- assert resolved == "2026-01-10"+ assert resolved == "2026-01-11"++ def test_a_decision_is_scored_from_the_first_close_after_its_date(self):+ """#1518: news published after the close on the decision date reaches the+ analysts, so the close of that date is not a price they could have traded at."""+ stock_prices = [100.0, 150.0, 150.0, 150.0, 150.0, 150.0, 165.0] # jumps on the next close+ spy_prices = [400.0] * 7+ with patch("yfinance.Ticker") as mock_ticker_cls:+ def _make_ticker(sym):+ m = MagicMock()+ m.history.return_value = _price_df(spy_prices if sym == "SPY" else stock_prices)+ return m+ mock_ticker_cls.side_effect = _make_ticker+ raw, alpha, days, resolved = settlement.fetch_returns("NVDA", "2026-01-05")+ assert raw == pytest.approx(0.10) and alpha == pytest.approx(0.10) # 150 -> 165, not 100 -> 150+ assert resolved == "2026-01-11"def test_fetch_returns_incomplete_window_stays_pending(self):"""#1169: a rerun before the full holding window has traded returns@@ -1082,10 +1097,10 @@ class TestSettlementWindow:raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")- # The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8.- assert resolved == "2026-01-08"- assert raw == pytest.approx(105.0 / 100.0 - 1)- assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1))+ # Decided Sat 3, the coin is held Sun 4 -> Fri 9; SPY over the same span is Fri 2 -> Fri 9.+ assert resolved == "2026-01-09"+ assert raw == pytest.approx(106.0 / 101.0 - 1)+ assert alpha == pytest.approx(raw - (spy["2026-01-09"] / spy["2026-01-02"] - 1))def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch):# Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight@@ -1098,9 +1113,9 @@ class TestSettlementWindow:raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")- assert resolved == "2026-01-08"+ assert resolved == "2026-01-09"spy_by_day = spy.tz_localize(None)- assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1))+ assert alpha == pytest.approx(raw - (spy_by_day["2026-01-09"] / spy_by_day["2026-01-02"] - 1))def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch):coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))@@ -1117,9 +1132,10 @@ class TestSettlementWindow:raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")- assert raw == pytest.approx(105.0 / 100.0 - 1)- assert resolved == "2026-01-13"- assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1))+ # Entered at the first real close after the decision: Wed 7, past the missing Tue 6.+ assert raw == pytest.approx(106.0 / 101.0 - 1)+ assert resolved == "2026-01-14"+ assert alpha == pytest.approx(raw - (spy["2026-01-14"] / spy["2026-01-07"] - 1))@pytest.mark.unit@@ -1130,7 +1146,7 @@ def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch):spy = pd.Series([400.0 + i for i in range(8)], index=idx)monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))- raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")+ raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-02", 5, "SPY")assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real closediff --git a/tests/test_symbol_normalization_paths.py b/tests/test_symbol_normalization_paths.pyindex a1971445d..9b4cf62d3 100644--- a/tests/test_symbol_normalization_paths.py+++ b/tests/test_symbol_normalization_paths.py@@ -54,7 +54,7 @@ def test_fetch_returns_normalizes_symbol(monkeypatch):assert queried[0] == "GC=F" # stock symbol normalized (#984)assert queried[1] == "SPY" # benchmark left as the canonical symbolassert raw is not None and days is not None- assert resolved == "2025-01-07" # resolution date recorded (#1251)+ assert resolved == "2025-01-08" # resolution date recorded (#1251)def test_news_lookup_normalizes_symbol(monkeypatch):diff --git a/tradingagents/backtest.py b/tradingagents/backtest.pyindex c4a33cb84..1b3503118 100644--- a/tradingagents/backtest.py+++ b/tradingagents/backtest.py@@ -119,7 +119,7 @@ class BacktestSummary:if self.pending:lines.append("Pending cells are not scored above; re-run to settle them.")lines.append(- f"Alpha is measured over {self.holding} after each analysis date. "+ f"Alpha is measured over {self.holding} from the first close after each analysis date. ""One model sampling per cell, and text feeds are not archived, so ""these figures are indicative rather than repeatable.")diff --git a/tradingagents/default_config.py b/tradingagents/default_config.pyindex 5723a70c0..2ce4fadc6 100644--- a/tradingagents/default_config.py+++ b/tradingagents/default_config.py@@ -175,8 +175,8 @@ def build_default_config() -> dict:# based on the ticker's exchange suffix. SPY remains the US default# so the reflection label keeps reading "Alpha vs SPY" for US tickers# while non-US tickers get their regional index automatically.- # Trading days after the analysis date over which a decision's outcome is- # measured, for reflection and for the backtest figures.+ # Trading days from the first close after the analysis date over which a+ # decision's outcome is measured, for reflection and for the backtest figures."holding_period_days": 5,"benchmark_ticker": None,"benchmark_map": {diff --git a/tradingagents/memory/reflection.py b/tradingagents/memory/reflection.pyindex ba8bf5f31..cf23c6395 100644--- a/tradingagents/memory/reflection.py+++ b/tradingagents/memory/reflection.py@@ -21,7 +21,7 @@ class Reflector:"""return ("You are a trading analyst reviewing your own past decision now that the outcome is known.\n"- f"The outcome covers {holding_days} trading days after the analysis date, "+ f"The outcome covers {holding_days} trading days from the first close after the analysis date, ""which may be shorter than the horizon the decision was written for.\n""Write exactly 2-4 sentences of plain prose (no bullets, no headers, no markdown).\n\n""Cover in order:\n"diff --git a/tradingagents/memory/settlement.py b/tradingagents/memory/settlement.pyindex 032b3c68b..c997b19f2 100644--- a/tradingagents/memory/settlement.py+++ b/tradingagents/memory/settlement.py@@ -53,7 +53,10 @@ def fetch_returns(ticker: str, trade_date: str, holding_days: int = 5,benchmark: str = "SPY",) -> tuple[float | None, float | None, int | None, str | None]:- """Fetch raw and alpha return for ticker over holding_days from trade_date.+ """Fetch raw and alpha return for ticker over holding_days after trade_date.++ The position is entered at the first close after ``trade_date``: a run dated+ ``trade_date`` reads news published after that day's close (#1518).``benchmark`` is the index used as the alpha baseline (resolved by thecaller via ``resolve_benchmark``). Returns ``(raw_return, alpha_return,@@ -64,13 +67,15 @@ def fetch_returns(to report."""start = datetime.strptime(trade_date, "%Y-%m-%d")- # holding_days counts trading days, so ask for the calendar span they- # occupy (about 7 for every 5) plus a week for holidays.- end = start + timedelta(days=round(holding_days * 7 / 5) + 7)+ # holding_days counts trading days after the entry session, so ask for the+ # calendar span they occupy (about 7 for every 5) plus a week for holidays.+ end = start + timedelta(days=round((holding_days + 1) * 7 / 5) + 7)end_str = end.strftime("%Y-%m-%d")- # Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984).+ # Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984),+ # from the first session after the decision date.stock = _by_day(get_closes(ticker, trade_date, end_str))+ stock = stock[stock.index > start]# From a week earlier, so the benchmark has a close on or before entry.bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d")bench = _by_day(get_closes(benchmark, bench_start, end_str))