Python
fix(memory): score a decision against its benchmark over the same dates
Yijia-Xiao13cf1e813880
diff --git a/tests/test_memory_log.py b/tests/test_memory_log.pyindex 73876d299..b3516b3fb 100644--- a/tests/test_memory_log.py+++ b/tests/test_memory_log.py@@ -1060,3 +1060,76 @@ def test_concurrent_writers_keep_every_decision_and_outcome(tmp_path):assert errors == []assert len(entries) == len(tickers) * len(dates)assert all(not e["pending"] for e in entries)+++def _closes(by_symbol):+ """A get_closes stand-in serving each symbol's series between the dates asked."""+ def get_closes(symbol, start_date, end_date):+ series = by_symbol[symbol]+ return series[(series.index >= start_date) & (series.index < end_date)]+ return get_closes+++class TestSettlementWindow:+ """Stock and benchmark are measured over the same dates, whatever their calendars."""++ def test_a_seven_day_market_is_scored_against_the_benchmark_over_the_same_dates(self, monkeypatch):+ coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))+ weekdays = pd.bdate_range("2025-12-29", "2026-01-16")+ spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))++ raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")++ # The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8.+ assert resolved == "2026-01-08"+ assert raw == pytest.approx(105.0 / 100.0 - 1)+ assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1))++ def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch):+ # Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight+ # New York, five hours later: the same day, not the day after.+ coin = pd.Series([100.0 + i for i in range(10)],+ index=pd.date_range("2026-01-03", periods=10, freq="D", tz="UTC"))+ weekdays = pd.bdate_range("2025-12-29", "2026-01-16", tz="America/New_York")+ spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))++ raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")++ assert resolved == "2026-01-08"+ spy_by_day = spy.tz_localize(None)+ assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1))++ def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch):+ coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))+ spy = pd.Series([400.0, 401.0, 402.0], index=pd.to_datetime(["2026-01-02", "2026-01-05", "2026-01-06"]))+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))++ assert settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") == (None, None, None, None)++ def test_a_missing_close_is_skipped_not_scored(self, monkeypatch):+ idx = pd.bdate_range("2026-01-05", periods=8)+ stock = pd.Series([100.0, float("nan"), 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)+ spy = pd.Series([400.0 + i for i in range(8)], index=idx)+ monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))++ raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")++ assert raw == pytest.approx(105.0 / 100.0 - 1)+ assert resolved == "2026-01-13"+ assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1))+++def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch):+ """A zero close would score as an infinite return and be written to the memory log."""+ idx = pd.bdate_range("2026-01-05", periods=8)+ stock = pd.Series([0.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)+ spy = pd.Series([400.0 + i for i in range(8)], index=idx)+ monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))++ raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")++ assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real closediff --git a/tradingagents/memory/settlement.py b/tradingagents/memory/settlement.pyindex 6f8354de8..de14c9b11 100644--- a/tradingagents/memory/settlement.py+++ b/tradingagents/memory/settlement.py@@ -34,6 +34,20 @@ def resolve_benchmark(ticker: str, config: dict) -> str:return benchmark_map.get("", "SPY")+def _by_day(closes):+ """Closes keyed by the calendar day of their bar, missing closes dropped.++ Yahoo stamps a daily bar at midnight in its market's time zone (UTC for a+ coin, New York for SPY), so two series meet on their dates, not their+ instants. A missing or non-positive close is no price at all, not a bar to+ score on.+ """+ closes = closes[closes > 0]+ if getattr(closes.index, "tz", None) is not None:+ closes = closes.tz_localize(None)+ return closes++def fetch_returns(ticker: str, trade_date: str, holding_days: int = 5,benchmark: str = "SPY",@@ -56,21 +70,31 @@ def fetch_returns(end_str = end.strftime("%Y-%m-%d")# Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984).- stock = get_closes(ticker, trade_date, end_str)- bench = get_closes(benchmark, trade_date, end_str)-- # Require the full holding window in both series. A rerun before it- # has traded leaves the entry pending to retry next run, rather than- # settling on a premature partial return (#1169).- if len(stock) <= holding_days or len(bench) <= holding_days:+ stock = _by_day(get_closes(ticker, trade_date, end_str))+ # From a week earlier, so the benchmark has a close on or before entry.+ bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d")+ bench = _by_day(get_closes(benchmark, bench_start, end_str))++ # Require the full holding window to have traded. A rerun before it has+ # leaves the entry pending to retry next run, rather than settling on a+ # premature partial return (#1169).+ if len(stock) <= holding_days:+ return None, None, None, None+ entry, exit_ = stock.index[0], stock.index[holding_days]+ # The window is the stock's own sessions; the benchmark is valued at its+ # last close by each end, so both returns span the same dates even when+ # the calendars differ (a coin trades at weekends, an index does not).+ # It must have traded through the exit, or its close there may yet move.+ if bench.empty or bench.index[0] > entry or bench.index[-1] < exit_:return None, None, None, Noneraw = float((stock.iloc[holding_days] - stock.iloc[0]) / stock.iloc[0])- bench_ret = float((bench.iloc[holding_days] - bench.iloc[0]) / bench.iloc[0])+ bench_entry, bench_exit = bench.asof(entry), bench.asof(exit_)+ bench_ret = float((bench_exit - bench_entry) / bench_entry)alpha = raw - bench_ret- # The date of the last price bar used is when this outcome became- # known — the point-in-time cutoff for injecting the lesson (#1251).- resolution_date = stock.index[holding_days].strftime("%Y-%m-%d")+ # Every close used is known by the exit: the point-in-time cutoff for+ # injecting the lesson (#1251).+ resolution_date = exit_.strftime("%Y-%m-%d")return raw, alpha, holding_days, resolution_dateexcept Exception as e:logger.warning(