Skip to content
Python

fix(memory): score a decision against its benchmark over the same dates

Yijia-Xiao13cf1e813880
diff --git a/tests/test_memory_log.py b/tests/test_memory_log.py
index 73876d299..b3516b3fb 100644
--- a/tests/test_memory_log.py
+++ b/tests/test_memory_log.py
@@ -1060,3 +1060,76 @@ def test_concurrent_writers_keep_every_decision_and_outcome(tmp_path):
assert errors == []
assert len(entries) == len(tickers) * len(dates)
assert all(not e["pending"] for e in entries)
+
+
+def _closes(by_symbol):
+ """A get_closes stand-in serving each symbol's series between the dates asked."""
+ def get_closes(symbol, start_date, end_date):
+ series = by_symbol[symbol]
+ return series[(series.index >= start_date) & (series.index < end_date)]
+ return get_closes
+
+
+class TestSettlementWindow:
+ """Stock and benchmark are measured over the same dates, whatever their calendars."""
+
+ def test_a_seven_day_market_is_scored_against_the_benchmark_over_the_same_dates(self, monkeypatch):
+ coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
+ weekdays = pd.bdate_range("2025-12-29", "2026-01-16")
+ spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
+
+ raw, alpha, days, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
+
+ # The coin is held Sat 3 -> Thu 8; SPY over the same span is Fri 2 -> Thu 8.
+ assert resolved == "2026-01-08"
+ assert raw == pytest.approx(105.0 / 100.0 - 1)
+ assert alpha == pytest.approx(raw - (spy["2026-01-08"] / spy["2026-01-02"] - 1))
+
+ def test_bars_stamped_in_different_time_zones_meet_on_their_dates(self, monkeypatch):
+ # Yahoo stamps a coin's daily bar at midnight UTC and SPY's at midnight
+ # New York, five hours later: the same day, not the day after.
+ coin = pd.Series([100.0 + i for i in range(10)],
+ index=pd.date_range("2026-01-03", periods=10, freq="D", tz="UTC"))
+ weekdays = pd.bdate_range("2025-12-29", "2026-01-16", tz="America/New_York")
+ spy = pd.Series([400.0 + 2 * i for i in range(len(weekdays))], index=weekdays)
+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
+
+ raw, alpha, _, resolved = settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY")
+
+ assert resolved == "2026-01-08"
+ spy_by_day = spy.tz_localize(None)
+ assert alpha == pytest.approx(raw - (spy_by_day["2026-01-08"] / spy_by_day["2026-01-02"] - 1))
+
+ def test_the_outcome_waits_for_the_benchmark_to_trade_through_the_window(self, monkeypatch):
+ coin = pd.Series([100.0 + i for i in range(10)], index=pd.date_range("2026-01-03", periods=10, freq="D"))
+ spy = pd.Series([400.0, 401.0, 402.0], index=pd.to_datetime(["2026-01-02", "2026-01-05", "2026-01-06"]))
+ monkeypatch.setattr(settlement, "get_closes", _closes({"BTC-USD": coin, "SPY": spy}))
+
+ assert settlement.fetch_returns("BTC-USD", "2026-01-03", 5, "SPY") == (None, None, None, None)
+
+ def test_a_missing_close_is_skipped_not_scored(self, monkeypatch):
+ idx = pd.bdate_range("2026-01-05", periods=8)
+ stock = pd.Series([100.0, float("nan"), 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
+ spy = pd.Series([400.0 + i for i in range(8)], index=idx)
+ monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
+
+ raw, alpha, _, resolved = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
+
+ assert raw == pytest.approx(105.0 / 100.0 - 1)
+ assert resolved == "2026-01-13"
+ assert alpha == pytest.approx(raw - (spy["2026-01-13"] / spy["2026-01-05"] - 1))
+
+
+def test_a_close_that_is_not_a_price_is_skipped_not_scored(monkeypatch):
+ """A zero close would score as an infinite return and be written to the memory log."""
+ idx = pd.bdate_range("2026-01-05", periods=8)
+ stock = pd.Series([0.0, 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0], index=idx)
+ spy = pd.Series([400.0 + i for i in range(8)], index=idx)
+ monkeypatch.setattr(settlement, "get_closes", _closes({"NVDA": stock, "SPY": spy}))
+
+ raw, alpha, _, _ = settlement.fetch_returns("NVDA", "2026-01-05", 5, "SPY")
+
+ assert raw == pytest.approx(105.0 / 100.0 - 1) # the window opens on the first real close
diff --git a/tradingagents/memory/settlement.py b/tradingagents/memory/settlement.py
index 6f8354de8..de14c9b11 100644
--- a/tradingagents/memory/settlement.py
+++ b/tradingagents/memory/settlement.py
@@ -34,6 +34,20 @@ def resolve_benchmark(ticker: str, config: dict) -> str:
return benchmark_map.get("", "SPY")
+def _by_day(closes):
+ """Closes keyed by the calendar day of their bar, missing closes dropped.
+
+ Yahoo stamps a daily bar at midnight in its market's time zone (UTC for a
+ coin, New York for SPY), so two series meet on their dates, not their
+ instants. A missing or non-positive close is no price at all, not a bar to
+ score on.
+ """
+ closes = closes[closes > 0]
+ if getattr(closes.index, "tz", None) is not None:
+ closes = closes.tz_localize(None)
+ return closes
+
+
def fetch_returns(
ticker: str, trade_date: str, holding_days: int = 5,
benchmark: str = "SPY",
@@ -56,21 +70,31 @@ def fetch_returns(
end_str = end.strftime("%Y-%m-%d")
# Closes for the instrument the analysis priced (XAUUSD -> GC=F, #984).
- stock = get_closes(ticker, trade_date, end_str)
- bench = get_closes(benchmark, trade_date, end_str)
-
- # Require the full holding window in both series. A rerun before it
- # has traded leaves the entry pending to retry next run, rather than
- # settling on a premature partial return (#1169).
- if len(stock) <= holding_days or len(bench) <= holding_days:
+ stock = _by_day(get_closes(ticker, trade_date, end_str))
+ # From a week earlier, so the benchmark has a close on or before entry.
+ bench_start = (start - timedelta(days=7)).strftime("%Y-%m-%d")
+ bench = _by_day(get_closes(benchmark, bench_start, end_str))
+
+ # Require the full holding window to have traded. A rerun before it has
+ # leaves the entry pending to retry next run, rather than settling on a
+ # premature partial return (#1169).
+ if len(stock) <= holding_days:
+ return None, None, None, None
+ entry, exit_ = stock.index[0], stock.index[holding_days]
+ # The window is the stock's own sessions; the benchmark is valued at its
+ # last close by each end, so both returns span the same dates even when
+ # the calendars differ (a coin trades at weekends, an index does not).
+ # It must have traded through the exit, or its close there may yet move.
+ if bench.empty or bench.index[0] > entry or bench.index[-1] < exit_:
return None, None, None, None
raw = float((stock.iloc[holding_days] - stock.iloc[0]) / stock.iloc[0])
- bench_ret = float((bench.iloc[holding_days] - bench.iloc[0]) / bench.iloc[0])
+ bench_entry, bench_exit = bench.asof(entry), bench.asof(exit_)
+ bench_ret = float((bench_exit - bench_entry) / bench_entry)
alpha = raw - bench_ret
- # The date of the last price bar used is when this outcome became
- # known — the point-in-time cutoff for injecting the lesson (#1251).
- resolution_date = stock.index[holding_days].strftime("%Y-%m-%d")
+ # Every close used is known by the exit: the point-in-time cutoff for
+ # injecting the lesson (#1251).
+ resolution_date = exit_.strftime("%Y-%m-%d")
return raw, alpha, holding_days, resolution_date
except Exception as e:
logger.warning(