Skip to content
Python

fix(dataflows): don't silently drop the latest OHLCV bar

Yijia-Xiao63be7fe7f1fe
diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.py
new file mode 100644
index 000000000..7a26b0e00
--- /dev/null
+++ b/tests/test_ohlcv_latest_bar.py
@@ -0,0 +1,136 @@
+"""The latest trading day's bar must not silently vanish (#1201).
+
+yfinance can return the newest in-range bar with a NaN close (an unsettled or
+glitched session). The old path parsed dates without normalizing timezone and
+dropped every NaN-close row before applying the curr_date cutoff, so the latest
+bar disappeared and the previous trading day looked like the latest. Now dates
+are normalized, and a latest in-range bar with no close raises rather than
+silently falling back.
+"""
+from __future__ import annotations
+
+import pandas as pd
+import pytest
+
+from tradingagents.dataflows import stockstats_utils as su
+from tradingagents.dataflows.symbol_utils import NoMarketDataError
+
+# --- date normalization -----------------------------------------------------
+
+def test_normalize_dates_strips_tz_and_normalizes_to_midnight():
+ aware = pd.Series(pd.to_datetime(
+ ["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"]
+ ))
+ out = su._normalize_dates(aware)
+ assert out.dt.tz is None
+ assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
+
+
+def test_normalize_dates_leaves_naive_dates_at_midnight():
+ naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"]))
+ out = su._normalize_dates(naive)
+ assert out.dt.tz is None
+ assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]
+
+
+def test_normalize_dates_handles_mixed_dst_offsets():
+ # 5y of US bars span DST; via a cache CSV they arrive as mixed-offset
+ # strings, which pd.to_datetime can't unify. Each keeps its own local date.
+ mixed = pd.Series([
+ "2026-01-08 00:00:00-05:00", # EST
+ "2026-06-08 00:00:00-04:00", # EDT
+ "not-a-date", # -> NaT
+ ])
+ out = su._normalize_dates(mixed)
+ assert out.iloc[0] == pd.Timestamp("2026-01-08")
+ assert out.iloc[1] == pd.Timestamp("2026-06-08")
+ assert pd.isna(out.iloc[2])
+
+
+def test_normalize_dates_keeps_positive_offset_local_date():
+ # A Tokyo bar at local midnight (+09:00) must stay on its own calendar day,
+ # not shift to the previous UTC day (which utc=True parsing would cause).
+ jst = pd.Series(["2026-05-08 00:00:00+09:00"])
+ assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08")
+
+
+# --- fill vs guard responsibilities ----------------------------------------
+
+def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect():
+ # _clean_dataframe normalizes but no longer drops the NaN close itself.
+ df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]})
+ cleaned = su._clean_dataframe(df)
+ assert len(cleaned) == 2
+ assert pd.isna(cleaned["Close"].iloc[-1])
+
+
+def test_fill_price_gaps_drops_nan_close_rows():
+ df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]),
+ "Close": [float("nan"), 100.0]})
+ filled = su._fill_price_gaps(df)
+ assert len(filled) == 1
+ assert filled["Close"].iloc[0] == 100.0
+
+
+# --- load_ohlcv end-to-end (with a mocked cache read) -----------------------
+
+def _run_load(monkeypatch, tmp_path, frame, curr_date):
+ """Drive load_ohlcv against a pre-seeded cache frame (no network)."""
+ monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)})
+ today = pd.Timestamp(curr_date)
+ monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today))
+ start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d")
+ end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d")
+ (tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False))
+
+ def _fail_download(*a, **k):
+ raise AssertionError("should use the seeded cache, not download")
+ monkeypatch.setattr(su.yf, "download", _fail_download)
+ monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None)
+ return su.load_ohlcv("AAPL", curr_date)
+
+
+def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path):
+ # Newest bar (the curr_date) has no close -> raise, don't return Thursday.
+ frame = pd.DataFrame({
+ "Date": ["2026-05-07", "2026-05-08"],
+ "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
+ "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],
+ })
+ with pytest.raises(NoMarketDataError, match="no closing price"):
+ _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+
+
+def test_older_nan_close_row_is_still_dropped(monkeypatch, tmp_path):
+ # A stale gap mid-series is dropped; the valid latest bar is served.
+ frame = pd.DataFrame({
+ "Date": ["2026-05-06", "2026-05-07", "2026-05-08"],
+ "Open": [100.0, 101.0, 102.0], "High": [101.0, 102.0, 103.0],
+ "Low": [99.0, 100.0, 101.0],
+ "Close": [100.5, float("nan"), 102.5], "Volume": [1_000_000, 1_000_000, 1_000_000],
+ })
+ out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+ assert out["Close"].iloc[-1] == 102.5
+ assert (out["Date"] == pd.Timestamp("2026-05-07")).sum() == 0 # the NaN row is gone
+
+
+def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path):
+ # A tz-aware/intraday latest bar on the cutoff day must not be filtered out
+ # by a naive-vs-aware comparison.
+ frame = pd.DataFrame({
+ "Date": ["2026-05-07 09:30:00-04:00", "2026-05-08 09:30:00-04:00"],
+ "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],
+ "Close": [100.5, 101.5], "Volume": [1_000_000, 1_000_000],
+ })
+ out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")
+ assert out["Close"].iloc[-1] == 101.5
+ assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")
diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.py
index 4ffe80422..94ba07ea8 100644
--- a/tradingagents/dataflows/stockstats_utils.py
+++ b/tradingagents/dataflows/stockstats_utils.py
@@ -60,17 +60,53 @@ def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame:
return data
+def _local_midnight(value) -> pd.Timestamp:
+ """A single timestamp as its naive, midnight-normalized local date (or NaT)."""
+ if pd.isna(value):
+ return pd.NaT
+ try:
+ ts = pd.Timestamp(value)
+ except (ValueError, TypeError):
+ return pd.NaT
+ if ts.tzinfo is not None:
+ ts = ts.tz_localize(None) # drop tz, keep the local wall-clock date
+ return ts.normalize()
+
+
+def _normalize_dates(dates) -> pd.Series:
+ """Parse to naive, midnight-normalized dates so tz-aware or intraday
+ timestamps compare correctly against the naive ``curr_date`` cutoff (#1201).
+
+ Normalized per element: 5 years of yfinance bars span daylight-saving
+ changes (and cache CSVs round-trip the offsets as strings), so the series can
+ carry mixed UTC offsets that ``pd.to_datetime`` cannot unify without
+ ``utc=True`` — which would shift non-US (positive-offset) markets to the
+ previous day. Keeping each bar's own local date avoids both.
+ """
+ return pd.to_datetime(pd.Series(dates).map(_local_midnight))
+
+
def _clean_dataframe(data: pd.DataFrame) -> pd.DataFrame:
- """Normalize a stock DataFrame for stockstats: parse dates, drop invalid rows, fill price gaps."""
+ """Normalize a stock DataFrame for stockstats: parse/normalize dates and
+ coerce prices to numeric (NaN where invalid). Dropping incomplete rows and
+ filling gaps is left to ``_fill_price_gaps`` so the caller can first inspect
+ the latest in-range bar (#1201)."""
data = _ensure_date_column(data)
- data["Date"] = pd.to_datetime(data["Date"], errors="coerce")
+ data["Date"] = _normalize_dates(data["Date"])
data = data.dropna(subset=["Date"])
price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]
data[price_cols] = data[price_cols].apply(pd.to_numeric, errors="coerce")
- data = data.dropna(subset=["Close"])
- data[price_cols] = data[price_cols].ffill().bfill()
+ return data
+
+def _fill_price_gaps(data: pd.DataFrame) -> pd.DataFrame:
+ """Drop rows with no close and forward/back-fill remaining price gaps so
+ indicators compute on a continuous series."""
+ price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]
+ # copy() so a filtered (sliced) input is written to safely, not via a view.
+ data = data.dropna(subset=["Close"]).copy()
+ data[price_cols] = data[price_cols].ffill().bfill()
return data
@@ -159,7 +195,7 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
safe_symbol = safe_ticker_component(canonical)
config = get_config()
- curr_date_dt = pd.to_datetime(curr_date)
+ curr_date_dt = pd.to_datetime(curr_date).normalize()
# Cache uses a fixed window (5y to today) so one file per symbol.
today_date = pd.Timestamp.today()
@@ -211,9 +247,20 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:
data = _clean_dataframe(data)
- # Filter to curr_date to prevent look-ahead bias in backtesting
+ # Filter to curr_date to prevent look-ahead bias in backtesting.
data = data[data["Date"] <= curr_date_dt]
+ # Guard the latest in-range bar before dropping incomplete rows: a newest bar
+ # with no close is "not settled yet", not "does not exist". Silently dropping
+ # it would make the previous trading day look like the latest (#1201); raise
+ # instead so the router surfaces it rather than fabricating a fallback.
+ if not data.empty and pd.isna(data["Close"].iloc[-1]):
+ raise NoMarketDataError(
+ symbol, canonical, "latest in-range OHLCV bar has no closing price"
+ )
+
+ data = _fill_price_gaps(data)
+
# Reject a stale frame (latest row far older than curr_date) rather than
# feeding year-old prices into indicators (#1021).
_assert_ohlcv_not_stale(data, curr_date, symbol, canonical)