Python
fix(dataflows): don't silently drop the latest OHLCV bar
Yijia-Xiao63be7fe7f1fe
diff --git a/tests/test_ohlcv_latest_bar.py b/tests/test_ohlcv_latest_bar.pynew file mode 100644index 000000000..7a26b0e00--- /dev/null+++ b/tests/test_ohlcv_latest_bar.py@@ -0,0 +1,136 @@+"""The latest trading day's bar must not silently vanish (#1201).++yfinance can return the newest in-range bar with a NaN close (an unsettled or+glitched session). The old path parsed dates without normalizing timezone and+dropped every NaN-close row before applying the curr_date cutoff, so the latest+bar disappeared and the previous trading day looked like the latest. Now dates+are normalized, and a latest in-range bar with no close raises rather than+silently falling back.+"""+from __future__ import annotations++import pandas as pd+import pytest++from tradingagents.dataflows import stockstats_utils as su+from tradingagents.dataflows.symbol_utils import NoMarketDataError++# --- date normalization -----------------------------------------------------++def test_normalize_dates_strips_tz_and_normalizes_to_midnight():+ aware = pd.Series(pd.to_datetime(+ ["2026-05-08 09:30:00-04:00", "2026-05-09 16:00:00-04:00"]+ ))+ out = su._normalize_dates(aware)+ assert out.dt.tz is None+ assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]+++def test_normalize_dates_leaves_naive_dates_at_midnight():+ naive = pd.Series(pd.to_datetime(["2026-05-08 14:30:00", "2026-05-09 00:00:00"]))+ out = su._normalize_dates(naive)+ assert out.dt.tz is None+ assert list(out) == [pd.Timestamp("2026-05-08"), pd.Timestamp("2026-05-09")]+++def test_normalize_dates_handles_mixed_dst_offsets():+ # 5y of US bars span DST; via a cache CSV they arrive as mixed-offset+ # strings, which pd.to_datetime can't unify. Each keeps its own local date.+ mixed = pd.Series([+ "2026-01-08 00:00:00-05:00", # EST+ "2026-06-08 00:00:00-04:00", # EDT+ "not-a-date", # -> NaT+ ])+ out = su._normalize_dates(mixed)+ assert out.iloc[0] == pd.Timestamp("2026-01-08")+ assert out.iloc[1] == pd.Timestamp("2026-06-08")+ assert pd.isna(out.iloc[2])+++def test_normalize_dates_keeps_positive_offset_local_date():+ # A Tokyo bar at local midnight (+09:00) must stay on its own calendar day,+ # not shift to the previous UTC day (which utc=True parsing would cause).+ jst = pd.Series(["2026-05-08 00:00:00+09:00"])+ assert su._normalize_dates(jst).iloc[0] == pd.Timestamp("2026-05-08")+++# --- fill vs guard responsibilities ----------------------------------------++def test_clean_dataframe_keeps_nan_close_for_the_caller_to_inspect():+ # _clean_dataframe normalizes but no longer drops the NaN close itself.+ df = pd.DataFrame({"Date": ["2026-05-08", "2026-05-09"], "Close": [100.0, float("nan")]})+ cleaned = su._clean_dataframe(df)+ assert len(cleaned) == 2+ assert pd.isna(cleaned["Close"].iloc[-1])+++def test_fill_price_gaps_drops_nan_close_rows():+ df = pd.DataFrame({"Date": pd.to_datetime(["2026-05-07", "2026-05-08"]),+ "Close": [float("nan"), 100.0]})+ filled = su._fill_price_gaps(df)+ assert len(filled) == 1+ assert filled["Close"].iloc[0] == 100.0+++# --- load_ohlcv end-to-end (with a mocked cache read) -----------------------++def _run_load(monkeypatch, tmp_path, frame, curr_date):+ """Drive load_ohlcv against a pre-seeded cache frame (no network)."""+ monkeypatch.setattr(su, "get_config", lambda: {"data_cache_dir": str(tmp_path)})+ today = pd.Timestamp(curr_date)+ monkeypatch.setattr(su.pd.Timestamp, "today", staticmethod(lambda: today))+ start = (today - pd.DateOffset(years=5)).strftime("%Y-%m-%d")+ end = (today + pd.Timedelta(days=1)).strftime("%Y-%m-%d")+ (tmp_path / f"AAPL-YFin-data-{start}-{end}.csv").write_text(frame.to_csv(index=False))++ def _fail_download(*a, **k):+ raise AssertionError("should use the seeded cache, not download")+ monkeypatch.setattr(su.yf, "download", _fail_download)+ monkeypatch.setattr(su, "_assert_ohlcv_not_stale", lambda *a, **k: None)+ return su.load_ohlcv("AAPL", curr_date)+++def test_latest_in_range_nan_close_raises_not_silent_fallback(monkeypatch, tmp_path):+ # Newest bar (the curr_date) has no close -> raise, don't return Thursday.+ frame = pd.DataFrame({+ "Date": ["2026-05-07", "2026-05-08"],+ "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],+ "Close": [100.5, float("nan")], "Volume": [1_000_000, 1_000_000],+ })+ with pytest.raises(NoMarketDataError, match="no closing price"):+ _run_load(monkeypatch, tmp_path, frame, "2026-05-08")+++def test_older_nan_close_row_is_still_dropped(monkeypatch, tmp_path):+ # A stale gap mid-series is dropped; the valid latest bar is served.+ frame = pd.DataFrame({+ "Date": ["2026-05-06", "2026-05-07", "2026-05-08"],+ "Open": [100.0, 101.0, 102.0], "High": [101.0, 102.0, 103.0],+ "Low": [99.0, 100.0, 101.0],+ "Close": [100.5, float("nan"), 102.5], "Volume": [1_000_000, 1_000_000, 1_000_000],+ })+ out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")+ assert out["Close"].iloc[-1] == 102.5+ assert (out["Date"] == pd.Timestamp("2026-05-07")).sum() == 0 # the NaN row is gone+++def test_tz_aware_latest_bar_is_kept_at_the_cutoff(monkeypatch, tmp_path):+ # A tz-aware/intraday latest bar on the cutoff day must not be filtered out+ # by a naive-vs-aware comparison.+ frame = pd.DataFrame({+ "Date": ["2026-05-07 09:30:00-04:00", "2026-05-08 09:30:00-04:00"],+ "Open": [100.0, 101.0], "High": [101.0, 102.0], "Low": [99.0, 100.0],+ "Close": [100.5, 101.5], "Volume": [1_000_000, 1_000_000],+ })+ out = _run_load(monkeypatch, tmp_path, frame, "2026-05-08")+ assert out["Close"].iloc[-1] == 101.5+ assert out["Date"].iloc[-1] == pd.Timestamp("2026-05-08")diff --git a/tradingagents/dataflows/stockstats_utils.py b/tradingagents/dataflows/stockstats_utils.pyindex 4ffe80422..94ba07ea8 100644--- a/tradingagents/dataflows/stockstats_utils.py+++ b/tradingagents/dataflows/stockstats_utils.py@@ -60,17 +60,53 @@ def _ensure_date_column(data: pd.DataFrame) -> pd.DataFrame:return data+def _local_midnight(value) -> pd.Timestamp:+ """A single timestamp as its naive, midnight-normalized local date (or NaT)."""+ if pd.isna(value):+ return pd.NaT+ try:+ ts = pd.Timestamp(value)+ except (ValueError, TypeError):+ return pd.NaT+ if ts.tzinfo is not None:+ ts = ts.tz_localize(None) # drop tz, keep the local wall-clock date+ return ts.normalize()+++def _normalize_dates(dates) -> pd.Series:+ """Parse to naive, midnight-normalized dates so tz-aware or intraday+ timestamps compare correctly against the naive ``curr_date`` cutoff (#1201).++ Normalized per element: 5 years of yfinance bars span daylight-saving+ changes (and cache CSVs round-trip the offsets as strings), so the series can+ carry mixed UTC offsets that ``pd.to_datetime`` cannot unify without+ ``utc=True`` — which would shift non-US (positive-offset) markets to the+ previous day. Keeping each bar's own local date avoids both.+ """+ return pd.to_datetime(pd.Series(dates).map(_local_midnight))++def _clean_dataframe(data: pd.DataFrame) -> pd.DataFrame:- """Normalize a stock DataFrame for stockstats: parse dates, drop invalid rows, fill price gaps."""+ """Normalize a stock DataFrame for stockstats: parse/normalize dates and+ coerce prices to numeric (NaN where invalid). Dropping incomplete rows and+ filling gaps is left to ``_fill_price_gaps`` so the caller can first inspect+ the latest in-range bar (#1201)."""data = _ensure_date_column(data)- data["Date"] = pd.to_datetime(data["Date"], errors="coerce")+ data["Date"] = _normalize_dates(data["Date"])data = data.dropna(subset=["Date"])price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]data[price_cols] = data[price_cols].apply(pd.to_numeric, errors="coerce")- data = data.dropna(subset=["Close"])- data[price_cols] = data[price_cols].ffill().bfill()+ return data++def _fill_price_gaps(data: pd.DataFrame) -> pd.DataFrame:+ """Drop rows with no close and forward/back-fill remaining price gaps so+ indicators compute on a continuous series."""+ price_cols = [c for c in ["Open", "High", "Low", "Close", "Volume"] if c in data.columns]+ # copy() so a filtered (sliced) input is written to safely, not via a view.+ data = data.dropna(subset=["Close"]).copy()+ data[price_cols] = data[price_cols].ffill().bfill()return data@@ -159,7 +195,7 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:safe_symbol = safe_ticker_component(canonical)config = get_config()- curr_date_dt = pd.to_datetime(curr_date)+ curr_date_dt = pd.to_datetime(curr_date).normalize()# Cache uses a fixed window (5y to today) so one file per symbol.today_date = pd.Timestamp.today()@@ -211,9 +247,20 @@ def load_ohlcv(symbol: str, curr_date: str) -> pd.DataFrame:data = _clean_dataframe(data)- # Filter to curr_date to prevent look-ahead bias in backtesting+ # Filter to curr_date to prevent look-ahead bias in backtesting.data = data[data["Date"] <= curr_date_dt]+ # Guard the latest in-range bar before dropping incomplete rows: a newest bar+ # with no close is "not settled yet", not "does not exist". Silently dropping+ # it would make the previous trading day look like the latest (#1201); raise+ # instead so the router surfaces it rather than fabricating a fallback.+ if not data.empty and pd.isna(data["Close"].iloc[-1]):+ raise NoMarketDataError(+ symbol, canonical, "latest in-range OHLCV bar has no closing price"+ )++ data = _fill_price_gaps(data)+# Reject a stale frame (latest row far older than curr_date) rather than# feeding year-old prices into indicators (#1021)._assert_ohlcv_not_stale(data, curr_date, symbol, canonical)