"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(loaders): never mix adjusted and raw prices in one series
cgycoreyebb97f8ef173
diff --git a/agent/backtest/loaders/fmp_loader.py b/agent/backtest/loaders/fmp_loader.pyindex 0adeaf6c4..60262f438 100644--- a/agent/backtest/loaders/fmp_loader.py+++ b/agent/backtest/loaders/fmp_loader.py@@ -21,6 +21,7 @@ Auth: set ``FMP_API_KEY`` in the environment. Covers US equities only.from __future__ import annotationsimport logging+import mathfrom typing import Any, Dict, List, Optionalimport pandas as pd@@ -189,6 +190,33 @@ class DataLoader:return _parse_historical(payload)+def _adjusted_bar(bar: dict) -> Optional[tuple[float, float, float, float]]:+ """The adjusted ``(open, high, low, close)`` one FMP bar yields, or ``None``.++ FMP carries only ``adjClose``, so the rest of a bar is its raw OHLC scaled+ by the ``adjClose/close`` factor. ``None`` means the bar cannot be part of+ an adjusted series — a price that is missing, non-numeric, non-finite or+ non-positive, or a factor outside the sane 0.01-100x window — so it may+ neither supply the series' basis nor, once another bar supplies it, be+ replaced with raw prices.+ """+ try:+ o, h, lo, c = (+ float(bar[field]) for field in ("open", "high", "low", "close")+ )+ adj_close = float(bar["adjClose"])+ except (KeyError, TypeError, ValueError):+ return None+ if not all(+ math.isfinite(value) and value > 0 for value in (o, h, lo, c, adj_close)+ ):+ return None+ ratio = adj_close / c+ if not 0.01 <= ratio <= 100:+ return None+ return o * ratio, h * ratio, lo * ratio, adj_close++def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:"""Convert an FMP historical-price body into an ascending OHLCV frame.@@ -201,6 +229,10 @@ def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:eastmoney/tencent/yahoo/yfinance serve, and what this source is registeredas in ``PRICE_CALIBER_BY_SOURCE``. Volume is never scaled.+ A bar that cannot yield a complete, finite, positive adjusted bar is+ dropped whenever any other bar can, so one series never mixes the two+ price bases; a response in which no bar can stays consistently raw.+Args:payload: Decoded JSON body from the historical-price endpoint.Accepts both the legacy ``{"historical": [...]}`` dict and the@@ -220,33 +252,58 @@ def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:if not historical:return None+ has_adjusted_data = any(+ isinstance(bar, dict)+ and bar.get("date") is not None+ and _adjusted_bar(bar) is not None+ for bar in historical+ )rows = []+ dropped = 0for bar in historical:- if not isinstance(bar, dict) or "date" not in bar:+ if not isinstance(bar, dict) or bar.get("date") is None:+ continue+ basis = _adjusted_bar(bar)+ if basis is not None:+ o, h, lo, c = basis+ elif has_adjusted_data:+ # Preserve a single price basis: a bar whose adjustment is missing+ # or unusable cannot be replaced with raw OHLC when others are+ # adjusted. A response with no usable adjustment at all stays raw,+ # consistently.+ dropped += 1continue- adj_close = bar.get("adjClose")- close_raw = bar.get("close")- ratio: Optional[float] = None- try:- if adj_close is not None and close_raw is not None:- ac = float(adj_close)- cr = float(close_raw)- if cr > 0 and ac > 0:- r = ac / cr- if 0.01 <= r <= 100:- ratio = r- except (TypeError, ValueError):- ratio = None- row = {"trade_date": bar["date"]}- for field in _OHLCV_FIELDS:- val = bar.get(field)- if field != "volume" and ratio is not None and val is not None:- try:- val = float(val) * ratio- except (TypeError, ValueError):- pass- row[field] = val- rows.append(row)+ else:+ # Nothing to adjust against: the whole series stays on the raw+ # basis rather than the response being discarded.+ o, h, lo, c = (+ bar.get("open"),+ bar.get("high"),+ bar.get("low"),+ bar.get("close"),+ )+ rows.append(+ {+ "trade_date": bar["date"],+ "open": o,+ "high": h,+ "low": lo,+ "close": c,+ "volume": bar.get("volume"),+ }+ )++ if dropped:+ # Say so rather than truncate in silence: a long-history symbol whose+ # older bars fall outside the 0.01-100x factor window loses them, and+ # the backtest then runs a shorter window than it asked for.+ logger.warning(+ "FMP: dropped %d of %d bars that carry no usable adjustment while "+ "the rest do; the series keeps one price basis instead of mixing "+ "raw and adjusted prices",+ dropped,+ len(historical),+ )if not rows:return Nonediff --git a/agent/backtest/loaders/tiingo_loader.py b/agent/backtest/loaders/tiingo_loader.pyindex 4cf6c0dfb..e0fbcbb62 100644--- a/agent/backtest/loaders/tiingo_loader.py+++ b/agent/backtest/loaders/tiingo_loader.py@@ -20,7 +20,8 @@ rejects unspaced bursts.from __future__ import annotationsimport logging-from typing import Dict, List, Optional+import math+from typing import Any, Dict, List, Optionalimport pandas as pd@@ -83,14 +84,71 @@ def _to_tiingo_symbol(code: str) -> Optional[str]:return upper.lower()+def _positive_prices(*values: Any) -> Optional[tuple[float, ...]]:+ """Finite, strictly positive floats for every value, or ``None``.++ A missing, non-numeric, non-finite or non-positive price makes the whole+ set unusable: zero and infinity are not prices, and a bar built from one is+ a fabricated return rather than a missing one.+ """+ try:+ numbers = tuple(float(value) for value in values)+ except (TypeError, ValueError):+ return None+ if not all(math.isfinite(number) and number > 0 for number in numbers):+ return None+ return numbers+++def _adjusted_basis(+ row: dict,+) -> Optional[tuple[float, float, float, float]]:+ """The adjusted ``(open, high, low, close)`` one bar yields, or ``None``.++ Tiingo's fully adjusted OHLC is preferred; when only ``adjClose`` is+ present the raw OHLC is scaled by the ``adjClose/close`` factor so the four+ stay consistent. ``None`` means the bar cannot be part of an adjusted+ series — a price in the set being used that is missing, non-numeric,+ non-finite or non-positive, or a factor outside the sane 0.01-100x window —+ so it may neither supply the series' basis nor, once another bar supplies+ it, be replaced with raw prices.+ """+ if all(+ row.get(key) is not None for key in ("adjOpen", "adjHigh", "adjLow", "adjClose")+ ):+ prices = _positive_prices(+ row["adjOpen"], row["adjHigh"], row["adjLow"], row["adjClose"]+ )+ if prices is None:+ return None+ o, h, lo, c = prices+ return o, h, lo, c+ prices = _positive_prices(+ row.get("open"),+ row.get("high"),+ row.get("low"),+ row.get("close"),+ row.get("adjClose"),+ )+ if prices is None:+ return None+ o, h, lo, c, adj_close = prices+ ratio = adj_close / c+ if not 0.01 <= ratio <= 100:+ return None+ return o * ratio, h * ratio, lo * ratio, adj_close++def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:"""Convert Tiingo's per-day records into the normalized OHLCV frame.Prefers Tiingo's adjusted OHLC (``adjOpen``/``adjHigh``/``adjLow``/``adjClose``) when present so the frame is dividend- and split-adjusted- (qfq). Falls back to raw ``open``/``high``/``low``/``close`` when adjusted- fields are absent, and to an ``adjClose/close`` ratio when only- ``adjClose`` is present.+ (qfq). Falls back to an ``adjClose/close`` ratio when only ``adjClose`` is+ present, and to raw ``open``/``high``/``low``/``close`` when no bar carries+ a usable adjustment at all. A bar whose adjustment is missing or unusable+ is dropped whenever other bars are adjusted, so one series never mixes two+ price bases.Args:rows: JSON array decoded from the prices endpoint; each item carries a@@ -102,70 +160,32 @@ def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:with float ``open``/``high``/``low``/``close``/``volume`` columns(adjusted when available), or ``None`` when no usable bar is present."""+ dated = [+ row for row in rows if isinstance(row, dict) and row.get("date") is not None+ ]+ has_adjusted_data = any(_adjusted_basis(row) is not None for row in dated)parsed: List[dict] = []- for row in rows:+ dropped = 0+ for row in dated:date = row.get("date")- if date is None:+ basis = _adjusted_basis(row)+ if basis is not None:+ o, h, lo, c = basis+ elif has_adjusted_data:+ # Do not fill a missing/invalid adjusted bar with raw prices: that+ # would mix bases within this series.+ dropped += 1continue- # Prefer fully adjusted OHLC when Tiingo provides them.- has_adj_ohlc = all(- row.get(k) is not None for k in ("adjOpen", "adjHigh", "adjLow", "adjClose")- )- if has_adj_ohlc:+ else:+ # No bar carries a usable adjustment (older/unadjusted endpoint+ # responses, or a response that is unusable throughout): the whole+ # series consistently remains raw rather than being discarded.o, h, lo, c = (- row.get("adjOpen"),- row.get("adjHigh"),- row.get("adjLow"),- row.get("adjClose"),+ row.get("open"),+ row.get("high"),+ row.get("low"),+ row.get("close"),)- else:- # If only adjClose is present, derive the dividend/split factor and- # scale raw OHLC so high/low/open stay consistent with the adjusted close.- adj_close = row.get("adjClose")- raw_close = row.get("close")- ratio = None- try:- if adj_close is not None and raw_close is not None:- ac = float(adj_close)- rc = float(raw_close)- if rc > 0 and ac > 0:- r = ac / rc- if 0.01 <= r <= 100:- ratio = r- except (TypeError, ValueError):- ratio = None- if ratio is not None:- try:- o = (- float(row.get("open")) * ratio- if row.get("open") is not None- else None- )- h = (- float(row.get("high")) * ratio- if row.get("high") is not None- else None- )- lo = (- float(row.get("low")) * ratio- if row.get("low") is not None- else None- )- c = float(adj_close)- except (TypeError, ValueError):- o, h, lo, c = (- row.get("open"),- row.get("high"),- row.get("low"),- row.get("close"),- )- else:- o, h, lo, c = (- row.get("open"),- row.get("high"),- row.get("low"),- row.get("close"),- )parsed.append({"trade_date": date,@@ -176,6 +196,18 @@ def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:"volume": row.get("volume"),})+ if dropped:+ # Say so rather than truncate in silence: older bars of a long-history+ # symbol can fall outside the 0.01-100x factor window, and the backtest+ # then runs a shorter window than it asked for.+ logger.warning(+ "Tiingo: dropped %d of %d bars that carry no usable adjustment "+ "while the rest do; the series keeps one price basis instead of "+ "mixing raw and adjusted prices",+ dropped,+ len(dated),+ )+if not parsed:return Nonediff --git a/agent/tests/test_fmp_loader.py b/agent/tests/test_fmp_loader.pyindex 761aecefb..03de7dea6 100644--- a/agent/tests/test_fmp_loader.py+++ b/agent/tests/test_fmp_loader.py@@ -109,6 +109,140 @@ class TestParseHistorical:]assert _parse_historical(_body("AAPL", bars)) is None+ @pytest.mark.parametrize("missing", [None, "absent"])+ def test_missing_adjusted_close_does_not_mix_raw_prices(self, missing):+ second = {"date": "2024-01-04", "open": 100, "high": 102, "low": 99, "close": 100, "volume": 1000}+ if missing is None:+ second["adjClose"] = None+ bars = [+ {"date": "2024-01-03", "open": 100, "high": 102, "low": 99, "close": 100, "adjClose": 50, "volume": 1000},+ second,+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ assert list(df["close"]) == [50.0]++ @pytest.mark.parametrize("unusable", ["0", "", "abc", 20000])+ def test_unusable_adjusted_close_does_not_mix_raw_prices(self, unusable):+ bars = [+ {+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,+ "close": 100, "adjClose": 50, "volume": 1000,+ },+ {+ "date": "2024-01-04", "open": 100, "high": 102, "low": 99,+ "close": 100, "adjClose": unusable, "volume": 1000,+ },+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ # An adjustment that cannot be computed is not a reason to fall back to+ # the raw basis: that mixes two price scales in one series.+ assert list(df["close"]) == [50.0]++ def test_a_response_with_no_usable_adjustment_stays_one_raw_basis(self):+ bars = [+ {+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,+ "close": 100, "adjClose": "", "volume": 1000,+ },+ {+ "date": "2024-01-04", "open": 101, "high": 103, "low": 100,+ "close": 101, "adjClose": "0", "volume": 1000,+ },+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ # Nothing to adjust against: the whole series stays on the raw basis+ # rather than the response being discarded.+ assert list(df["close"]) == [100.0, 101.0]++ def test_a_bar_without_a_usable_date_is_dropped(self):+ """A bar that cannot be placed in time is not a bar.++ The date cell becomes the frame index, so a null one used to enter the+ frame as ``NaT`` rather than being discarded.+ """+ bars = [+ {"date": "2024-01-03", "open": 1.0, "high": 2.0, "low": 0.5, "close": 1.5, "volume": 100.0},+ {"date": None, "open": 9.0, "high": 9.0, "low": 9.0, "close": 9.0, "volume": 100.0},+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ assert list(df["close"]) == [1.5]++ def test_a_dropped_bar_is_reported_not_truncated_in_silence(self, caplog):+ """A long-history symbol can lose old bars to the 0.01-100x factor window."""+ bars = [+ # A 1997-style bar: a 600x cumulative split factor puts adjClose/close+ # far below the 0.01 floor, so it cannot join an adjusted series.+ {+ "date": "1997-05-15", "open": 1.0, "high": 1.5, "low": 0.9,+ "close": 1.2, "adjClose": 0.002, "volume": 1000,+ },+ {+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,+ "close": 100, "adjClose": 50, "volume": 1000,+ },+ ]++ with caplog.at_level("WARNING", logger="backtest.loaders.fmp_loader"):+ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ assert list(df["close"]) == [50.0]+ assert "dropped 1 of 2" in caplog.text++ def test_a_bar_that_cannot_be_emitted_does_not_set_the_adjusted_basis(self):+ """A computable factor on an incomplete bar must not claim the basis.++ Such a bar is dropped either way; what it must not do is drag its+ unadjusted siblings down with it, which turned a usable raw series+ into no series at all.+ """+ bars = [+ {+ "date": "2024-01-03", "open": None, "high": 102, "low": 99,+ "close": 100, "adjClose": 50, "volume": 1000,+ },+ {+ "date": "2024-01-04", "open": 101, "high": 103, "low": 100,+ "close": 101, "volume": 1000,+ },+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ assert list(df["close"]) == [101.0]++ def test_an_infinite_price_is_never_scaled_into_the_series(self):+ """A non-finite leg is not a price, so the bar cannot be adjusted."""+ bars = [+ {+ "date": "2024-01-03", "open": float("inf"), "high": 102, "low": 99,+ "close": 100, "adjClose": 50, "volume": 1000,+ },+ {+ "date": "2024-01-04", "open": 100, "high": 102, "low": 99,+ "close": 100, "adjClose": 50, "volume": 1000,+ },+ ]++ df = _parse_historical(_body("AAPL", bars))++ assert df is not None+ assert len(df) == 1+ assert list(df["close"]) == [50.0]+class TestFetch:"""End-to-end fetch with the HTTP layer mocked."""diff --git a/agent/tests/test_tiingo_loader.py b/agent/tests/test_tiingo_loader.pyindex 721e62c4e..ec21fd208 100644--- a/agent/tests/test_tiingo_loader.py+++ b/agent/tests/test_tiingo_loader.py@@ -113,6 +113,143 @@ def test_rows_to_frame_drops_rows_missing_ohlc() -> None:assert len(df) == 2 # row with no OHLC dropped+def test_rows_with_incomplete_adjusted_ohlc_do_not_mix_price_bases() -> None:+ rows = [+ {+ "date": "2024-01-02T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": None, "adjHigh": None, "adjLow": None, "adjClose": None,+ },+ ]++ df = _rows_to_frame(rows)++ assert df is not None+ # This loader promises split/dividend-adjusted prices, so reject the row+ # without adjusted prices rather than fabricate a 100% return from raw data.+ assert list(df["close"]) == [50.0]+++def test_zero_adjusted_ohlc_is_rejected_not_priced() -> None:+ rows = [+ {+ "date": "2024-01-02T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,+ },+ ]++ df = _rows_to_frame(rows)++ assert df is not None+ # A zero-adjusted bar is not a price: accepting it fabricates a -100% bar.+ assert list(df["close"]) == [50.0]+++def test_an_all_unusable_adjusted_response_stays_one_raw_basis() -> None:+ rows = [+ {+ "date": "2024-01-02T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 101, "high": 102, "low": 100, "close": 101, "volume": 1000,+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,+ },+ ]++ df = _rows_to_frame(rows)++ # No bar carries a usable adjustment, so the series stays consistently raw+ # (one basis) rather than being discarded or priced at zero.+ assert df is not None+ assert list(df["close"]) == [100.0, 101.0]+++def test_a_bar_that_cannot_be_emitted_does_not_set_the_adjusted_basis() -> None:+ """A computable factor on an incomplete bar must not claim the basis.++ Such a bar is dropped either way; what it must not do is drag its+ unadjusted siblings down with it, which turned a usable raw series into no+ series at all.+ """+ rows = [+ {+ "date": "2024-01-02T00:00:00.000Z",+ "open": None, "high": 102, "low": 99, "close": 100, "volume": 1000,+ "adjClose": 50,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 101, "high": 103, "low": 100, "close": 101, "volume": 1000,+ },+ ]++ df = _rows_to_frame(rows)++ assert df is not None+ assert list(df["close"]) == [101.0]+++def test_a_non_finite_adjusted_price_is_not_served() -> None:+ """An adjusted leg of infinity is not a price: the bar is not adjusted."""+ rows = [+ {+ "date": "2024-01-02T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": float("inf"), "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,+ },+ ]++ df = _rows_to_frame(rows)++ assert df is not None+ assert list(df["close"]) == [50.0]+++def test_a_dropped_row_is_reported_not_truncated_in_silence(caplog) -> None:+ """A long-history symbol can lose old rows to the 0.01-100x factor window."""+ rows = [+ # A 1997-style row: only adjClose is present and its 600x cumulative+ # factor (0.002/1.2) sits below the 0.01 floor, so it cannot join the+ # adjusted series.+ {+ "date": "1997-05-15T00:00:00.000Z",+ "open": 1.0, "high": 1.5, "low": 0.9, "close": 1.2, "volume": 1000,+ "adjClose": 0.002,+ },+ {+ "date": "2024-01-03T00:00:00.000Z",+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,+ },+ ]++ with caplog.at_level("WARNING", logger="backtest.loaders.tiingo_loader"):+ df = _rows_to_frame(rows)++ assert df is not None+ assert list(df["close"]) == [50.0]+ assert "dropped 1 of 2" in caplog.text++# ---------------------------------------------------------------------------# fetch() behavior (HTTP mocked)# ---------------------------------------------------------------------------