Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(loaders): never mix adjusted and raw prices in one series

cgycoreyebb97f8ef173
diff --git a/agent/backtest/loaders/fmp_loader.py b/agent/backtest/loaders/fmp_loader.py
index 0adeaf6c4..60262f438 100644
--- a/agent/backtest/loaders/fmp_loader.py
+++ b/agent/backtest/loaders/fmp_loader.py
@@ -21,6 +21,7 @@ Auth: set ``FMP_API_KEY`` in the environment. Covers US equities only.
from __future__ import annotations
import logging
+import math
from typing import Any, Dict, List, Optional
import pandas as pd
@@ -189,6 +190,33 @@ class DataLoader:
return _parse_historical(payload)
+def _adjusted_bar(bar: dict) -> Optional[tuple[float, float, float, float]]:
+ """The adjusted ``(open, high, low, close)`` one FMP bar yields, or ``None``.
+
+ FMP carries only ``adjClose``, so the rest of a bar is its raw OHLC scaled
+ by the ``adjClose/close`` factor. ``None`` means the bar cannot be part of
+ an adjusted series — a price that is missing, non-numeric, non-finite or
+ non-positive, or a factor outside the sane 0.01-100x window — so it may
+ neither supply the series' basis nor, once another bar supplies it, be
+ replaced with raw prices.
+ """
+ try:
+ o, h, lo, c = (
+ float(bar[field]) for field in ("open", "high", "low", "close")
+ )
+ adj_close = float(bar["adjClose"])
+ except (KeyError, TypeError, ValueError):
+ return None
+ if not all(
+ math.isfinite(value) and value > 0 for value in (o, h, lo, c, adj_close)
+ ):
+ return None
+ ratio = adj_close / c
+ if not 0.01 <= ratio <= 100:
+ return None
+ return o * ratio, h * ratio, lo * ratio, adj_close
+
+
def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:
"""Convert an FMP historical-price body into an ascending OHLCV frame.
@@ -201,6 +229,10 @@ def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:
eastmoney/tencent/yahoo/yfinance serve, and what this source is registered
as in ``PRICE_CALIBER_BY_SOURCE``. Volume is never scaled.
+ A bar that cannot yield a complete, finite, positive adjusted bar is
+ dropped whenever any other bar can, so one series never mixes the two
+ price bases; a response in which no bar can stays consistently raw.
+
Args:
payload: Decoded JSON body from the historical-price endpoint.
Accepts both the legacy ``{"historical": [...]}`` dict and the
@@ -220,33 +252,58 @@ def _parse_historical(payload: Any) -> Optional[pd.DataFrame]:
if not historical:
return None
+ has_adjusted_data = any(
+ isinstance(bar, dict)
+ and bar.get("date") is not None
+ and _adjusted_bar(bar) is not None
+ for bar in historical
+ )
rows = []
+ dropped = 0
for bar in historical:
- if not isinstance(bar, dict) or "date" not in bar:
+ if not isinstance(bar, dict) or bar.get("date") is None:
+ continue
+ basis = _adjusted_bar(bar)
+ if basis is not None:
+ o, h, lo, c = basis
+ elif has_adjusted_data:
+ # Preserve a single price basis: a bar whose adjustment is missing
+ # or unusable cannot be replaced with raw OHLC when others are
+ # adjusted. A response with no usable adjustment at all stays raw,
+ # consistently.
+ dropped += 1
continue
- adj_close = bar.get("adjClose")
- close_raw = bar.get("close")
- ratio: Optional[float] = None
- try:
- if adj_close is not None and close_raw is not None:
- ac = float(adj_close)
- cr = float(close_raw)
- if cr > 0 and ac > 0:
- r = ac / cr
- if 0.01 <= r <= 100:
- ratio = r
- except (TypeError, ValueError):
- ratio = None
- row = {"trade_date": bar["date"]}
- for field in _OHLCV_FIELDS:
- val = bar.get(field)
- if field != "volume" and ratio is not None and val is not None:
- try:
- val = float(val) * ratio
- except (TypeError, ValueError):
- pass
- row[field] = val
- rows.append(row)
+ else:
+ # Nothing to adjust against: the whole series stays on the raw
+ # basis rather than the response being discarded.
+ o, h, lo, c = (
+ bar.get("open"),
+ bar.get("high"),
+ bar.get("low"),
+ bar.get("close"),
+ )
+ rows.append(
+ {
+ "trade_date": bar["date"],
+ "open": o,
+ "high": h,
+ "low": lo,
+ "close": c,
+ "volume": bar.get("volume"),
+ }
+ )
+
+ if dropped:
+ # Say so rather than truncate in silence: a long-history symbol whose
+ # older bars fall outside the 0.01-100x factor window loses them, and
+ # the backtest then runs a shorter window than it asked for.
+ logger.warning(
+ "FMP: dropped %d of %d bars that carry no usable adjustment while "
+ "the rest do; the series keeps one price basis instead of mixing "
+ "raw and adjusted prices",
+ dropped,
+ len(historical),
+ )
if not rows:
return None
diff --git a/agent/backtest/loaders/tiingo_loader.py b/agent/backtest/loaders/tiingo_loader.py
index 4cf6c0dfb..e0fbcbb62 100644
--- a/agent/backtest/loaders/tiingo_loader.py
+++ b/agent/backtest/loaders/tiingo_loader.py
@@ -20,7 +20,8 @@ rejects unspaced bursts.
from __future__ import annotations
import logging
-from typing import Dict, List, Optional
+import math
+from typing import Any, Dict, List, Optional
import pandas as pd
@@ -83,14 +84,71 @@ def _to_tiingo_symbol(code: str) -> Optional[str]:
return upper.lower()
+def _positive_prices(*values: Any) -> Optional[tuple[float, ...]]:
+ """Finite, strictly positive floats for every value, or ``None``.
+
+ A missing, non-numeric, non-finite or non-positive price makes the whole
+ set unusable: zero and infinity are not prices, and a bar built from one is
+ a fabricated return rather than a missing one.
+ """
+ try:
+ numbers = tuple(float(value) for value in values)
+ except (TypeError, ValueError):
+ return None
+ if not all(math.isfinite(number) and number > 0 for number in numbers):
+ return None
+ return numbers
+
+
+def _adjusted_basis(
+ row: dict,
+) -> Optional[tuple[float, float, float, float]]:
+ """The adjusted ``(open, high, low, close)`` one bar yields, or ``None``.
+
+ Tiingo's fully adjusted OHLC is preferred; when only ``adjClose`` is
+ present the raw OHLC is scaled by the ``adjClose/close`` factor so the four
+ stay consistent. ``None`` means the bar cannot be part of an adjusted
+ series — a price in the set being used that is missing, non-numeric,
+ non-finite or non-positive, or a factor outside the sane 0.01-100x window —
+ so it may neither supply the series' basis nor, once another bar supplies
+ it, be replaced with raw prices.
+ """
+ if all(
+ row.get(key) is not None for key in ("adjOpen", "adjHigh", "adjLow", "adjClose")
+ ):
+ prices = _positive_prices(
+ row["adjOpen"], row["adjHigh"], row["adjLow"], row["adjClose"]
+ )
+ if prices is None:
+ return None
+ o, h, lo, c = prices
+ return o, h, lo, c
+ prices = _positive_prices(
+ row.get("open"),
+ row.get("high"),
+ row.get("low"),
+ row.get("close"),
+ row.get("adjClose"),
+ )
+ if prices is None:
+ return None
+ o, h, lo, c, adj_close = prices
+ ratio = adj_close / c
+ if not 0.01 <= ratio <= 100:
+ return None
+ return o * ratio, h * ratio, lo * ratio, adj_close
+
+
def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:
"""Convert Tiingo's per-day records into the normalized OHLCV frame.
Prefers Tiingo's adjusted OHLC (``adjOpen``/``adjHigh``/``adjLow``/
``adjClose``) when present so the frame is dividend- and split-adjusted
- (qfq). Falls back to raw ``open``/``high``/``low``/``close`` when adjusted
- fields are absent, and to an ``adjClose/close`` ratio when only
- ``adjClose`` is present.
+ (qfq). Falls back to an ``adjClose/close`` ratio when only ``adjClose`` is
+ present, and to raw ``open``/``high``/``low``/``close`` when no bar carries
+ a usable adjustment at all. A bar whose adjustment is missing or unusable
+ is dropped whenever other bars are adjusted, so one series never mixes two
+ price bases.
Args:
rows: JSON array decoded from the prices endpoint; each item carries a
@@ -102,70 +160,32 @@ def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:
with float ``open``/``high``/``low``/``close``/``volume`` columns
(adjusted when available), or ``None`` when no usable bar is present.
"""
+ dated = [
+ row for row in rows if isinstance(row, dict) and row.get("date") is not None
+ ]
+ has_adjusted_data = any(_adjusted_basis(row) is not None for row in dated)
parsed: List[dict] = []
- for row in rows:
+ dropped = 0
+ for row in dated:
date = row.get("date")
- if date is None:
+ basis = _adjusted_basis(row)
+ if basis is not None:
+ o, h, lo, c = basis
+ elif has_adjusted_data:
+ # Do not fill a missing/invalid adjusted bar with raw prices: that
+ # would mix bases within this series.
+ dropped += 1
continue
- # Prefer fully adjusted OHLC when Tiingo provides them.
- has_adj_ohlc = all(
- row.get(k) is not None for k in ("adjOpen", "adjHigh", "adjLow", "adjClose")
- )
- if has_adj_ohlc:
+ else:
+ # No bar carries a usable adjustment (older/unadjusted endpoint
+ # responses, or a response that is unusable throughout): the whole
+ # series consistently remains raw rather than being discarded.
o, h, lo, c = (
- row.get("adjOpen"),
- row.get("adjHigh"),
- row.get("adjLow"),
- row.get("adjClose"),
+ row.get("open"),
+ row.get("high"),
+ row.get("low"),
+ row.get("close"),
)
- else:
- # If only adjClose is present, derive the dividend/split factor and
- # scale raw OHLC so high/low/open stay consistent with the adjusted close.
- adj_close = row.get("adjClose")
- raw_close = row.get("close")
- ratio = None
- try:
- if adj_close is not None and raw_close is not None:
- ac = float(adj_close)
- rc = float(raw_close)
- if rc > 0 and ac > 0:
- r = ac / rc
- if 0.01 <= r <= 100:
- ratio = r
- except (TypeError, ValueError):
- ratio = None
- if ratio is not None:
- try:
- o = (
- float(row.get("open")) * ratio
- if row.get("open") is not None
- else None
- )
- h = (
- float(row.get("high")) * ratio
- if row.get("high") is not None
- else None
- )
- lo = (
- float(row.get("low")) * ratio
- if row.get("low") is not None
- else None
- )
- c = float(adj_close)
- except (TypeError, ValueError):
- o, h, lo, c = (
- row.get("open"),
- row.get("high"),
- row.get("low"),
- row.get("close"),
- )
- else:
- o, h, lo, c = (
- row.get("open"),
- row.get("high"),
- row.get("low"),
- row.get("close"),
- )
parsed.append(
{
"trade_date": date,
@@ -176,6 +196,18 @@ def _rows_to_frame(rows: List[dict]) -> Optional[pd.DataFrame]:
"volume": row.get("volume"),
}
)
+ if dropped:
+ # Say so rather than truncate in silence: older bars of a long-history
+ # symbol can fall outside the 0.01-100x factor window, and the backtest
+ # then runs a shorter window than it asked for.
+ logger.warning(
+ "Tiingo: dropped %d of %d bars that carry no usable adjustment "
+ "while the rest do; the series keeps one price basis instead of "
+ "mixing raw and adjusted prices",
+ dropped,
+ len(dated),
+ )
+
if not parsed:
return None
diff --git a/agent/tests/test_fmp_loader.py b/agent/tests/test_fmp_loader.py
index 761aecefb..03de7dea6 100644
--- a/agent/tests/test_fmp_loader.py
+++ b/agent/tests/test_fmp_loader.py
@@ -109,6 +109,140 @@ class TestParseHistorical:
]
assert _parse_historical(_body("AAPL", bars)) is None
+ @pytest.mark.parametrize("missing", [None, "absent"])
+ def test_missing_adjusted_close_does_not_mix_raw_prices(self, missing):
+ second = {"date": "2024-01-04", "open": 100, "high": 102, "low": 99, "close": 100, "volume": 1000}
+ if missing is None:
+ second["adjClose"] = None
+ bars = [
+ {"date": "2024-01-03", "open": 100, "high": 102, "low": 99, "close": 100, "adjClose": 50, "volume": 1000},
+ second,
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ assert list(df["close"]) == [50.0]
+
+ @pytest.mark.parametrize("unusable", ["0", "", "abc", 20000])
+ def test_unusable_adjusted_close_does_not_mix_raw_prices(self, unusable):
+ bars = [
+ {
+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,
+ "close": 100, "adjClose": 50, "volume": 1000,
+ },
+ {
+ "date": "2024-01-04", "open": 100, "high": 102, "low": 99,
+ "close": 100, "adjClose": unusable, "volume": 1000,
+ },
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ # An adjustment that cannot be computed is not a reason to fall back to
+ # the raw basis: that mixes two price scales in one series.
+ assert list(df["close"]) == [50.0]
+
+ def test_a_response_with_no_usable_adjustment_stays_one_raw_basis(self):
+ bars = [
+ {
+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,
+ "close": 100, "adjClose": "", "volume": 1000,
+ },
+ {
+ "date": "2024-01-04", "open": 101, "high": 103, "low": 100,
+ "close": 101, "adjClose": "0", "volume": 1000,
+ },
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ # Nothing to adjust against: the whole series stays on the raw basis
+ # rather than the response being discarded.
+ assert list(df["close"]) == [100.0, 101.0]
+
+ def test_a_bar_without_a_usable_date_is_dropped(self):
+ """A bar that cannot be placed in time is not a bar.
+
+ The date cell becomes the frame index, so a null one used to enter the
+ frame as ``NaT`` rather than being discarded.
+ """
+ bars = [
+ {"date": "2024-01-03", "open": 1.0, "high": 2.0, "low": 0.5, "close": 1.5, "volume": 100.0},
+ {"date": None, "open": 9.0, "high": 9.0, "low": 9.0, "close": 9.0, "volume": 100.0},
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ assert list(df["close"]) == [1.5]
+
+ def test_a_dropped_bar_is_reported_not_truncated_in_silence(self, caplog):
+ """A long-history symbol can lose old bars to the 0.01-100x factor window."""
+ bars = [
+ # A 1997-style bar: a 600x cumulative split factor puts adjClose/close
+ # far below the 0.01 floor, so it cannot join an adjusted series.
+ {
+ "date": "1997-05-15", "open": 1.0, "high": 1.5, "low": 0.9,
+ "close": 1.2, "adjClose": 0.002, "volume": 1000,
+ },
+ {
+ "date": "2024-01-03", "open": 100, "high": 102, "low": 99,
+ "close": 100, "adjClose": 50, "volume": 1000,
+ },
+ ]
+
+ with caplog.at_level("WARNING", logger="backtest.loaders.fmp_loader"):
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ assert list(df["close"]) == [50.0]
+ assert "dropped 1 of 2" in caplog.text
+
+ def test_a_bar_that_cannot_be_emitted_does_not_set_the_adjusted_basis(self):
+ """A computable factor on an incomplete bar must not claim the basis.
+
+ Such a bar is dropped either way; what it must not do is drag its
+ unadjusted siblings down with it, which turned a usable raw series
+ into no series at all.
+ """
+ bars = [
+ {
+ "date": "2024-01-03", "open": None, "high": 102, "low": 99,
+ "close": 100, "adjClose": 50, "volume": 1000,
+ },
+ {
+ "date": "2024-01-04", "open": 101, "high": 103, "low": 100,
+ "close": 101, "volume": 1000,
+ },
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ assert list(df["close"]) == [101.0]
+
+ def test_an_infinite_price_is_never_scaled_into_the_series(self):
+ """A non-finite leg is not a price, so the bar cannot be adjusted."""
+ bars = [
+ {
+ "date": "2024-01-03", "open": float("inf"), "high": 102, "low": 99,
+ "close": 100, "adjClose": 50, "volume": 1000,
+ },
+ {
+ "date": "2024-01-04", "open": 100, "high": 102, "low": 99,
+ "close": 100, "adjClose": 50, "volume": 1000,
+ },
+ ]
+
+ df = _parse_historical(_body("AAPL", bars))
+
+ assert df is not None
+ assert len(df) == 1
+ assert list(df["close"]) == [50.0]
+
class TestFetch:
"""End-to-end fetch with the HTTP layer mocked."""
diff --git a/agent/tests/test_tiingo_loader.py b/agent/tests/test_tiingo_loader.py
index 721e62c4e..ec21fd208 100644
--- a/agent/tests/test_tiingo_loader.py
+++ b/agent/tests/test_tiingo_loader.py
@@ -113,6 +113,143 @@ def test_rows_to_frame_drops_rows_missing_ohlc() -> None:
assert len(df) == 2 # row with no OHLC dropped
+def test_rows_with_incomplete_adjusted_ohlc_do_not_mix_price_bases() -> None:
+ rows = [
+ {
+ "date": "2024-01-02T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": None, "adjHigh": None, "adjLow": None, "adjClose": None,
+ },
+ ]
+
+ df = _rows_to_frame(rows)
+
+ assert df is not None
+ # This loader promises split/dividend-adjusted prices, so reject the row
+ # without adjusted prices rather than fabricate a 100% return from raw data.
+ assert list(df["close"]) == [50.0]
+
+
+def test_zero_adjusted_ohlc_is_rejected_not_priced() -> None:
+ rows = [
+ {
+ "date": "2024-01-02T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,
+ },
+ ]
+
+ df = _rows_to_frame(rows)
+
+ assert df is not None
+ # A zero-adjusted bar is not a price: accepting it fabricates a -100% bar.
+ assert list(df["close"]) == [50.0]
+
+
+def test_an_all_unusable_adjusted_response_stays_one_raw_basis() -> None:
+ rows = [
+ {
+ "date": "2024-01-02T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 101, "high": 102, "low": 100, "close": 101, "volume": 1000,
+ "adjOpen": 0, "adjHigh": 0, "adjLow": 0, "adjClose": 0,
+ },
+ ]
+
+ df = _rows_to_frame(rows)
+
+ # No bar carries a usable adjustment, so the series stays consistently raw
+ # (one basis) rather than being discarded or priced at zero.
+ assert df is not None
+ assert list(df["close"]) == [100.0, 101.0]
+
+
+def test_a_bar_that_cannot_be_emitted_does_not_set_the_adjusted_basis() -> None:
+ """A computable factor on an incomplete bar must not claim the basis.
+
+ Such a bar is dropped either way; what it must not do is drag its
+ unadjusted siblings down with it, which turned a usable raw series into no
+ series at all.
+ """
+ rows = [
+ {
+ "date": "2024-01-02T00:00:00.000Z",
+ "open": None, "high": 102, "low": 99, "close": 100, "volume": 1000,
+ "adjClose": 50,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 101, "high": 103, "low": 100, "close": 101, "volume": 1000,
+ },
+ ]
+
+ df = _rows_to_frame(rows)
+
+ assert df is not None
+ assert list(df["close"]) == [101.0]
+
+
+def test_a_non_finite_adjusted_price_is_not_served() -> None:
+ """An adjusted leg of infinity is not a price: the bar is not adjusted."""
+ rows = [
+ {
+ "date": "2024-01-02T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": float("inf"), "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,
+ },
+ ]
+
+ df = _rows_to_frame(rows)
+
+ assert df is not None
+ assert list(df["close"]) == [50.0]
+
+
+def test_a_dropped_row_is_reported_not_truncated_in_silence(caplog) -> None:
+ """A long-history symbol can lose old rows to the 0.01-100x factor window."""
+ rows = [
+ # A 1997-style row: only adjClose is present and its 600x cumulative
+ # factor (0.002/1.2) sits below the 0.01 floor, so it cannot join the
+ # adjusted series.
+ {
+ "date": "1997-05-15T00:00:00.000Z",
+ "open": 1.0, "high": 1.5, "low": 0.9, "close": 1.2, "volume": 1000,
+ "adjClose": 0.002,
+ },
+ {
+ "date": "2024-01-03T00:00:00.000Z",
+ "open": 100, "high": 101, "low": 99, "close": 100, "volume": 1000,
+ "adjOpen": 50, "adjHigh": 50.5, "adjLow": 49.5, "adjClose": 50,
+ },
+ ]
+
+ with caplog.at_level("WARNING", logger="backtest.loaders.tiingo_loader"):
+ df = _rows_to_frame(rows)
+
+ assert df is not None
+ assert list(df["close"]) == [50.0]
+ assert "dropped 1 of 2" in caplog.text
+
+
# ---------------------------------------------------------------------------
# fetch() behavior (HTTP mocked)
# ---------------------------------------------------------------------------