Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(loaders): page ccxt ohlcv at the exchange's own candle cap

Rudy Celeklic0e63a13e2ac
diff --git a/agent/backtest/loaders/ccxt_loader.py b/agent/backtest/loaders/ccxt_loader.py
index 1b410896a..b3764c837 100644
--- a/agent/backtest/loaders/ccxt_loader.py
+++ b/agent/backtest/loaders/ccxt_loader.py
@@ -47,6 +47,22 @@ _TIMEFRAME_DELTA = {
"1M": pd.Timedelta(days=31),
}
+
+def _page_limit(exchange) -> int:
+ """OHLCV page size to request: 1000, or less when the exchange caps lower.
+
+ Pagination stops on a page shorter than the request, so asking OKX (300)
+ or Kraken (720) for 1000 rows would end the history after one page.
+ """
+ features = getattr(exchange, "features", None)
+ spot = features.get("spot") if isinstance(features, dict) else None
+ spec = spot.get("fetchOHLCV") if isinstance(spot, dict) else None
+ cap = spec.get("limit") if isinstance(spec, dict) else None
+ if isinstance(cap, int) and not isinstance(cap, bool) and 0 < cap < 1000:
+ return cap
+ return 1000
+
+
# P12-b: ccxt had no request timeout and an unbounded paginated fetch with
# no retry budget, so a transient disconnect hung get_market_data for 10+
# minutes. Cap each HTTP call, bound transient retries, and enforce a hard
@@ -438,7 +454,7 @@ class DataLoader:
all_rows: list = []
cursor = since_ms
- limit = 1000
+ limit = _page_limit(exchange)
deadline = time.monotonic() + _CCXT_FETCH_BUDGET_S
label = f"ccxt fetch for {symbol}"
hit_page_cap = True
diff --git a/agent/tests/test_ccxt_loader_bounded.py b/agent/tests/test_ccxt_loader_bounded.py
index 137cfdfd4..8e05b4ee2 100644
--- a/agent/tests/test_ccxt_loader_bounded.py
+++ b/agent/tests/test_ccxt_loader_bounded.py
@@ -143,3 +143,33 @@ def test_valid_timeout_env_values_are_honored_on_reload(monkeypatch):
monkeypatch.delenv("CCXT_TIMEOUT_MS", raising=False)
monkeypatch.delenv("CCXT_FETCH_BUDGET_S", raising=False)
importlib.reload(cl)
+
+
+
+def test_exchange_page_cap_below_1000_keeps_paginating():
+ hour_ms = 3_600_000
+ since = int(pd.Timestamp("2024-01-01").timestamp() * 1000)
+ end = int(pd.Timestamp("2024-03-01").timestamp() * 1000)
+
+ class ThreeHundredPerPage:
+ """OKX-style exchange: at most 300 candles per call, as ccxt declares."""
+
+ features = {"spot": {"fetchOHLCV": {"limit": 300}}}
+ calls = 0
+
+ def fetch_ohlcv(self, symbol, timeframe, since=None, limit=None):
+ del symbol, timeframe
+ self.calls += 1
+ first = -(-since // hour_ms) * hour_ms
+ stop = min(first + min(limit, 300) * hour_ms, end)
+ return [
+ [ts, 100.0, 101.0, 99.0, 100.0, 10.0]
+ for ts in range(first, stop, hour_ms)
+ ]
+
+ ex = ThreeHundredPerPage()
+ df = DataLoader._fetch_one(ex, "BTC/USDT", "1h", since, end)
+ assert len(df) == 60 * 24
+ assert df.index[0] == pd.Timestamp("2024-01-01 00:00")
+ assert df.index[-1] == pd.Timestamp("2024-02-29 23:00")
+ assert ex.calls == 5