"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(loaders): page ccxt ohlcv at the exchange's own candle cap
Rudy Celeklic0e63a13e2ac
diff --git a/agent/backtest/loaders/ccxt_loader.py b/agent/backtest/loaders/ccxt_loader.pyindex 1b410896a..b3764c837 100644--- a/agent/backtest/loaders/ccxt_loader.py+++ b/agent/backtest/loaders/ccxt_loader.py@@ -47,6 +47,22 @@ _TIMEFRAME_DELTA = {"1M": pd.Timedelta(days=31),}++def _page_limit(exchange) -> int:+ """OHLCV page size to request: 1000, or less when the exchange caps lower.++ Pagination stops on a page shorter than the request, so asking OKX (300)+ or Kraken (720) for 1000 rows would end the history after one page.+ """+ features = getattr(exchange, "features", None)+ spot = features.get("spot") if isinstance(features, dict) else None+ spec = spot.get("fetchOHLCV") if isinstance(spot, dict) else None+ cap = spec.get("limit") if isinstance(spec, dict) else None+ if isinstance(cap, int) and not isinstance(cap, bool) and 0 < cap < 1000:+ return cap+ return 1000++# P12-b: ccxt had no request timeout and an unbounded paginated fetch with# no retry budget, so a transient disconnect hung get_market_data for 10+# minutes. Cap each HTTP call, bound transient retries, and enforce a hard@@ -438,7 +454,7 @@ class DataLoader:all_rows: list = []cursor = since_ms- limit = 1000+ limit = _page_limit(exchange)deadline = time.monotonic() + _CCXT_FETCH_BUDGET_Slabel = f"ccxt fetch for {symbol}"hit_page_cap = Truediff --git a/agent/tests/test_ccxt_loader_bounded.py b/agent/tests/test_ccxt_loader_bounded.pyindex 137cfdfd4..8e05b4ee2 100644--- a/agent/tests/test_ccxt_loader_bounded.py+++ b/agent/tests/test_ccxt_loader_bounded.py@@ -143,3 +143,33 @@ def test_valid_timeout_env_values_are_honored_on_reload(monkeypatch):monkeypatch.delenv("CCXT_TIMEOUT_MS", raising=False)monkeypatch.delenv("CCXT_FETCH_BUDGET_S", raising=False)importlib.reload(cl)++++def test_exchange_page_cap_below_1000_keeps_paginating():+ hour_ms = 3_600_000+ since = int(pd.Timestamp("2024-01-01").timestamp() * 1000)+ end = int(pd.Timestamp("2024-03-01").timestamp() * 1000)++ class ThreeHundredPerPage:+ """OKX-style exchange: at most 300 candles per call, as ccxt declares."""++ features = {"spot": {"fetchOHLCV": {"limit": 300}}}+ calls = 0++ def fetch_ohlcv(self, symbol, timeframe, since=None, limit=None):+ del symbol, timeframe+ self.calls += 1+ first = -(-since // hour_ms) * hour_ms+ stop = min(first + min(limit, 300) * hour_ms, end)+ return [+ [ts, 100.0, 101.0, 99.0, 100.0, 10.0]+ for ts in range(first, stop, hour_ms)+ ]++ ex = ThreeHundredPerPage()+ df = DataLoader._fetch_one(ex, "BTC/USDT", "1h", since, end)+ assert len(df) == 60 * 24+ assert df.index[0] == pd.Timestamp("2024-01-01 00:00")+ assert df.index[-1] == pd.Timestamp("2024-02-29 23:00")+ assert ex.calls == 5