OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
fix: add openai retry and check (#2247)
Lucas Oliveira74a904a3e16e
diff --git a/src/api/provider_validation.py b/src/api/provider_validation.pyindex d23753cb3..8864df992 100644--- a/src/api/provider_validation.py+++ b/src/api/provider_validation.py@@ -1,5 +1,6 @@"""Provider validation utilities for testing API keys and models during onboarding."""+import asyncioimport jsonimport re@@ -696,6 +697,80 @@ async def test_embedding(raise ValueError(f"Unknown provider: {provider}")+async def _http_request_with_retry(+ method: str,+ url: str,+ *,+ max_retries: int = 2,+ backoff_factor: float = 0.5,+ retryable_status_codes: tuple[int, ...] = (429, 500, 502, 503, 504),+ retry_timeout_on_post: bool = False,+ client: httpx.AsyncClient | None = None,+ **kwargs,+) -> httpx.Response:+ """Execute an HTTP request with retries for transient network/timeout/server errors.++ Retries GET requests on network timeouts and 429/5xx status codes.+ For POST requests, retries status codes (429/5xx) by default, and only retries+ network timeouts if `retry_timeout_on_post=True` is explicitly set (e.g. for test probes).+ """+ last_exc: Exception | None = None+ is_post = method.upper() == "POST"+ for attempt in range(max_retries + 1):+ try:+ if client is not None:+ if method.upper() == "GET":+ response = await client.get(url, **kwargs)+ elif is_post:+ response = await client.post(url, **kwargs)+ else:+ response = await client.request(method, url, **kwargs)+ else:+ async with httpx.AsyncClient() as ac:+ if method.upper() == "GET":+ response = await ac.get(url, **kwargs)+ elif is_post:+ response = await ac.post(url, **kwargs)+ else:+ response = await ac.request(method, url, **kwargs)++ if attempt < max_retries and response.status_code in retryable_status_codes:+ logger.warning(+ "Transient HTTP %d from %s (attempt %d/%d), retrying in %.2fs...",+ response.status_code,+ url,+ attempt + 1,+ max_retries + 1,+ backoff_factor * (2**attempt),+ )+ await asyncio.sleep(backoff_factor * (2**attempt))+ continue+ return response+ except (+ httpx.TimeoutException,+ httpx.NetworkError,+ httpx.RemoteProtocolError,+ httpx.ConnectError,+ ) as exc:+ last_exc = exc+ if attempt < max_retries and (not is_post or retry_timeout_on_post):+ logger.warning(+ "Transient %s requesting %s (attempt %d/%d), retrying in %.2fs...",+ type(exc).__name__,+ url,+ attempt + 1,+ max_retries + 1,+ backoff_factor * (2**attempt),+ )+ await asyncio.sleep(backoff_factor * (2**attempt))+ else:+ raise++ if last_exc:+ raise last_exc from None+ raise RuntimeError(f"HTTP request to {url} failed after retries")++# OpenAI validation functionsasync def _test_openai_lightweight_health(api_key: str) -> None:"""Test OpenAI API key validity with lightweight check.@@ -709,22 +784,22 @@ async def _test_openai_lightweight_health(api_key: str) -> None:"Content-Type": "application/json",}- async with httpx.AsyncClient() as client:- # Use /v1/models endpoint which validates the key without consuming credits- response = await client.get(- "https://api.openai.com/v1/models",- headers=headers,- timeout=10.0, # Short timeout for lightweight check- )+ # Use /v1/models endpoint which validates the key without consuming credits+ response = await _http_request_with_retry(+ "GET",+ "https://api.openai.com/v1/models",+ headers=headers,+ timeout=30.0,+ )- if response.status_code != 200:- error_details = _extract_error_details(response)- logger.error(- f"OpenAI lightweight health check failed: {response.status_code} - {error_details}"- )- raise Exception(error_details)+ if response.status_code != 200:+ error_details = _extract_error_details(response)+ logger.error(+ f"OpenAI lightweight health check failed: {response.status_code} - {error_details}"+ )+ raise Exception(error_details)- logger.info("OpenAI lightweight health check passed")+ logger.info("OpenAI lightweight health check passed")except httpx.TimeoutException:logger.error("OpenAI lightweight health check timed out")@@ -764,37 +839,38 @@ async def _test_openai_completion_with_tools(api_key: str, llm_model: str) -> No],}- async with httpx.AsyncClient() as client:- # Try with max_tokens first- payload = {**base_payload, "max_tokens": 50}- response = await client.post(+ # Try with max_tokens first+ payload = {**base_payload, "max_tokens": 50}+ response = await _http_request_with_retry(+ "POST",+ "https://api.openai.com/v1/chat/completions",+ headers=headers,+ json=payload,+ timeout=45.0,+ retry_timeout_on_post=True,+ )++ # If max_tokens doesn't work, try with max_completion_tokens+ if response.status_code != 200:+ logger.warning(+ "[API] max_tokens parameter failed, trying max_completion_tokens instead"+ )+ payload = {**base_payload, "max_completion_tokens": 50}+ response = await _http_request_with_retry(+ "POST","https://api.openai.com/v1/chat/completions",headers=headers,json=payload,- timeout=30.0,+ timeout=45.0,+ retry_timeout_on_post=True,)- # If max_tokens doesn't work, try with max_completion_tokens- if response.status_code != 200:- logger.warning(- "[API] max_tokens parameter failed, trying max_completion_tokens instead"- )- payload = {**base_payload, "max_completion_tokens": 50}- response = await client.post(- "https://api.openai.com/v1/chat/completions",- headers=headers,- json=payload,- timeout=30.0,- )-- if response.status_code != 200:- error_details = _extract_error_details(response)- logger.error(- f"OpenAI completion test failed: {response.status_code} - {error_details}"- )- raise Exception(error_details)+ if response.status_code != 200:+ error_details = _extract_error_details(response)+ logger.error(f"OpenAI completion test failed: {response.status_code} - {error_details}")+ raise Exception(error_details)- logger.info("OpenAI completion with tool calling test passed")+ logger.info("OpenAI completion with tool calling test passed")except httpx.TimeoutException:logger.error("OpenAI completion test timed out")@@ -817,26 +893,25 @@ async def _test_openai_embedding(api_key: str, embedding_model: str) -> None:"input": "test embedding",}- async with httpx.AsyncClient() as client:- response = await client.post(- "https://api.openai.com/v1/embeddings",- headers=headers,- json=payload,- timeout=30.0,- )+ response = await _http_request_with_retry(+ "POST",+ "https://api.openai.com/v1/embeddings",+ headers=headers,+ json=payload,+ timeout=45.0,+ retry_timeout_on_post=True,+ )- if response.status_code != 200:- error_details = _extract_error_details(response)- logger.error(- f"OpenAI embedding test failed: {response.status_code} - {error_details}"- )- raise Exception(error_details)+ if response.status_code != 200:+ error_details = _extract_error_details(response)+ logger.error(f"OpenAI embedding test failed: {response.status_code} - {error_details}")+ raise Exception(error_details)- data = response.json()- if not data.get("data") or len(data["data"]) == 0:- raise Exception("No embedding data returned")+ data = response.json()+ if not data.get("data") or len(data["data"]) == 0:+ raise Exception("No embedding data returned")- logger.info("OpenAI embedding test passed")+ logger.info("OpenAI embedding test passed")except httpx.TimeoutException:logger.error("OpenAI embedding test timed out")@@ -1199,21 +1274,21 @@ async def _test_anthropic_lightweight_health(api_key: str) -> None:"Content-Type": "application/json",}- async with httpx.AsyncClient() as client:- response = await client.get(- "https://api.anthropic.com/v1/models",- headers=headers,- timeout=10.0, # Short timeout for lightweight check- )+ response = await _http_request_with_retry(+ "GET",+ "https://api.anthropic.com/v1/models",+ headers=headers,+ timeout=30.0,+ )- if response.status_code != 200:- error_details = _extract_error_details(response)- logger.error(- f"Anthropic lightweight health check failed: {response.status_code} - {error_details}"- )- raise Exception(error_details)+ if response.status_code != 200:+ error_details = _extract_error_details(response)+ logger.error(+ f"Anthropic lightweight health check failed: {response.status_code} - {error_details}"+ )+ raise Exception(error_details)- logger.info("Anthropic lightweight health check passed")+ logger.info("Anthropic lightweight health check passed")except httpx.TimeoutException:logger.error("Anthropic lightweight health check timed out")diff --git a/src/services/models_service.py b/src/services/models_service.pyindex d78cdb345..d8864c130 100644--- a/src/services/models_service.py+++ b/src/services/models_service.py@@ -3,7 +3,11 @@ import reimport httpx-from api.provider_validation import _extract_error_details, format_provider_error_message+from api.provider_validation import (+ _extract_error_details,+ _http_request_with_retry,+ format_provider_error_message,+)from config.embedding_constants import OPENAI_DEFAULT_EMBEDDING_MODEL, OPENAI_EMBEDDING_MODEL_PREFIXfrom config.model_constants import (ANTHROPIC_DEFAULT_LANGUAGE_MODEL,@@ -289,12 +293,13 @@ class ModelsService:"Content-Type": "application/json",}- async with httpx.AsyncClient() as client:- # Lightweight validation: just check if API key is valid- # This doesn't consume credits, only validates the key- response = await client.get(- "https://api.openai.com/v1/models", headers=headers, timeout=10.0- )+ # Lightweight validation: check if API key is valid with retry logic+ response = await _http_request_with_retry(+ "GET",+ "https://api.openai.com/v1/models",+ headers=headers,+ timeout=30.0,+ )if response.status_code == 200:data = response.json()@@ -382,12 +387,12 @@ class ModelsService:}# Validate API key and return all models from Anthropic's chat-oriented list API- async with httpx.AsyncClient() as client:- response = await client.get(- "https://api.anthropic.com/v1/models",- headers=headers,- timeout=10.0,- )+ response = await _http_request_with_retry(+ "GET",+ "https://api.anthropic.com/v1/models",+ headers=headers,+ timeout=30.0,+ )if response.status_code == 200:data = response.json()diff --git a/tests/unit/test_openai_provider_check_retry.py b/tests/unit/test_openai_provider_check_retry.pynew file mode 100644index 000000000..ff411d668--- /dev/null+++ b/tests/unit/test_openai_provider_check_retry.py@@ -0,0 +1,127 @@+"""Unit tests for OpenAI provider check timeout & retry behavior."""++from __future__ import annotations++from unittest.mock import AsyncMock++import httpx+import pytest++from api.provider_validation import _http_request_with_retry, _test_openai_lightweight_health+++async def test_http_request_with_retry_recovers_from_transient_timeout():+ mock_client = AsyncMock(spec=httpx.AsyncClient)+ # Attempt 1 times out, Attempt 2 returns 200 OK+ ok_response = httpx.Response(200, json={"data": [{"id": "gpt-4o"}]})+ mock_client.get.side_effect = [+ httpx.TimeoutException("Read timed out"),+ ok_response,+ ]++ resp = await _http_request_with_retry(+ "GET",+ "https://api.openai.com/v1/models",+ max_retries=2,+ backoff_factor=0.01,+ client=mock_client,+ )++ assert resp.status_code == 200+ assert mock_client.get.call_count == 2+++async def test_http_request_with_retry_recovers_from_503_status():+ mock_client = AsyncMock(spec=httpx.AsyncClient)+ service_unavailable = httpx.Response(503, text="Service Unavailable")+ ok_response = httpx.Response(200, json={"data": []})+ mock_client.get.side_effect = [service_unavailable, ok_response]++ resp = await _http_request_with_retry(+ "GET",+ "https://api.openai.com/v1/models",+ max_retries=2,+ backoff_factor=0.01,+ client=mock_client,+ )++ assert resp.status_code == 200+ assert mock_client.get.call_count == 2+++async def test_http_request_with_retry_does_not_retry_auth_failure():+ mock_client = AsyncMock(spec=httpx.AsyncClient)+ unauthorized = httpx.Response(401, json={"error": {"message": "Invalid API key"}})+ mock_client.get.return_value = unauthorized++ resp = await _http_request_with_retry(+ "GET",+ "https://api.openai.com/v1/models",+ max_retries=2,+ backoff_factor=0.01,+ client=mock_client,+ )++ assert resp.status_code == 401+ assert mock_client.get.call_count == 1+++async def test_post_timeout_does_not_retry_without_flag():+ mock_client = AsyncMock(spec=httpx.AsyncClient)+ mock_client.post.side_effect = httpx.TimeoutException("POST timed out")++ with pytest.raises(httpx.TimeoutException):+ await _http_request_with_retry(+ "POST",+ "https://api.openai.com/v1/chat/completions",+ max_retries=2,+ backoff_factor=0.01,+ retry_timeout_on_post=False,+ client=mock_client,+ )++ # Must NOT retry POST on ambiguous network timeout without explicit flag+ assert mock_client.post.call_count == 1+++async def test_post_timeout_retries_with_explicit_flag():+ mock_client = AsyncMock(spec=httpx.AsyncClient)+ ok_response = httpx.Response(200, json={"choices": []})+ mock_client.post.side_effect = [+ httpx.TimeoutException("POST timed out"),+ ok_response,+ ]++ resp = await _http_request_with_retry(+ "POST",+ "https://api.openai.com/v1/chat/completions",+ max_retries=2,+ backoff_factor=0.01,+ retry_timeout_on_post=True,+ client=mock_client,+ )++ assert resp.status_code == 200+ assert mock_client.post.call_count == 2+++async def test_openai_lightweight_health_succeeds_with_retry(monkeypatch):+ ok_response = httpx.Response(200, json={"data": [{"id": "gpt-4o"}]})+ attempts = 0++ async def fake_retry(method, url, **kwargs):+ nonlocal attempts+ attempts += 1+ return ok_response++ monkeypatch.setattr("api.provider_validation._http_request_with_retry", fake_retry)++ # Should run cleanly without raising exception+ await _test_openai_lightweight_health("sk-test-key")+ assert attempts == 1