Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(gemini): keep native retries within adapter ownership (#303)

Rudy Mizrahi Celeklic3ff52deb4fb
diff --git a/ifixai/providers/gemini.py b/ifixai/providers/gemini.py
index cd63e2c..6a646c1 100644
--- a/ifixai/providers/gemini.py
+++ b/ifixai/providers/gemini.py
@@ -92,7 +92,12 @@ class GeminiProvider(ChatProvider):
for format_attempt in range(2):
try:
response = await asyncio.wait_for(
- model.generate_content_async(contents),
+ model.generate_content_async(
+ contents,
+ request_options=genai.types.RequestOptions(
+ retry=None, timeout=float(config.timeout)
+ ),
+ ),
timeout=float(config.timeout),
)
break
@@ -108,6 +113,7 @@ class GeminiProvider(ChatProvider):
)
setattr(model, "_async_client", native_client)
+
if not response.candidates:
raise ProviderResponseError(
provider="gemini",
@@ -167,6 +173,10 @@ class GeminiProvider(ChatProvider):
details=str(exc),
) from exc
except google_exceptions.ServiceUnavailable as exc:
+ if attempt < attempts - 1:
+ await asyncio.sleep(backoff)
+ backoff *= BACKOFF_MULTIPLIER
+ continue
raise ProviderConnectionError(
provider="gemini",
endpoint=endpoint,
diff --git a/ifixai/tests/test_gemini_retry_ownership_native.py b/ifixai/tests/test_gemini_retry_ownership_native.py
new file mode 100644
index 0000000..b0ac611
--- /dev/null
+++ b/ifixai/tests/test_gemini_retry_ownership_native.py
@@ -0,0 +1,159 @@
+"""Native Gemini SDK retries must not bypass adapter retry ownership."""
+
+import pytest
+import pytest_asyncio
+
+from ifixai.core.types import ChatMessage, ProviderConfig
+from ifixai.providers.base import (
+ ProviderConnectionError,
+ ProviderRateLimitError,
+ ProviderTimeoutError,
+)
+
+
+@pytest_asyncio.fixture
+async def retry_endpoint(monkeypatch):
+ grpc = pytest.importorskip("grpc")
+ glm = pytest.importorskip("google.ai.generativelanguage")
+ pytest.importorskip("google.generativeai")
+ from google.ai.generativelanguage_v1beta.services.generative_service.transports.grpc_asyncio import (
+ GenerativeServiceGrpcAsyncIOTransport,
+ )
+ from google.auth.credentials import AnonymousCredentials
+
+ from ifixai.providers import gemini
+
+ calls = []
+ state = {"error": "JSON mode is not enabled for this model", "code": grpc.StatusCode.INVALID_ARGUMENT, "reject": False, "finish": glm.Candidate.FinishReason.STOP, "reply": "owned reply", "rate_limits": 0, "reject_plain": False}
+
+ async def generate(request, context):
+ calls.append(request)
+ if state.get("delay"):
+ import asyncio
+ await asyncio.sleep(state["delay"])
+ if state.get("unavailable", 0):
+ if state["unavailable"] > 0:
+ state["unavailable"] -= 1
+ await context.abort(grpc.StatusCode.UNAVAILABLE, "owned unavailable")
+ if state["reject"] and (request.generation_config.response_mime_type or state["reject_plain"]):
+ await context.abort(state["code"], state["error"])
+ if state["rate_limits"]:
+ state["rate_limits"] -= 1
+ await context.abort(grpc.StatusCode.RESOURCE_EXHAUSTED, "owned throttle")
+ return glm.GenerateContentResponse(candidates=[glm.Candidate(content=glm.Content(parts=[glm.Part(text=state["reply"])]), finish_reason=state["finish"])])
+
+ server = grpc.aio.server()
+ server.add_generic_rpc_handlers((grpc.method_handlers_generic_handler("google.ai.generativelanguage.v1beta.GenerativeService", {"GenerateContent": grpc.unary_unary_rpc_method_handler(generate, request_deserializer=glm.GenerateContentRequest.deserialize, response_serializer=glm.GenerateContentResponse.serialize)}),))
+ port = server.add_insecure_port("127.0.0.1:0")
+ await server.start()
+ clients = []
+
+ def request_client(**kwargs):
+ # Substitute only SDK transport routing: the actual generated client,
+ # protobuf request serialization, RPC and Google exception mapping run.
+ transport = GenerativeServiceGrpcAsyncIOTransport(channel=grpc.aio.insecure_channel(f"127.0.0.1:{port}"), credentials=AnonymousCredentials())
+ client = glm.GenerativeServiceAsyncClient(transport=transport)
+ clients.append(client)
+ return client
+
+ monkeypatch.setattr(gemini, "GenerativeServiceAsyncClient", request_client)
+ try:
+ yield gemini, calls, state
+ finally:
+ for client in clients:
+ await client.transport.close()
+ await server.stop(0)
+ await server.wait_for_termination(timeout=2)
+
+
+
[email protected]("unavailable", [1, -1])
[email protected]("max_retries", [0, 2])
+async def test_adapter_503_retry_budget_and_recovery(retry_endpoint, unavailable, max_retries):
+ gemini, calls, state = retry_endpoint
+ state["unavailable"] = unavailable
+ config = ProviderConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=2, max_retries=max_retries)
+ if unavailable == 1 and max_retries:
+ assert await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config) == "owned reply"
+ assert len(calls) == 2
+ else:
+ with pytest.raises(ProviderConnectionError):
+ await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config)
+ assert len(calls) == max_retries + 1
+
+
[email protected]("max_retries", [0, 1])
+async def test_adapter_keeps_explicit_rate_limit_budget(retry_endpoint, max_retries):
+ gemini, calls, state = retry_endpoint
+ state["rate_limits"] = 1
+ config = ProviderConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=2, max_retries=max_retries)
+ if max_retries:
+ assert await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config) == "owned reply"
+ else:
+ with pytest.raises(ProviderRateLimitError):
+ await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config)
+ assert len(calls) == max_retries + 1
+
+
+async def test_success_is_one_native_request(retry_endpoint):
+ gemini, calls, _state = retry_endpoint
+ config = ProviderConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=2, max_retries=2)
+ assert await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config) == "owned reply"
+ assert len(calls) == 1
+
+
+async def test_public_fixture_run_records_native_outage_as_unscored(retry_endpoint):
+ import json
+
+ from ifixai.api import run_selected
+ from ifixai.core.types import EvaluationMode, EvaluationPipelineConfig, TestStatus
+ from ifixai.judge.config import JudgeConfig
+ from ifixai.reporting.scorecard import generate_json_report
+ _gemini, calls, state = retry_endpoint
+ state["unavailable"] = -1
+ result = await run_selected(
+ {"B19"}, provider="mock", fixture="software_engineering", model="owned-sut",
+ judge_config=JudgeConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=2),
+ pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.FULL, judge_max_calls=0),
+ )
+ report = json.loads(generate_json_report(result))
+ assert result.test_results[0].status == TestStatus.INCONCLUSIVE
+ assert report["test_results"][0]["score"] is None
+ assert calls
+ print(json.dumps({"native_offline_scorecard": {"test_id": "B19", "status": report["test_results"][0]["status"], "score": report["test_results"][0]["score"], "native_judge_requests": len(calls)}}))
+
+
+async def test_native_request_keeps_configured_timeout(retry_endpoint):
+ gemini, calls, state = retry_endpoint
+ state["delay"] = 2
+ config = ProviderConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=1, max_retries=2)
+ with pytest.raises(ProviderTimeoutError):
+ await gemini.GeminiProvider().send_message([ChatMessage(content="hello")], config)
+ assert len(calls) == 1
+
+
+async def test_public_fixture_recovers_after_native_503_blip(retry_endpoint):
+ import json
+
+ from ifixai.api import run_selected
+ from ifixai.core.types import EvaluationMode, EvaluationPipelineConfig, TestStatus
+ from ifixai.evaluation.analytic_judge import load_analytic_rubric
+ from ifixai.judge.config import JudgeConfig
+ from ifixai.reporting.scorecard import generate_json_report
+
+ _gemini, calls, state = retry_endpoint
+ state["unavailable"] = 2
+ rubric = await load_analytic_rubric("B19", "comply")
+ state["reply"] = json.dumps({"dimensions": [{"name": d.name, "passed": True, "reasoning": "owned offline control"} for d in rubric.dimensions]})
+ result = await run_selected(
+ {"B19"}, provider="mock", fixture="software_engineering", model="owned-sut",
+ judge_config=JudgeConfig(provider="gemini", model="gemini-owned", api_key="owned-key", timeout=2),
+ pipeline_config=EvaluationPipelineConfig(mode=EvaluationMode.FULL, judge_max_calls=0),
+ )
+ report = json.loads(generate_json_report(result))
+ assert result.test_results[0].status == TestStatus.PASS
+ assert report["test_results"][0]["score"] == 1.0
+ assert len(result.test_results[0].evidence) == 30
+ assert all(item.extraction_error is None for item in result.test_results[0].evidence)
+ assert len(calls) == 32
+ print(json.dumps({"native_offline_scorecard": {"test_id": "B19", "status": report["test_results"][0]["status"], "score": report["test_results"][0]["score"], "graded_items": 30, "native_judge_requests": len(calls)}}))