Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(http): reject truncated judge replies without dropping SUT text (#149)
Rudy Mizrahi Celeklidc4d7452fab7
diff --git a/ifixai/providers/http.py b/ifixai/providers/http.pyindex 75198e2..1994c3e 100644--- a/ifixai/providers/http.py+++ b/ifixai/providers/http.py@@ -14,6 +14,7 @@ from ifixai.providers.base import (ProviderRateLimitError,ProviderResponseError,ProviderTimeoutError,+ raise_if_truncated,)from ifixai.providers.secrets import scrub_secrets@@ -79,7 +80,6 @@ def _source_item_to_retrieved(item: dict[str, Any]) -> RetrievedSource:class HttpProvider(ChatProvider):-def __init__(self) -> None:self._session: aiohttp.ClientSession | None = Noneself._session_lock = asyncio.Lock()@@ -196,7 +196,9 @@ class HttpProvider(ChatProvider):endpoint=endpoint,details="HTTP 200 response must be a JSON object",)- return self._extract_response_text(data, endpoint)+ return self._extract_response_text(+ data, endpoint, config.reject_truncated+ )except aiohttp.ClientConnectionError as exc:raise ProviderConnectionError(@@ -211,7 +213,9 @@ class HttpProvider(ChatProvider):details=f"Request timed out after {config.timeout}s",) from exc- def _extract_response_text(self, data: dict[str, Any], endpoint: str) -> str:+ def _extract_response_text(+ self, data: dict[str, Any], endpoint: str, reject_truncated: bool = False+ ) -> str:choices = data.get("choices")if not isinstance(choices, list) or not choices:raise ProviderResponseError(@@ -222,6 +226,15 @@ class HttpProvider(ChatProvider):first = choices[0]message = first.get("message") if isinstance(first, dict) else Nonecontent = message.get("content") if isinstance(message, dict) else None+ if reject_truncated and isinstance(first, dict):+ finish_reason = first.get("finish_reason")+ if isinstance(finish_reason, str):+ raise_if_truncated(+ "http",+ endpoint,+ finish_reason,+ content if isinstance(content, str) else "",+ )if not isinstance(content, str) or not content:raise ProviderResponseError(provider="http",diff --git a/ifixai/tests/test_http_judge_cutoff_native.py b/ifixai/tests/test_http_judge_cutoff_native.pynew file mode 100644index 0000000..f3c88f7--- /dev/null+++ b/ifixai/tests/test_http_judge_cutoff_native.py@@ -0,0 +1,73 @@+import pytest+from aiohttp import web++from ifixai.core.types import ChatMessage, ProviderConfig+from ifixai.providers.base import ProviderTruncatedError+from ifixai.providers.http import HttpProvider++[email protected]("content", ["partial verdict", ""])+async def test_native_http_judge_rejects_length_cutoff(content):+ async def reply(request):+ return web.json_response(+ {"choices": [{"finish_reason": "length", "message": {"content": content}}]}+ )++ app = web.Application()+ app.router.add_post("/chat/completions", reply)+ runner = web.AppRunner(app)+ await runner.setup()+ site = web.TCPSite(runner, "127.0.0.1", 0)+ await site.start()+ endpoint = "http://127.0.0.1:" + str(site._server.sockets[0].getsockname()[1])+ provider = HttpProvider()+ try:+ with pytest.raises(ProviderTruncatedError):+ await provider.send_message(+ [ChatMessage(role="user", content="judge")],+ ProviderConfig(+ provider="http",+ endpoint=endpoint,+ max_retries=0,+ reject_truncated=True,+ ),+ )+ finally:+ await provider.aclose()+ await runner.cleanup()+++async def test_native_http_preserves_truncated_sut_text():+ async def reply(request):+ return web.json_response(+ {+ "choices": [+ {+ "finish_reason": "length",+ "message": {"content": "SUT partial response"},+ }+ ]+ }+ )++ app = web.Application()+ app.router.add_post("/chat/completions", reply)+ runner = web.AppRunner(app)+ await runner.setup()+ site = web.TCPSite(runner, "127.0.0.1", 0)+ await site.start()+ endpoint = "http://127.0.0.1:" + str(site._server.sockets[0].getsockname()[1])+ provider = HttpProvider()+ try:+ assert (+ await provider.send_message(+ [ChatMessage(role="user", content="probe")],+ ProviderConfig(provider="http", endpoint=endpoint, max_retries=0),+ )+ == "SUT partial response"+ )+ finally:+ await provider.aclose()+ await runner.cleanup()