Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(judge): count actual provider grading attempts (#182)
Rudy Mizrahi Celekli008f86297b39
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.pyindex b0682d2..2cbaa6e 100644--- a/ifixai/evaluation/analytic_judge.py+++ b/ifixai/evaluation/analytic_judge.py@@ -1090,7 +1090,7 @@ class AnalyticRubricJudge:await asyncio.sleep(_BACKOFF_BASE * (2 ** (failures_so_far - 1)))try:raw_response = await asyncio.wait_for(- self._judge._provider.send_message(messages, judge_config),+ self._judge.send_message(messages, judge_config),timeout=_JUDGE_TIMEOUT,)except Exception as exc:diff --git a/ifixai/evaluation/atomic_claims.py b/ifixai/evaluation/atomic_claims.pyindex b6fe4d3..33f4d59 100644--- a/ifixai/evaluation/atomic_claims.py+++ b/ifixai/evaluation/atomic_claims.py@@ -261,7 +261,7 @@ async def _send_atomic(update={"max_tokens": _ATOMIC_MAX_TOKENS, "reject_truncated": True})return await asyncio.wait_for(- evaluator._provider.send_message(messages, capped_config),+ evaluator.send_message(messages, capped_config),timeout=_ATOMIC_JUDGE_TIMEOUT,)diff --git a/ifixai/judge/evaluator.py b/ifixai/judge/evaluator.pyindex cbe4d09..3bf36c1 100644--- a/ifixai/judge/evaluator.py+++ b/ifixai/judge/evaluator.py@@ -1,8 +1,24 @@-from ifixai.core.types import ClassifierPair, ProviderConfig+from ifixai.core.types import ChatMessage, ClassifierPair, ProviderConfigfrom ifixai.judge.config import JudgeConfig, JudgeProviderSpec+from ifixai.providers.base import ChatProviderfrom ifixai.providers.resolver import resolve_provider+class _ClassifierJudgeProvider(ChatProvider):+ """Keep borrowed classifier sends on the evaluator's accounting boundary."""++ def __init__(self, evaluator: "JudgeEvaluator") -> None:+ self._evaluator = evaluator++ async def send_message(+ self, messages: list[ChatMessage], config: ProviderConfig+ ) -> str:+ return await self._evaluator.send_message(messages, config)++ async def aclose(self) -> None:+ await self._evaluator.aclose()++class JudgeEvaluator:def __init__(self, config: JudgeConfig) -> None:@@ -21,8 +37,21 @@ class JudgeEvaluator:self._fallback_grades: dict[str, int] = {}self._transport_failures: dict[str, int] = {}+ async def send_message(+ self, messages: list[ChatMessage], config: ProviderConfig+ ) -> str:+ """Count provider sends, including failed and retried grading attempts.++ Internal HTTP/SDK retries belong to the adapter and are not observable+ here. The pipeline's separate budget still counts logical evaluations.+ """+ self._call_count += 1+ return await self._provider.send_message(messages, config)+def provider_pair(self) -> ClassifierPair:- return ClassifierPair(provider=self._provider, config=self._provider_config)+ return ClassifierPair(+ provider=_ClassifierJudgeProvider(self), config=self._provider_config+ )@propertydef provider_name(self) -> str:@@ -65,6 +94,11 @@ class JudgeEvaluator:return self._cap_reacheddef get_stats(self) -> dict[str, object]:+ """Provider-send counts; items_escalated remains the legacy count alias.++ These are not the pipeline's logical-grade budget or a count of SDK+ internal retries, which happen inside a single provider send.+ """return {"total_calls": self._call_count,"items_escalated": self._call_count,diff --git a/ifixai/tests/evaluation/test_judge_call_stats.py b/ifixai/tests/evaluation/test_judge_call_stats.pynew file mode 100644index 0000000..c46973f--- /dev/null+++ b/ifixai/tests/evaluation/test_judge_call_stats.py@@ -0,0 +1,91 @@+import json+from contextlib import asynccontextmanager++import pytest+from aiohttp import web++from ifixai.core.types import (+ AnalyticRubric,+ EvaluationCriteria,+ EvaluationPipelineConfig,+ RubricDimension,+ TestRunResult,+)+from ifixai.evaluation.analytic_judge import (+ AnalyticRubricJudge,+ EnsembleAnalyticRubricJudge,+)+from ifixai.evaluation.pipeline import EvaluationPipeline+from ifixai.judge.config import JudgeConfig, JudgeProviderSpec+from ifixai.judge.evaluator import EnsembleJudgeEvaluator, JudgeEvaluator+from ifixai.reporting.scorecard import generate_json_report++RUBRIC = AnalyticRubric(test_id='B06', outcome_type='comply', dimensions=[+ RubricDimension(name='accurate', description='Facts match.', weight=1.0),+])+++@asynccontextmanager+async def native_http_judge():+ requests = []+ async def reply(request):+ body = await request.json()+ requests.append(body)+ prompt = body['messages'][0]['content']+ if len(requests) == 1:+ content = 'not JSON'+ elif prompt.startswith('Classify assistant responses'):+ content = '{"class":"answer"}'+ elif 'atomic claim' in prompt:+ content = '[{"claim":"Receipt required","supported":true,"reason":"policy"}]'+ else:+ content = '{"dimensions":[{"name":"accurate","passed":true}]}'+ return web.json_response({'choices': [{'message': {'content': content}, 'finish_reason': 'stop'}]})+ app = web.Application()+ app.router.add_post('/v1/chat/completions', reply)+ runner = web.AppRunner(app)+ await runner.setup()+ site = web.TCPSite(runner, '127.0.0.1', 0)+ await site.start()+ endpoint = f'http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}/v1'+ try:+ yield endpoint, requests+ finally:+ await runner.cleanup()+++async def test_native_judge_stats_count_retries_atomic_and_classifier_calls():+ async with native_http_judge() as (endpoint, requests):+ evaluator = JudgeEvaluator(JudgeConfig(provider='http', endpoint=endpoint))+ pipeline = EvaluationPipeline(EvaluationPipelineConfig(), AnalyticRubricJudge(evaluator))+ try:+ assert (await pipeline.evaluate('Receipt required.', EvaluationCriteria(), RUBRIC)).passed+ assert evaluator.get_stats()['total_calls'] == len(requests) == 2+ atomic = await pipeline.evaluate_atomic('Receipt required.', 'Receipt required.', 'grounding')+ assert atomic is not None and not atomic.error+ assert evaluator.get_stats()['total_calls'] == len(requests) == 3+ assert await pipeline.classify('Receipt required.', 'Receipt policy?') is not None+ stats = evaluator.get_stats()+ assert stats['total_calls'] == stats['items_escalated'] == len(requests) == 4+ assert pipeline.judge_calls_used == 3, 'budget counts logical grades, stats count provider sends'+ scorecard = json.loads(generate_json_report(TestRunResult(judge_stats=stats)))+ assert scorecard['metadata']['judge_stats']['total_calls'] == 4+ finally:+ await evaluator.aclose()+++async def test_ensemble_stats_sum_each_judge_send():+ evaluator = EnsembleJudgeEvaluator(JudgeConfig(providers=[+ JudgeProviderSpec(provider='mock'), JudgeProviderSpec(provider='mock'),+ ]))+ judge = EnsembleAnalyticRubricJudge(evaluator)+ try:+ verdict = await judge.evaluate_with_rubric('Receipt required.', RUBRIC, 'receipt policy')+ assert len(verdict.per_judge) == 2+ stats = evaluator.get_stats()+ assert stats['total_calls'] == 2+ assert [s['total_calls'] for s in stats['per_judge_stats']] == [1, 1]+ finally:+ await evaluator.aclose()