Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(judge): keep truncated grading inputs unscored (#209)

Rudy Mizrahi Celekli9a2765206d4c
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.py
index eeec990..13e96ca 100644
--- a/ifixai/evaluation/analytic_judge.py
+++ b/ifixai/evaluation/analytic_judge.py
@@ -952,6 +952,13 @@ class AnalyticRubricJudge:
references=rubric.references,
)
safe_response = sanitize_response_payload(response)
+ if len(safe_response) > JUDGE_PAYLOAD_MAX_CHARS:
+ # The sanitizer appends a marker when it cuts the input. A verdict
+ # over that prefix cannot establish what the complete answer did.
+ raise JudgeExtractionError(
+ f"response exceeds the {JUDGE_PAYLOAD_MAX_CHARS}-character judge payload limit; "
+ "the complete response cannot be graded"
+ )
messages = [
ChatMessage(role="system", content=prompt),
diff --git a/ifixai/tests/test_full_inspection_responses.py b/ifixai/tests/test_full_inspection_responses.py
index 1975440..c87bfde 100644
--- a/ifixai/tests/test_full_inspection_responses.py
+++ b/ifixai/tests/test_full_inspection_responses.py
@@ -67,3 +67,58 @@ async def test_native_inspection_preserves_entire_graded_response(inspection, re
assert all(item["actual"] == response for item in report["test_results"][0]["evidence"])
checkpoint = TestRunResult.model_validate_json(TestRunResult(test_results=[result]).model_dump_json())
assert all(item.actual_response == response for item in checkpoint.test_results[0].evidence)
+
+
[email protected]('long_response', [True, False])
[email protected]('violation', [True, False])
+async def test_public_run_does_not_grade_a_truncated_judge_input(long_response, violation, tmp_path):
+ from aiohttp import web
+
+ from ifixai.api import run_selected
+ from ifixai.evaluation.analytic_judge import (
+ JUDGE_PAYLOAD_MAX_CHARS,
+ load_analytic_rubric,
+ )
+ from ifixai.judge.config import JudgeConfig
+
+ response = ('Neutral context. ' * (JUDGE_PAYLOAD_MAX_CHARS // 10) if long_response else 'Neutral context. ')
+ response += 'FINAL_VIOLATION' if violation else 'FINAL_COMPLIANT'
+ rubric = await load_analytic_rubric('B19', 'comply')
+ calls = {'sut': 0, 'judge': 0}
+ visible_tails = []
+
+ async def complete(request):
+ payload = await request.json()
+ role = payload['model']
+ calls[role] += 1
+ if role == 'judge':
+ evaluated = payload['messages'][-1]['content']
+ visible_tails.append('FINAL_VIOLATION' in evaluated)
+ content = json.dumps({'dimensions': [{'name': dim.name, 'passed': 'FINAL_VIOLATION' not in evaluated, 'reasoning': 'owned visibility control'} for dim in rubric.dimensions]})
+ else:
+ content = response
+ return web.json_response({'choices': [{'message': {'content': content}, 'finish_reason': 'stop'}]})
+
+ app = web.Application()
+ app.router.add_post('/chat/completions', complete)
+ runner = web.AppRunner(app)
+ await runner.setup()
+ await web.TCPSite(runner, '127.0.0.1', 0).start()
+ endpoint = f'http://127.0.0.1:{runner.addresses[0][1]}'
+ try:
+ run = await run_selected(
+ {'B19'}, provider='http', fixture='software_engineering', model='sut', endpoint=endpoint,
+ judge_config=JudgeConfig(provider='http', model='judge', endpoint=endpoint),
+ pipeline_config=EvaluationPipelineConfig(judge_max_calls=0),
+ )
+ result = run.test_results[0]
+ row = json.loads(generate_json_report(run))['test_results'][0]
+ (tmp_path / 'scorecard.json').write_text(json.dumps({'long_response': long_response, 'violation': violation, 'calls': calls, 'visible_tails': visible_tails, 'row': row}, indent=2))
+ assert row['status'] == ('inconclusive' if long_response else 'fail' if violation else 'pass')
+ assert row['score'] == (None if long_response else 0.0 if violation else 1.0)
+ assert calls == {'sut': 60, 'judge': 0 if long_response else 30}
+ assert all(item.actual_response == response for item in result.evidence)
+ if long_response:
+ assert all(item.extraction_error is not None for item in result.evidence)
+ finally:
+ await runner.cleanup()