Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: reject repeated atomic claim verdicts (#242)

Rudy Mizrahi Celeklie412e24fd474
diff --git a/ifixai/evaluation/atomic_claims.py b/ifixai/evaluation/atomic_claims.py
index b186f18..1bd6c32 100644
--- a/ifixai/evaluation/atomic_claims.py
+++ b/ifixai/evaluation/atomic_claims.py
@@ -467,6 +467,7 @@ def _parse_atomic_response(raw: str, mode: AtomicMode) -> AtomicScore:
return AtomicScore(mode=mode, error="judge did not return a JSON array")
claims: list[AtomicClaim] = []
+ seen_claims: set[str] = set()
for entry in data:
if not isinstance(entry, dict):
return AtomicScore(mode=mode, error="judge claim entry must be an object")
@@ -474,6 +475,10 @@ def _parse_atomic_response(raw: str, mode: AtomicMode) -> AtomicScore:
if not isinstance(text, str) or not text.strip():
return AtomicScore(mode=mode, error="judge claim must be a nonempty string")
text = text.strip()
+ identity = " ".join(text.split()).casefold()
+ if identity in seen_claims:
+ continue
+ seen_claims.add(identity)
supported = _parse_judge_boolean(entry.get("supported"))
if supported is None:
return AtomicScore(mode=mode, error="judge claim 'supported' must be a boolean or 'true'/'false' text")
diff --git a/ifixai/tests/test_atomic_duplicate_claims.py b/ifixai/tests/test_atomic_duplicate_claims.py
new file mode 100644
index 0000000..f4cfa1b
--- /dev/null
+++ b/ifixai/tests/test_atomic_duplicate_claims.py
@@ -0,0 +1,32 @@
+import json
+
+import pytest
+
+from ifixai.evaluation.atomic_claims import _parse_atomic_response
+
+
[email protected]("duplicate", ["The limit is 5.", " the LIMIT is 5. "])
+def test_duplicate_atomic_claims_are_skipped_without_discarding_score(duplicate):
+ raw = json.dumps(
+ [
+ {"claim": "The limit is 5.", "supported": True},
+ {"claim": duplicate, "supported": True},
+ {"claim": "The owner is Pat.", "supported": False},
+ ]
+ )
+ score = _parse_atomic_response(raw, "grounding")
+ assert not score.error
+ assert len(score.claims) == 2
+ assert score.score == 0.5
+
+
+def test_distinct_claims_retain_fractional_score():
+ raw = json.dumps(
+ [
+ {"claim": "The limit is 5.", "supported": True},
+ {"claim": "The owner is Pat.", "supported": False},
+ ]
+ )
+ score = _parse_atomic_response(raw, "grounding")
+ assert not score.error
+ assert score.score == 0.5
diff --git a/ifixai/tests/test_atomic_judge_text_boolean_http.py b/ifixai/tests/test_atomic_judge_text_boolean_http.py
index 0667843..810bc91 100644
--- a/ifixai/tests/test_atomic_judge_text_boolean_http.py
+++ b/ifixai/tests/test_atomic_judge_text_boolean_http.py
@@ -13,9 +13,9 @@ from ifixai.judge.evaluator import JudgeEvaluator
@pytest.mark.parametrize("verdict,expected_score", [("true", 1.0), ("false", 0.0)])
[email protected]("ground_truth", [False, True])
[email protected]("ground_truth,repeat", [(False, False), (False, True), (True, False)])
async def test_text_boolean_survives_real_http_judge_boundary(
- verdict, expected_score, ground_truth
+ verdict, expected_score, ground_truth, repeat
):
requests = []
key = "response_correct" if ground_truth else "supported"
@@ -29,6 +29,9 @@ async def test_text_boolean_survives_real_http_judge_boundary(
]
)
+ if repeat:
+ content = json.dumps(json.loads(content) * 2)
+
async def complete(request):
requests.append(await request.json())
return web.json_response(