Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: require positive finite judge timeout overrides (#239)

Rudy Mizrahi Celekli394f94ec76ae
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.py
index fffacf1..7a6240c 100644
--- a/ifixai/evaluation/analytic_judge.py
+++ b/ifixai/evaluation/analytic_judge.py
@@ -2,6 +2,7 @@ import asyncio
import difflib
import json
import logging
+import math
import os
import re
import secrets
@@ -78,12 +79,16 @@ def _judge_timeout_from_env(default: float = 60.0) -> float:
if not raw:
return default
try:
- return float(raw)
+ value = float(raw)
except ValueError:
logger.warning(
"Ignoring invalid IFIXAI_JUDGE_TIMEOUT=%r; using %.0fs.", raw, default
)
return default
+ if math.isfinite(value) and value > 0:
+ return value
+ logger.warning("Ignoring invalid IFIXAI_JUDGE_TIMEOUT=%r; using %.0fs.", raw, default)
+ return default
_JUDGE_TIMEOUT: float = _judge_timeout_from_env()
diff --git a/ifixai/evaluation/atomic_claims.py b/ifixai/evaluation/atomic_claims.py
index ac69152..b186f18 100644
--- a/ifixai/evaluation/atomic_claims.py
+++ b/ifixai/evaluation/atomic_claims.py
@@ -1,6 +1,7 @@
import asyncio
import json
import logging
+import math
import os
from pathlib import Path
from typing import TYPE_CHECKING, Final, Literal
@@ -30,10 +31,14 @@ def _atomic_judge_timeout_from_env(default: float = 60.0) -> float:
if not raw:
return default
try:
- return float(raw)
+ value = float(raw)
except ValueError:
logger.warning("Ignoring invalid IFIXAI_JUDGE_TIMEOUT=%r; using %.0fs.", raw, default)
return default
+ if math.isfinite(value) and value > 0:
+ return value
+ logger.warning("Ignoring invalid IFIXAI_JUDGE_TIMEOUT=%r; using %.0fs.", raw, default)
+ return default
_ATOMIC_JUDGE_TIMEOUT: Final[float] = _atomic_judge_timeout_from_env()
diff --git a/ifixai/tests/test_judge_timeout_validation.py b/ifixai/tests/test_judge_timeout_validation.py
new file mode 100644
index 0000000..38b586a
--- /dev/null
+++ b/ifixai/tests/test_judge_timeout_validation.py
@@ -0,0 +1,21 @@
+import pytest
+
+from ifixai.evaluation.analytic_judge import _judge_timeout_from_env
+from ifixai.evaluation.atomic_claims import _atomic_judge_timeout_from_env
+
+
+ "parser", [_judge_timeout_from_env, _atomic_judge_timeout_from_env]
+)
[email protected]("value", ["nan", "inf", "-inf", "0", "-1"])
+def test_invalid_timeout_falls_back(parser, value, monkeypatch):
+ monkeypatch.setenv("IFIXAI_JUDGE_TIMEOUT", value)
+ assert parser(60.0) == 60.0
+
+
+ "parser", [_judge_timeout_from_env, _atomic_judge_timeout_from_env]
+)
+def test_positive_timeout_is_preserved(parser, monkeypatch):
+ monkeypatch.setenv("IFIXAI_JUDGE_TIMEOUT", "300.5")
+ assert parser() == 300.5