Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(judge): match fallback provider keys case-insensitively (#342)

Rudy Mizrahi Celekli354fc88aa6bc
diff --git a/ifixai/judge/fallbacks.py b/ifixai/judge/fallbacks.py
index 2bced37..bc30a54 100644
--- a/ifixai/judge/fallbacks.py
+++ b/ifixai/judge/fallbacks.py
@@ -2,7 +2,7 @@ import json
import os
from pathlib import Path
-from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator
from ifixai.reporting.scorecard import grading_vendor
@@ -39,6 +39,20 @@ class JudgeFallbackPolicy(BaseModel):
version: int = 1
providers: dict[str, JudgeFallbackChain] = Field(default_factory=dict)
+ @field_validator("providers")
+ @classmethod
+ def _normalize_provider_keys(
+ cls, value: dict[str, JudgeFallbackChain]
+ ) -> dict[str, JudgeFallbackChain]:
+ # chain_for looks providers up case-insensitively, so keys must match that.
+ normalized: dict[str, JudgeFallbackChain] = {}
+ for key, chain in value.items():
+ name = key.strip().lower()
+ if name in normalized:
+ raise ValueError(f"duplicate provider key {key!r}")
+ normalized[name] = chain
+ return normalized
+
def chain_for(self, provider: str) -> JudgeFallbackChain:
"""The chain declared for `provider`, or an empty one (no fallbacks)."""
return self.providers.get(provider.strip().lower(), JudgeFallbackChain())
diff --git a/ifixai/tests/test_judge_fallback_provider_keys.py b/ifixai/tests/test_judge_fallback_provider_keys.py
new file mode 100644
index 0000000..6013dad
--- /dev/null
+++ b/ifixai/tests/test_judge_fallback_provider_keys.py
@@ -0,0 +1,34 @@
+import json
+
+import pytest
+
+from ifixai.judge.fallbacks import (
+ JudgeFallbackConfigError,
+ load_fallback_policy,
+ resolve_judge_model_chain,
+)
+
+
+def _write(tmp_path, providers):
+ path = tmp_path / "fallbacks.json"
+ path.write_text(json.dumps({"providers": providers}), encoding="utf-8")
+ return path
+
+
[email protected]("key", ["OpenRouter", " openrouter ", "OPENROUTER"])
+def test_mixed_case_provider_key_is_honored(tmp_path, key):
+ policy = load_fallback_policy(_write(tmp_path, {key: {"models": [{"model": "backup-model"}]}}))
+ assert [m.model for m in policy.chain_for("openrouter").models] == ["backup-model"]
+ assert resolve_judge_model_chain("OpenRouter", "primary-model", policy) == [
+ "primary-model",
+ "backup-model",
+ ]
+
+
+def test_provider_keys_differing_only_in_case_are_rejected(tmp_path):
+ path = _write(tmp_path, {
+ "openrouter": {"models": [{"model": "a"}]},
+ "OpenRouter": {"models": [{"model": "b"}]},
+ })
+ with pytest.raises(JudgeFallbackConfigError, match="duplicate provider key"):
+ load_fallback_policy(path)