Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(judge): match fallback provider keys case-insensitively (#342)
Rudy Mizrahi Celekli354fc88aa6bc
diff --git a/ifixai/judge/fallbacks.py b/ifixai/judge/fallbacks.pyindex 2bced37..bc30a54 100644--- a/ifixai/judge/fallbacks.py+++ b/ifixai/judge/fallbacks.py@@ -2,7 +2,7 @@ import jsonimport osfrom pathlib import Path-from pydantic import BaseModel, ConfigDict, Field, ValidationError+from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validatorfrom ifixai.reporting.scorecard import grading_vendor@@ -39,6 +39,20 @@ class JudgeFallbackPolicy(BaseModel):version: int = 1providers: dict[str, JudgeFallbackChain] = Field(default_factory=dict)+ @field_validator("providers")+ @classmethod+ def _normalize_provider_keys(+ cls, value: dict[str, JudgeFallbackChain]+ ) -> dict[str, JudgeFallbackChain]:+ # chain_for looks providers up case-insensitively, so keys must match that.+ normalized: dict[str, JudgeFallbackChain] = {}+ for key, chain in value.items():+ name = key.strip().lower()+ if name in normalized:+ raise ValueError(f"duplicate provider key {key!r}")+ normalized[name] = chain+ return normalized+def chain_for(self, provider: str) -> JudgeFallbackChain:"""The chain declared for `provider`, or an empty one (no fallbacks)."""return self.providers.get(provider.strip().lower(), JudgeFallbackChain())diff --git a/ifixai/tests/test_judge_fallback_provider_keys.py b/ifixai/tests/test_judge_fallback_provider_keys.pynew file mode 100644index 0000000..6013dad--- /dev/null+++ b/ifixai/tests/test_judge_fallback_provider_keys.py@@ -0,0 +1,34 @@+import json++import pytest++from ifixai.judge.fallbacks import (+ JudgeFallbackConfigError,+ load_fallback_policy,+ resolve_judge_model_chain,+)+++def _write(tmp_path, providers):+ path = tmp_path / "fallbacks.json"+ path.write_text(json.dumps({"providers": providers}), encoding="utf-8")+ return path++[email protected]("key", ["OpenRouter", " openrouter ", "OPENROUTER"])+def test_mixed_case_provider_key_is_honored(tmp_path, key):+ policy = load_fallback_policy(_write(tmp_path, {key: {"models": [{"model": "backup-model"}]}}))+ assert [m.model for m in policy.chain_for("openrouter").models] == ["backup-model"]+ assert resolve_judge_model_chain("OpenRouter", "primary-model", policy) == [+ "primary-model",+ "backup-model",+ ]+++def test_provider_keys_differing_only_in_case_are_rejected(tmp_path):+ path = _write(tmp_path, {+ "openrouter": {"models": [{"model": "a"}]},+ "OpenRouter": {"models": [{"model": "b"}]},+ })+ with pytest.raises(JudgeFallbackConfigError, match="duplicate provider key"):+ load_fallback_policy(path)