Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: exclude native SUT aliases from judge fallbacks (#312)

Rudy Mizrahi Celekli739e4c235c24
diff --git a/ifixai/core/runner.py b/ifixai/core/runner.py
index e6b4d3e..9d7453a 100644
--- a/ifixai/core/runner.py
+++ b/ifixai/core/runner.py
@@ -118,7 +118,9 @@ async def run_all(
judge = _build_judge_evaluator(judge_config)
- pipeline = _build_pipeline(pipeline_config, judge, sut_model=config.model)
+ pipeline = _build_pipeline(
+ pipeline_config, judge, sut_model=config.model, sut_provider=config.provider
+ )
to_run, reused_results = _split_cached(INSPECTION_REGISTRY, cached_results)
try:
@@ -197,7 +199,9 @@ async def run_strategic(
judge = _build_judge_evaluator(judge_config)
- pipeline = _build_pipeline(pipeline_config, judge, sut_model=config.model)
+ pipeline = _build_pipeline(
+ pipeline_config, judge, sut_model=config.model, sut_provider=config.provider
+ )
strategic_inspections = {
bid: inspection
@@ -298,7 +302,9 @@ async def run_selected(
judge = _build_judge_evaluator(judge_config)
- pipeline = _build_pipeline(pipeline_config, judge, sut_model=config.model)
+ pipeline = _build_pipeline(
+ pipeline_config, judge, sut_model=config.model, sut_provider=config.provider
+ )
selected_inspections = {
bid: inspection
@@ -391,7 +397,9 @@ async def run_single(
judge = _build_judge_evaluator(judge_config)
- pipeline = _build_pipeline(pipeline_config, judge, sut_model=config.model)
+ pipeline = _build_pipeline(
+ pipeline_config, judge, sut_model=config.model, sut_provider=config.provider
+ )
try:
return await inspection.execute(
@@ -697,20 +705,25 @@ def _build_pipeline(
pipeline_config: EvaluationPipelineConfig | None,
judge: JudgeEvaluator | EnsembleJudgeEvaluator | None,
sut_model: str | None = None,
+ sut_provider: str | None = None,
) -> EvaluationPipeline | None:
"""Wire the judge into an evaluation pipeline.
- `sut_model` is passed down so the judge's fallback chain can never
- substitute the system under test's own model for a failing judge.
+ SUT model and provider identify native and namespaced aliases so the judge's
+ fallback chain does not silently substitute the system under test's model.
"""
if pipeline_config is None:
return None
analytic_judge: AnalyticRubricJudge | EnsembleAnalyticRubricJudge | None = None
if isinstance(judge, EnsembleJudgeEvaluator):
- analytic_judge = EnsembleAnalyticRubricJudge(judge, sut_model=sut_model)
+ analytic_judge = EnsembleAnalyticRubricJudge(
+ judge, sut_model=sut_model, sut_provider=sut_provider
+ )
elif isinstance(judge, JudgeEvaluator):
- analytic_judge = AnalyticRubricJudge(judge, sut_model=sut_model)
+ analytic_judge = AnalyticRubricJudge(
+ judge, sut_model=sut_model, sut_provider=sut_provider
+ )
return EvaluationPipeline(
config=pipeline_config,
diff --git a/ifixai/evaluation/analytic_judge.py b/ifixai/evaluation/analytic_judge.py
index 7c1e00c..9205d08 100644
--- a/ifixai/evaluation/analytic_judge.py
+++ b/ifixai/evaluation/analytic_judge.py
@@ -967,8 +967,12 @@ class JudgeTransportExhaustedError(Exception):
class AnalyticRubricJudge:
-
- def __init__(self, judge: JudgeEvaluator, sut_model: str | None = None) -> None:
+ def __init__(
+ self,
+ judge: JudgeEvaluator,
+ sut_model: str | None = None,
+ sut_provider: str | None = None,
+ ) -> None:
self._judge = judge
policy = load_fallback_policy()
self._fallback_chain = policy.chain_for(judge.provider_name)
@@ -977,6 +981,7 @@ class AnalyticRubricJudge:
judge._provider_config.model,
policy=policy,
excluded_model=sut_model,
+ excluded_provider=sut_provider,
)
self._retired_models: set[str | None] = set()
# Model -> its failures in a row; a verdict resets it.
@@ -1291,11 +1296,15 @@ class EnsembleAnalyticRubricJudge:
"""Runs all ensemble judges in parallel and aggregates via mean score."""
def __init__(
- self, ensemble: EnsembleJudgeEvaluator, sut_model: str | None = None
+ self,
+ ensemble: EnsembleJudgeEvaluator,
+ sut_model: str | None = None,
+ sut_provider: str | None = None,
) -> None:
self._ensemble = ensemble
self._per_judge: list[AnalyticRubricJudge] = [
- AnalyticRubricJudge(e, sut_model=sut_model) for e in ensemble.evaluators
+ AnalyticRubricJudge(e, sut_model=sut_model, sut_provider=sut_provider)
+ for e in ensemble.evaluators
]
@property
diff --git a/ifixai/judge/fallbacks.py b/ifixai/judge/fallbacks.py
index c41a27e..2bced37 100644
--- a/ifixai/judge/fallbacks.py
+++ b/ifixai/judge/fallbacks.py
@@ -4,6 +4,8 @@ from pathlib import Path
from pydantic import BaseModel, ConfigDict, Field, ValidationError
+from ifixai.reporting.scorecard import grading_vendor
+
PACKAGED_FALLBACKS_PATH: Path = Path(__file__).parent / "judge_fallbacks.json"
PROJECT_FALLBACKS_FILENAME = "ifixai.judge-fallbacks.json"
FALLBACKS_PATH_ENV_VAR = "IFIXAI_JUDGE_FALLBACKS"
@@ -96,6 +98,7 @@ def resolve_judge_model_chain(
primary_model: str | None,
policy: JudgeFallbackPolicy | None = None,
excluded_model: str | None = None,
+ excluded_provider: str | None = None,
) -> list[str | None]:
"""Ordered judge models to try: the configured one, then its declared fallbacks.
@@ -103,17 +106,19 @@ def resolve_judge_model_chain(
file, so declaring a fallback never silently reorders the operator's choice.
``None`` means "whatever the provider defaults to" and is preserved as-is.
- `excluded_model` is the system under test. A fallback that happens to name the
- SUT's own model would quietly turn a cross-provider grade into self-judging
- while the run manifest still claims an independent judge — the exact thing
- ``build_manifest`` refuses to record — so it is dropped from the chain. The
- configured judge model is never dropped: an operator who points the judge at
- the SUT has chosen ``--eval-mode self``, which is declared, not silent.
+ ``excluded_provider`` and ``excluded_model`` identify the system under test.
+ Exclude every fallback from its vendor, using the scorecard's canonical
+ vendor resolution so dated IDs, related models and aggregator aliases cannot
+ silently turn an independent judge into a same-vendor judge. The configured
+ primary remains the operator's explicit choice, including self-grading.
"""
+ excluded_vendor = grading_vendor(excluded_provider, excluded_model) if excluded_provider else None
chain = (policy or load_fallback_policy()).chain_for(provider)
ordered: list[str | None] = [primary_model]
seen: set[str | None] = {primary_model, excluded_model}
for candidate in chain.models:
+ if excluded_vendor is not None and grading_vendor(provider, candidate.model) == excluded_vendor:
+ continue
if candidate.model not in seen:
ordered.append(candidate.model)
seen.add(candidate.model)
diff --git a/ifixai/tests/test_fallback_sut_identity.py b/ifixai/tests/test_fallback_sut_identity.py
new file mode 100644
index 0000000..ea63bbd
--- /dev/null
+++ b/ifixai/tests/test_fallback_sut_identity.py
@@ -0,0 +1,219 @@
+"""Fallbacks must not silently route a cross-provider judge to the SUT model."""
+
+import json
+
+import pytest
+from aiohttp import web
+
+from ifixai import api
+from ifixai.core.fixture_loader import load_fixture
+from ifixai.core.types import EvaluationPipelineConfig
+from ifixai.evaluation.analytic_judge import load_analytic_rubric
+from ifixai.judge.config import JudgeConfig
+
+
+ "sut_model,expected_fallback",
+ [
+ ("gpt-4o-mini", "anthropic/claude-haiku-4.5"),
+ ("gpt-4o", "anthropic/claude-haiku-4.5"),
+ ],
+)
+async def test_public_run_excludes_native_sut_alias_from_packaged_fallbacks(
+ sut_model, expected_fallback, monkeypatch
+):
+ # Ordinary native adapters and packaged policy; only network endpoints and
+ # owned model replies are local controls, not measured model quality.
+ monkeypatch.delenv("IFIXAI_JUDGE_FALLBACKS", raising=False)
+ rubric = await load_analytic_rubric("B19", "comply")
+ models = []
+
+ async def complete(request):
+ payload = await request.json()
+ model = payload["model"]
+ models.append(model)
+ if model == "google/gemini-2.5-flash":
+ return web.json_response(
+ {
+ "error": {
+ "message": "owned unavailable model",
+ "type": "invalid_request_error",
+ "code": "model_not_found",
+ }
+ },
+ status=400,
+ )
+ content = (
+ "owned response"
+ if model == sut_model
+ else json.dumps(
+ {
+ "dimensions": [
+ {"name": d.name, "passed": True, "reasoning": "owned verdict"}
+ for d in rubric.dimensions
+ ]
+ }
+ )
+ )
+ return web.json_response(
+ {
+ "id": "owned",
+ "object": "chat.completion",
+ "created": 1,
+ "model": model,
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": content},
+ "finish_reason": "stop",
+ }
+ ],
+ }
+ )
+
+ app = web.Application()
+ app.router.add_post("/v1/chat/completions", complete)
+ server = web.AppRunner(app)
+ await server.setup()
+ site = web.TCPSite(server, "127.0.0.1", 0)
+ await site.start()
+ port = site._server.sockets[0].getsockname()[1]
+ endpoint = f"http://127.0.0.1:{port}/v1"
+ try:
+ result = await api.run_single(
+ "B19",
+ "openai",
+ model=sut_model,
+ api_key="owned-loopback-key",
+ endpoint=endpoint,
+ fixture=load_fixture("default").model_copy(update={"governance": None}),
+ judge_config=JudgeConfig(
+ provider="openrouter",
+ model="google/gemini-2.5-flash",
+ api_key="owned-loopback-key",
+ endpoint=endpoint,
+ ),
+ pipeline_config=EvaluationPipelineConfig(judge_max_calls=0),
+ )
+ fallbacks = set(models) - {sut_model, "google/gemini-2.5-flash"}
+ assert fallbacks == {expected_fallback}
+ assert result.score == 1.0
+ assert len(result.evidence) >= 24
+ print(
+ json.dumps(
+ {
+ "sut": sut_model,
+ "fallbacks": sorted(fallbacks),
+ "request_count": len(models),
+ "score": result.score,
+ }
+ )
+ )
+ finally:
+ await server.cleanup()
+
+
+ "sut_provider,sut_model,expected",
+ [
+ ("openai", "gpt-4o-mini", ["primary", "anthropic/claude-haiku-4.5"]),
+ ("openrouter", "openai/gpt-4o-mini", ["primary", "anthropic/claude-haiku-4.5"]),
+ (
+ "http",
+ "gpt-4o-mini",
+ ["primary", "openai/gpt-4o-mini", "anthropic/claude-haiku-4.5"],
+ ),
+ (
+ None,
+ "gpt-4o-mini",
+ ["primary", "openai/gpt-4o-mini", "anthropic/claude-haiku-4.5"],
+ ),
+ (
+ "openai",
+ "gpt-4o",
+ ["primary", "anthropic/claude-haiku-4.5"],
+ ),
+ ("anthropic", "claude-haiku-4.5", ["primary", "openai/gpt-4o-mini"]),
+ ],
+)
+def test_fallback_excludes_same_sut_vendor(
+ sut_provider, sut_model, expected
+):
+ from ifixai.judge.fallbacks import JudgeFallbackPolicy, resolve_judge_model_chain
+
+ policy = JudgeFallbackPolicy.model_validate(
+ {
+ "providers": {
+ "openrouter": {
+ "models": [
+ {"model": "openai/gpt-4o-mini"},
+ {"model": "anthropic/claude-haiku-4.5"},
+ ]
+ }
+ }
+ }
+ )
+ assert (
+ resolve_judge_model_chain(
+ "openrouter",
+ "primary",
+ policy,
+ excluded_model=sut_model,
+ excluded_provider=sut_provider,
+ )
+ == expected
+ )
+
+
+def test_explicit_self_judge_primary_is_preserved():
+ from ifixai.judge.fallbacks import resolve_judge_model_chain
+
+ chain = resolve_judge_model_chain(
+ "openrouter",
+ "openai/gpt-4o-mini",
+ excluded_model="gpt-4o-mini",
+ excluded_provider="openai",
+ )
+ assert chain[0] == "openai/gpt-4o-mini"
+ assert chain.count("openai/gpt-4o-mini") == 1
+
+
+def test_ensemble_pipeline_propagates_sut_identity_to_each_judge():
+ from ifixai.core.runner import _build_judge_evaluator, _build_pipeline
+ from ifixai.judge.config import JudgeProviderSpec
+
+ judge = _build_judge_evaluator(
+ JudgeConfig(
+ providers=[
+ JudgeProviderSpec(
+ provider="openrouter", model="google/gemini-2.5-flash"
+ ),
+ JudgeProviderSpec(
+ provider="openrouter", model="anthropic/claude-haiku-4.5"
+ ),
+ ]
+ )
+ )
+ pipeline = _build_pipeline(
+ EvaluationPipelineConfig(),
+ judge,
+ sut_model="gpt-4o-mini",
+ sut_provider="openai",
+ )
+ assert pipeline is not None
+ for member in pipeline._judge._per_judge:
+ assert "openai/gpt-4o-mini" not in member._model_chain
+
+
[email protected]("sut_provider,sut_model,expected", [
+ ("anthropic", "claude-haiku-4-5", ["primary", "openai/gpt-4o-mini", "google/gemini-2.5-flash"]),
+ ("openai", "gpt-4o-mini-2024-07-18", ["primary", "anthropic/claude-haiku-4.5", "google/gemini-2.5-flash"]),
+ ("azure", "owned-deployment", ["primary", "anthropic/claude-haiku-4.5", "google/gemini-2.5-flash"]),
+ ("gemini", "gemini-2.5-flash-lite", ["primary", "openai/gpt-4o-mini", "anthropic/claude-haiku-4.5"]),
+])
+def test_vendor_exclusion_covers_aliases_dates_and_related_model_families(sut_provider, sut_model, expected):
+ from ifixai.judge.fallbacks import JudgeFallbackPolicy, resolve_judge_model_chain
+ policy = JudgeFallbackPolicy.model_validate({"providers": {"openrouter": {"models": [
+ {"model": "openai/gpt-4o-mini"}, {"model": "anthropic/claude-haiku-4.5"}, {"model": "google/gemini-2.5-flash"}]}}})
+ assert resolve_judge_model_chain("openrouter", "primary", policy, excluded_provider=sut_provider, excluded_model=sut_model) == expected