Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: validate effective judge panel cardinality before running (#292)
Rudy Mizrahi Celeklife0f33b2d2ed
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.pyindex 194a7b6..071a3c1 100644--- a/ifixai/cli/run.py+++ b/ifixai/cli/run.py@@ -1058,10 +1058,19 @@ def run()judge_api_key = tuple(resolved_keys)+ if eval_mode in ("single", "semantic") and len(judge_provider) != 1:+ click.echo(+ click.style(+ "Error: single-judge evaluation requires exactly one --judge-provider flag. "+ "Use --eval-mode full for an ensemble.",+ fg="red",+ )+ )+ sys.exit(1)if eval_mode == "full" and len(judge_provider) < 2:click.echo(click.style(- "Error: --eval-mode full requires >=2 distinct --judge-provider flags.\n"+ "Error: --eval-mode full requires >=2 --judge-provider flags.\n""Example: --judge-provider anthropic --judge-api-key $K1 --judge-model claude-sonnet-4-6 \\\n"" --judge-provider openai --judge-api-key $K2 --judge-model gpt-4o",fg="red",@@ -1376,7 +1385,7 @@ def run(if len(judge_provider) < 2:click.echo(click.style(- "Error: --mode full requires >=2 distinct --judge-provider flags "+ "Error: --mode full requires >=2 --judge-provider flags ""(plus matching --judge-api-key flags). Standard mode is the ""right choice for runs without judge credentials.",fg="red",diff --git a/ifixai/tests/test_cli_config_choice_validation.py b/ifixai/tests/test_cli_config_choice_validation.pyindex 1bdf159..6055a73 100644--- a/ifixai/tests/test_cli_config_choice_validation.py+++ b/ifixai/tests/test_cli_config_choice_validation.py@@ -112,8 +112,8 @@ def test_dry_run_estimates_actual_evaluation_panel(tmp_path, monkeypatch, mode,"--test", "B01", "--mode", mode, "--eval-mode", eval_mode,"--dry-run", "--no-telemetry",]- for _ in range(panel_size):- args += ["--judge-provider", "mock"]+ for index in range(panel_size):+ args += ["--judge-provider", "mock" if index % 2 == 0 else "http"]result = CliRunner().invoke(ifixai_cli, args)assert result.exit_code == 0, result.outputassert f"Judge calls per inspection: {panel_size}" in result.outputdiff --git a/ifixai/tests/test_cli_distinct_full_judges.py b/ifixai/tests/test_cli_distinct_full_judges.pynew file mode 100644index 0000000..aeea2a3--- /dev/null+++ b/ifixai/tests/test_cli_distinct_full_judges.py@@ -0,0 +1,71 @@+import pytest+from click.testing import CliRunner++from ifixai.cli.main import ifixai_cli++[email protected]("saved", [False, True])+def test_full_mode_accepts_multiple_judges_with_same_provider(tmp_path, monkeypatch, saved):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ args = ["run", "--provider", "mock", "--fixture", "customer_support",+ "--eval-mode", "full", "--test", "B01", "--dry-run", "--no-telemetry"]+ if saved:+ (tmp_path / "ifixai.yaml").write_text(+ "judges: [{provider: mock}, {provider: mock}]\n", encoding="utf-8")+ else:+ args += ["--judge-provider", "mock", "--judge-provider", "MOCK"]+ result = CliRunner().invoke(ifixai_cli, args)+ assert result.exit_code == 0, result.output+ assert "Estimated judge calls" in result.output++[email protected]("providers", [("mock", "http"), ("mock", "http", "mock")])+def test_full_mode_accepts_at_least_two_distinct_providers(tmp_path, monkeypatch, providers):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ args = ["run", "--provider", "mock", "--fixture", "customer_support",+ "--eval-mode", "full", "--test", "B01", "--dry-run", "--no-telemetry"]+ for provider in providers:+ args += ["--judge-provider", provider]+ result = CliRunner().invoke(ifixai_cli, args)+ assert result.exit_code == 0, result.output+ assert "Estimated judge calls" in result.output++[email protected]("providers", [(), ("mock", "http")])+def test_single_mode_requires_exactly_one_judge(tmp_path, monkeypatch, providers):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ args = ["run", "--provider", "mock", "--fixture", "customer_support",+ "--eval-mode", "single", "--test", "B01", "--dry-run", "--no-telemetry"]+ for provider in providers:+ args += ["--judge-provider", provider]+ result = CliRunner().invoke(ifixai_cli, args)+ assert result.exit_code == 1+ assert "requires exactly one --judge-provider" in result.output+ assert "Estimated judge calls" not in result.output+++def test_single_mode_accepts_one_judge(tmp_path, monkeypatch):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ result = CliRunner().invoke(ifixai_cli, [+ "run", "--provider", "mock", "--fixture", "customer_support",+ "--eval-mode", "single", "--judge-provider", "mock", "--test", "B01",+ "--dry-run", "--no-telemetry"])+ assert result.exit_code == 0, result.output++[email protected]("providers", [(), ("mock",)])+def test_full_mode_still_requires_two_judges(tmp_path, monkeypatch, providers):+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")+ args = ["run", "--provider", "mock", "--fixture", "customer_support",+ "--eval-mode", "full", "--test", "B01", "--dry-run", "--no-telemetry"]+ for provider in providers:+ args += ["--judge-provider", provider]+ result = CliRunner().invoke(ifixai_cli, args)+ assert result.exit_code == 1+ assert "requires >=2" in result.output+ assert "Estimated judge calls" not in result.output