Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(cli): honor explicit overrides for configured judges (#158)

Rudy Mizrahi Celekli1528ea7da7d6
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index 987f02e..5834ad0 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -844,15 +844,19 @@ def run(
ctx.get_parameter_source("judge_provider") == ParameterSource.DEFAULT
):
judge_provider = tuple(j.provider for j in config_obj.judges)
- if any(j.model for j in config_obj.judges):
+ if (
+ ctx.get_parameter_source("judge_model") == ParameterSource.DEFAULT
+ and any(j.model for j in config_obj.judges)
+ ):
judge_model = tuple((j.model or "") for j in config_obj.judges)
- resolved_judge_keys: list[str] = []
- for j in config_obj.judges:
- if j.provider == provider and api_key:
- resolved_judge_keys.append(api_key)
- else:
- resolved_judge_keys.append(_lookup_env_api_key(j.provider) or "")
- judge_api_key = tuple(resolved_judge_keys)
+ if ctx.get_parameter_source("judge_api_key") == ParameterSource.DEFAULT:
+ resolved_judge_keys: list[str] = []
+ for j in config_obj.judges:
+ if j.provider == provider and api_key:
+ resolved_judge_keys.append(api_key)
+ else:
+ resolved_judge_keys.append(_lookup_env_api_key(j.provider) or "")
+ judge_api_key = tuple(resolved_judge_keys)
if not quiet:
click.echo(
click.style(f"Using config: {CONFIG_FILENAME}", fg="cyan"), err=True
diff --git a/ifixai/tests/test_cli_config_judge_overrides.py b/ifixai/tests/test_cli_config_judge_overrides.py
new file mode 100644
index 0000000..4698c8a
--- /dev/null
+++ b/ifixai/tests/test_cli_config_judge_overrides.py
@@ -0,0 +1,63 @@
+import importlib
+
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.config_file import JudgeSpec, RunConfig, write_config
+from ifixai.cli.main import ifixai_cli
+
+
[email protected]("option", ["key", "model", "both", "neither"])
+def test_explicit_judge_flags_override_config_defaults(tmp_path, monkeypatch, option):
+ module = importlib.import_module("ifixai.cli.run")
+ build = module._build_judge_config
+ configs = []
+
+ def record(**kwargs):
+ config = build(**kwargs)
+ configs.append(config)
+ return config
+
+ monkeypatch.setattr(module, "_build_judge_config", record)
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")
+ monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "config"))
+ write_config(
+ RunConfig(
+ provider="mock",
+ fixture="default",
+ mode="standard",
+ eval_mode="single",
+ judges=[JudgeSpec(provider="mock", model="configured-model")],
+ )
+ )
+ args = [
+ "run",
+ "--api-key",
+ "sut-synthetic-key",
+ "--test",
+ "B01",
+ "--no-telemetry",
+ "--no-parallel",
+ "--grounding",
+ "fixture",
+ "--output",
+ str(tmp_path / "reports"),
+ "--reliability-out",
+ str(tmp_path / "reliability"),
+ ]
+ if option in ("key", "both"):
+ args += ["--judge-api-key", "explicit-synthetic-key"]
+ if option in ("model", "both"):
+ args += ["--judge-model", "explicit-model"]
+ result = CliRunner().invoke(ifixai_cli, args)
+ assert configs, result.output
+ assert configs[0].api_key == (
+ "explicit-synthetic-key" if option in ("key", "both") else "sut-synthetic-key"
+ )
+ assert configs[0].model == (
+ "explicit-model" if option in ("model", "both") else "configured-model"
+ )
+ # The production CLI also completes its offline run and writes a scorecard.
+ assert result.exit_code == 2, result.output
+ assert list((tmp_path / "reports").glob("*.json"))