Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(cli): estimate dry-run judge calls from the evaluation panel (#198)

Rudy Mizrahi Celekli2deef95def49
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index 46ce79c..14482a3 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -1155,12 +1155,12 @@ def run(
else:
estimated_tests = len(SPEC_BY_ID)
estimated_inspections = estimated_tests * 10
- if profile.lower() == "full":
- judge_calls_per_inspection = 3
- elif eval_mode != "deterministic":
- judge_calls_per_inspection = 1
- else:
+ if eval_mode == "deterministic":
judge_calls_per_inspection = 0
+ elif eval_mode == "full":
+ judge_calls_per_inspection = len(judge_provider)
+ else:
+ judge_calls_per_inspection = 1
estimated_judge_calls = estimated_inspections * judge_calls_per_inspection
click.echo()
click.echo(
diff --git a/ifixai/tests/test_cli_config_choice_validation.py b/ifixai/tests/test_cli_config_choice_validation.py
index bba5534..1bdf159 100644
--- a/ifixai/tests/test_cli_config_choice_validation.py
+++ b/ifixai/tests/test_cli_config_choice_validation.py
@@ -98,3 +98,24 @@ def test_saved_choices_use_cli_case_insensitive_contract(name, value, expected):
ctx = click.Context(run)
ctx.set_parameter_source(name, ParameterSource.DEFAULT)
assert _cfg_value(ctx, name, None, value) == expected
+
+
[email protected]("mode,eval_mode,panel_size", [
+ ("full", "full", 2), ("full", "full", 4),
+ ("standard", "full", 3), ("full", "deterministic", 0),
+ ("standard", "single", 1),
+])
+def test_dry_run_estimates_actual_evaluation_panel(tmp_path, monkeypatch, mode, eval_mode, panel_size):
+ monkeypatch.chdir(tmp_path)
+ args = [
+ "run", "--provider", "mock", "--fixture", "customer_support",
+ "--test", "B01", "--mode", mode, "--eval-mode", eval_mode,
+ "--dry-run", "--no-telemetry",
+ ]
+ for _ in range(panel_size):
+ args += ["--judge-provider", "mock"]
+ result = CliRunner().invoke(ifixai_cli, args)
+ assert result.exit_code == 0, result.output
+ assert f"Judge calls per inspection: {panel_size}" in result.output
+ assert f"Estimated judge calls: {10 * panel_size}" in result.output
+ assert not (tmp_path / "runs").exists()