Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix: bind resumed verdicts to explicit judge configuration (#286)

Rudy Mizrahi Celeklie7e6487b0ccf
diff --git a/ifixai/tests/fixtures/manifest_before_judge_config_v1.json b/ifixai/tests/fixtures/manifest_before_judge_config_v1.json
new file mode 100644
index 0000000..45f9b1c
--- /dev/null
+++ b/ifixai/tests/fixtures/manifest_before_judge_config_v1.json
@@ -0,0 +1,34 @@
+{
+ "mode": "standard",
+ "model_under_test": {
+ "provider": "mock",
+ "model_id": "owned",
+ "version": "1",
+ "family": null
+ },
+ "judge_models": [],
+ "judge_temperature": 0.0,
+ "judge_identity": null,
+ "normalizer_version": "1",
+ "test_versions": {
+ "B07": "1"
+ },
+ "rubric_hashes": {},
+ "fixture_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "governance_fixture_digest": null,
+ "governance_source": null,
+ "seed": null,
+ "mode_filter": [],
+ "strict_structured": false,
+ "b12_seed": 20260422,
+ "b14_seed": 20260422,
+ "b28_seed": 20260422,
+ "b30_seed": 20260422,
+ "sut_temperature": 0.0,
+ "sut_seed": null,
+ "effective_sut_temperature": 0.0,
+ "effective_sut_seed": null,
+ "seed_supported_by_provider": true,
+ "run_id": "efef57047e38b9c0",
+ "timestamp": "2026-10-01T00:00:00Z"
+}
diff --git a/ifixai/tests/fixtures/manifest_before_judge_config_v3.json b/ifixai/tests/fixtures/manifest_before_judge_config_v3.json
new file mode 100644
index 0000000..332109a
--- /dev/null
+++ b/ifixai/tests/fixtures/manifest_before_judge_config_v3.json
@@ -0,0 +1,46 @@
+{
+ "schema_version": 3,
+ "run_nonce": "0123456789abcdef",
+ "mode": "standard",
+ "model_under_test": {
+ "provider": "mock",
+ "model_id": "owned",
+ "version": "1",
+ "family": null
+ },
+ "judge_models": [],
+ "judge_temperature": 0.0,
+ "judge_identity": null,
+ "normalizer_version": "1",
+ "test_versions": {
+ "B07": "1"
+ },
+ "rubric_hashes": {},
+ "fixture_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "governance_fixture_digest": null,
+ "governance_source": null,
+ "seed": null,
+ "mode_filter": [],
+ "strict_structured": false,
+ "b12_seed": 20260422,
+ "b14_seed": 20260422,
+ "b28_seed": 20260422,
+ "b30_seed": 20260422,
+ "b12_seed_pinned": false,
+ "b14_seed_pinned": false,
+ "b28_seed_pinned": false,
+ "b30_seed_pinned": false,
+ "sut_temperature": 0.0,
+ "sut_seed": null,
+ "effective_sut_temperature": 0.0,
+ "effective_sut_seed": null,
+ "seed_supported_by_provider": true,
+ "holdout_seed": null,
+ "holdout_ids": {},
+ "b29_seed": 20260422,
+ "b32_seed": 20260422,
+ "b29_seed_pinned": false,
+ "b32_seed_pinned": false,
+ "run_id": "2e33be67c78847d7",
+ "timestamp": "2026-10-01T00:00:00Z"
+}
diff --git a/ifixai/tests/fixtures/manifest_before_judge_config_v4.json b/ifixai/tests/fixtures/manifest_before_judge_config_v4.json
new file mode 100644
index 0000000..c093f06
--- /dev/null
+++ b/ifixai/tests/fixtures/manifest_before_judge_config_v4.json
@@ -0,0 +1,47 @@
+{
+ "schema_version": 4,
+ "run_nonce": "0123456789abcdef",
+ "sut_context_digest": null,
+ "mode": "standard",
+ "model_under_test": {
+ "provider": "mock",
+ "model_id": "owned",
+ "version": "1",
+ "family": null
+ },
+ "judge_models": [],
+ "judge_temperature": 0.0,
+ "judge_identity": null,
+ "normalizer_version": "1",
+ "test_versions": {
+ "B07": "1"
+ },
+ "rubric_hashes": {},
+ "fixture_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ "governance_fixture_digest": null,
+ "governance_source": null,
+ "seed": null,
+ "mode_filter": [],
+ "strict_structured": false,
+ "b12_seed": 20260422,
+ "b14_seed": 20260422,
+ "b28_seed": 20260422,
+ "b30_seed": 20260422,
+ "b12_seed_pinned": false,
+ "b14_seed_pinned": false,
+ "b28_seed_pinned": false,
+ "b30_seed_pinned": false,
+ "sut_temperature": 0.0,
+ "sut_seed": null,
+ "effective_sut_temperature": 0.0,
+ "effective_sut_seed": null,
+ "seed_supported_by_provider": true,
+ "holdout_seed": null,
+ "holdout_ids": {},
+ "b29_seed": 20260422,
+ "b32_seed": 20260422,
+ "b29_seed_pinned": false,
+ "b32_seed_pinned": false,
+ "run_id": "d97f2f5e1da4c534",
+ "timestamp": "2026-10-01T00:00:00Z"
+}
diff --git a/ifixai/tests/test_resume_explicit_judge_config.py b/ifixai/tests/test_resume_explicit_judge_config.py
new file mode 100644
index 0000000..83d4649
--- /dev/null
+++ b/ifixai/tests/test_resume_explicit_judge_config.py
@@ -0,0 +1,16 @@
+"""Historical grading manifests retain their original hash verification."""
+import json
+from pathlib import Path
+
+import pytest
+
+from ifixai.evaluation.manifest import load_manifest, verify_run_id
+
+
[email protected]("version", [1, 3, 4])
+def test_frozen_pre_grading_context_manifests_keep_original_identity(version):
+ path = Path(__file__).parent / "fixtures" / f"manifest_before_judge_config_v{version}.json"
+ original = json.loads(path.read_text())
+ loaded = load_manifest(path)
+ assert loaded.run_id == original["run_id"]
+ assert verify_run_id(loaded)
diff --git a/ifixai/tests/test_resume_judge_compatibility.py b/ifixai/tests/test_resume_judge_compatibility.py
new file mode 100644
index 0000000..354c04b
--- /dev/null
+++ b/ifixai/tests/test_resume_judge_compatibility.py
@@ -0,0 +1,64 @@
+import importlib
+import json
+
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.main import ifixai_cli
+
+
+def flags(tmp_path):
+ return ["run", "--provider", "mock", "--model", "owned-sut", "--test", "B07",
+ "--eval-mode", "single", "--judge-provider", "mock", "--judge-model", "owned-judge-a",
+ "--judge-budget", "10000", "--output", str(tmp_path / "reports"),
+ "--reliability-out", str(tmp_path / "runs"), "--no-telemetry", "--no-parallel"]
+
+
+def owned_environment(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("HOME", str(tmp_path))
+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")
+ monkeypatch.setenv("DO_NOT_TRACK", "1")
+ return importlib.import_module("ifixai.cli.run")
+
[email protected]("change", ["model", "budget", "legacy"])
+def test_native_resume_allows_dead_judge_swap_and_legacy_identity(tmp_path, monkeypatch, change):
+ run = owned_environment(tmp_path, monkeypatch)
+ original = run.build_manifest
+ if change == "legacy":
+ from ifixai.evaluation.manifest import compute_run_id
+ def legacy(**kwargs):
+ kwargs["judge_models"] = []
+ kwargs["judge_identity"] = None
+ manifest = original(**kwargs)
+ payload = manifest.model_dump(mode="json")
+ payload.pop("evaluation_mode", None)
+ payload.pop("judge_budget", None)
+ payload["schema_version"] = 4
+ manifest.schema_version = 4
+ manifest.run_id = compute_run_id(payload)
+ return manifest
+ monkeypatch.setattr(run, "build_manifest", legacy)
+ first = CliRunner().invoke(ifixai_cli, flags(tmp_path))
+ paths = list((tmp_path / "runs").glob("*/manifest.json"))
+ assert len(paths) == 1, first.output
+ manifest = json.loads(paths[0].read_text())
+ monkeypatch.setattr(run, "build_manifest", original)
+ args = flags(tmp_path)
+ if change == "model":
+ args[args.index("owned-judge-a")] = "owned-judge-b"
+ elif change == "budget":
+ args[args.index("10000")] = "1"
+ resumed = CliRunner().invoke(ifixai_cli, [*args, "--resume", manifest["run_id"]])
+ assert "[reused] B07" in resumed.output, resumed.output
+ assert resumed.exit_code == first.exit_code
+
+
+def test_native_full_ensemble_with_same_sut_model_does_not_crash(tmp_path, monkeypatch):
+ owned_environment(tmp_path, monkeypatch)
+ args = flags(tmp_path)
+ args[args.index("single")] = "full"
+ args[args.index("owned-judge-a")] = "owned-sut"
+ result = CliRunner().invoke(ifixai_cli, [*args, "--judge-provider", "mock", "--judge-model", "owned-judge-b"])
+ assert result.exit_code == 2, (result.output, result.exception)
+ assert len(list((tmp_path / "runs").glob("*/manifest.json"))) == 1