Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(cli): retain exports for distinct run identities (#188)

Rudy Mizrahi Celekli730c4adfada0
diff --git a/ifixai/cli/reports.py b/ifixai/cli/reports.py
index a7d8313..fc9f479 100644
--- a/ifixai/cli/reports.py
+++ b/ifixai/cli/reports.py
@@ -20,7 +20,8 @@ def _slugify(value: str) -> str:
def save_reports(
- result: TestRunResult, output_dir: str, report_format: str, run_nonce: str | None = None
+ result: TestRunResult, output_dir: str, report_format: str, run_nonce: str | None = None,
+ *, run_id: str | None = None,
) -> None:
out_path = Path(output_dir)
out_path.mkdir(parents=True, exist_ok=True)
@@ -28,9 +29,11 @@ def save_reports(
system_slug = _slugify(result.system_name)
fixture_slug = _slugify(result.fixture_name)
base_name = f"ifixai-{system_slug}-{fixture_slug}"
- # Suffix with the run nonce so each run's files are distinct.
- if run_nonce:
- base_name = f"{base_name}-{run_nonce[:8]}"
+ # The manifest identity distinguishes configuration changes even with a
+ # pinned nonce. Direct callers without a manifest retain the complete nonce.
+ suffix = run_id or run_nonce
+ if suffix:
+ base_name = f"{base_name}-{_slugify(suffix)}"
click.echo(click.style("Reports saved:", bold=True))
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index e175b23..987f02e 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -1926,7 +1926,7 @@ def run(
click.echo(click.style(f"Total execution time: {elapsed}", fg="cyan"))
click.echo()
- save_reports(result, output, report_format, effective_run_nonce)
+ save_reports(result, output, report_format, effective_run_nonce, run_id=manifest.run_id)
if artifact_out:
is_mock = provider.lower() == "mock"
# Mirror the console self-judge advisory in the portable artifact so a
diff --git a/ifixai/tests/test_report_run_identity.py b/ifixai/tests/test_report_run_identity.py
new file mode 100644
index 0000000..fc3dbe0
--- /dev/null
+++ b/ifixai/tests/test_report_run_identity.py
@@ -0,0 +1,42 @@
+"""Distinct CLI run manifests must retain distinct exported reports."""
+
+from pathlib import Path
+
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.main import ifixai_cli
+
+
[email protected]("change", ["model", "nonce-suffix", "resume"])
+def test_native_cli_keeps_reports_for_distinct_run_ids(tmp_path, monkeypatch, change):
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("HOME", str(tmp_path))
+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")
+ monkeypatch.setenv("DO_NOT_TRACK", "1")
+ fixture = Path(__file__).parents[1] / "fixtures/examples/customer_support.yaml"
+ common = ["run", "--provider", "mock", "--api-key", "owned-unused",
+ "--name", "Owned deployment", "--fixture", str(fixture),
+ "--test", "B01", "--eval-mode", "self", "--format", "json",
+ "--output", str(tmp_path / "reports"), "--no-telemetry",
+ "--no-parallel", "--reliability-out", str(tmp_path / "runs")]
+ original = CliRunner().invoke(ifixai_cli, [*common, "--model", "owned-first",
+ "--run-nonce", "0123456789abcdef"])
+ assert original.exit_code == 2, original.output
+ assert "Total execution time:" in original.output
+ first_files = list((tmp_path / "reports").glob("*.json"))
+ assert len(first_files) == 1
+ original_bytes = first_files[0].read_bytes()
+ model = "owned-second" if change == "model" else "owned-first"
+ nonce = "0123456789fedcba" if change == "nonce-suffix" else "0123456789abcdef"
+ resume = []
+ if change == "resume":
+ run_id = next((tmp_path / "runs").glob("*/manifest.json")).parent.name
+ resume = ["--resume", run_id]
+ later = CliRunner().invoke(ifixai_cli, [*common, "--model", model, "--run-nonce", nonce, *resume])
+ assert later.exit_code == 2, later.output
+ expected = 1 if change == "resume" else 2
+ assert len(list((tmp_path / "runs").glob("*/manifest.json"))) == expected
+ assert len(list((tmp_path / "reports").glob("*.json"))) == expected
+ if change != "resume":
+ assert first_files[0].read_bytes() == original_bytes