Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(cli): reject invalid report destinations before probing (#299)

Rudy Mizrahi Celekli19a2ecaa4f93
diff --git a/ifixai/cli/run.py b/ifixai/cli/run.py
index efef07e..653fb54 100644
--- a/ifixai/cli/run.py
+++ b/ifixai/cli/run.py
@@ -464,12 +464,14 @@ def _validate_min_score(ctx: click.Context, param: click.Parameter, value: float
@click.option(
"--output",
"-o",
+ type=click.Path(file_okay=False, readable=False),
default="./ifixai-results/",
show_default=True,
help="Directory to save reports.",
)
@click.option(
"--artifact-out",
+ type=click.Path(dir_okay=False, readable=False),
default=None,
help="Also write the self-contained interactive HTML scorecard to this path.",
)
diff --git a/ifixai/tests/test_cli_report_destinations.py b/ifixai/tests/test_cli_report_destinations.py
new file mode 100644
index 0000000..f1d2e6e
--- /dev/null
+++ b/ifixai/tests/test_cli_report_destinations.py
@@ -0,0 +1,120 @@
+"""Known-invalid report destinations must fail before native provider calls."""
+
+import json
+import os
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.main import ifixai_cli
+
+
+def owned_endpoint():
+ calls = []
+
+ class Handler(BaseHTTPRequestHandler):
+ def do_POST(self):
+ calls.append(self.path)
+ self.rfile.read(int(self.headers.get("Content-Length", "0")))
+ body = json.dumps({"choices": [{"message": {"content": "Owned fixture response"}}]}).encode()
+ self.send_response(200)
+ self.send_header("Content-Type", "application/json")
+ self.send_header("Content-Length", str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+
+ def do_GET(self):
+ self.send_response(404)
+ self.end_headers()
+
+ def log_message(self, *_args):
+ pass
+
+ server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+ thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01})
+ thread.start()
+ try:
+ yield f"http://127.0.0.1:{server.server_port}/v1", calls
+ finally:
+ server.shutdown()
+ server.server_close()
+ thread.join(timeout=2)
+
+
+def invoke(tmp_path, monkeypatch, endpoint, extra):
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setenv("IFIXAI_TELEMETRY", "0")
+ monkeypatch.setenv("DO_NOT_TRACK", "1")
+ fixture = Path(__file__).parents[1] / "fixtures/examples/customer_support.yaml"
+ return CliRunner().invoke(ifixai_cli, [
+ "run", "--provider", "http", "--endpoint", endpoint, "--auth-method", "none",
+ "--fixture", str(fixture), "--mode", "standard", "--eval-mode", "self",
+ "--test", "B03", "--timeout", "2", "--min-score", "0", "--no-telemetry",
+ "--no-parallel", "--reliability-out", str(tmp_path / "runs"), *extra,
+ ])
+
+
[email protected]("destination", ["output", "artifact", "config_output"])
+def test_invalid_destinations_fail_before_owned_http_calls(tmp_path, monkeypatch, owned_endpoint, destination):
+ endpoint, calls = owned_endpoint
+ target = tmp_path / "invalid"
+ if destination == "artifact":
+ target.mkdir()
+ extra = ["--output", str(tmp_path / "reports"), "--artifact-out", str(target)]
+ option = "--artifact-out"
+ else:
+ target.write_text("previous artifact", encoding="utf-8")
+ option = "--output"
+ if destination == "config_output":
+ (tmp_path / "ifixai.yaml").write_text(f"output: {target}\n", encoding="utf-8")
+ extra = []
+ else:
+ extra = ["--output", str(target)]
+ result = invoke(tmp_path, monkeypatch, endpoint, extra)
+ assert calls == [], (calls, result.output)
+ assert result.exit_code == 2, result.output
+ assert option in result.output
+ if target.is_file():
+ assert target.read_text(encoding="utf-8") == "previous artifact"
+
+
[email protected]("existing", [False, True])
+def test_valid_destinations_preserve_native_calls_reports_and_stdout(tmp_path, monkeypatch, owned_endpoint, existing):
+ endpoint, calls = owned_endpoint
+ output = tmp_path / "reports"
+ artifact = tmp_path / "artifact.html"
+ if existing:
+ output.mkdir()
+ artifact.write_text("previous artifact", encoding="utf-8")
+ result = invoke(tmp_path, monkeypatch, endpoint, ["--output", str(output), "--artifact-out", str(artifact)])
+ assert calls, result.output
+ assert "Reports saved:" in result.output
+ assert "Interactive artifact ->" in result.output
+ assert len(list(output.glob("*.json"))) == 1
+ assert "<html" in artifact.read_text(encoding="utf-8").lower()
+
+
+def test_write_only_destinations_keep_existing_permission_policy(tmp_path, monkeypatch, owned_endpoint):
+ endpoint, calls = owned_endpoint
+ output = tmp_path / "reports"
+ artifact = tmp_path / "artifact.html"
+ output.mkdir()
+ artifact.write_text("previous artifact", encoding="utf-8")
+ output.chmod(0o300)
+ artifact.chmod(0o200)
+ try:
+ if os.access(output, os.R_OK) or os.access(artifact, os.R_OK):
+ pytest.skip("Platform does not enforce write-only destination modes")
+ result = invoke(tmp_path, monkeypatch, endpoint, ["--output", str(output), "--artifact-out", str(artifact)])
+ finally:
+ output.chmod(0o700)
+ artifact.chmod(0o600)
+ assert calls, result.output
+ assert "Reports saved:" in result.output
+ assert "Interactive artifact ->" in result.output
+ assert len(list(output.glob("*.json"))) == 1
+ assert "<html" in artifact.read_text(encoding="utf-8").lower()