Skip to content

Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.

fix(compare): report unreadable scorecards without tracebacks (#324)

bbans816d0b64e67f
diff --git a/ifixai/cli/compare.py b/ifixai/cli/compare.py
index 2f36e1f..1ea1115 100644
--- a/ifixai/cli/compare.py
+++ b/ifixai/cli/compare.py
@@ -17,8 +17,8 @@ from ifixai.reporting.scorecard import render_partial_banner, render_resumed_ban
@click.command()
[email protected]("baseline", type=click.Path(exists=True))
[email protected]("enhanced", type=click.Path(exists=True))
[email protected]("baseline", type=click.Path(exists=True, dir_okay=False))
[email protected]("enhanced", type=click.Path(exists=True, dir_okay=False))
def compare(baseline: str, enhanced: str) -> None:
baseline_result = load_result_from_json(baseline)
@@ -103,8 +103,11 @@ def compare(baseline: str, enhanced: str) -> None:
def load_result_from_json(path: str) -> TestRunResult:
- with open(path, encoding="utf-8") as fh:
- raw = json.load(fh)
+ try:
+ with open(path, encoding="utf-8") as fh:
+ raw = json.load(fh)
+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+ raise click.ClickException(f"Could not read scorecard {path!r}: {exc}") from exc
metadata = raw.get("metadata", {})
overall = raw.get("overall", {})
diff --git a/ifixai/tests/test_compare_input_errors.py b/ifixai/tests/test_compare_input_errors.py
new file mode 100644
index 0000000..0c77f68
--- /dev/null
+++ b/ifixai/tests/test_compare_input_errors.py
@@ -0,0 +1,50 @@
+import pytest
+from click.testing import CliRunner
+
+from ifixai.cli.compare import compare
+
+
[email protected]("argument", [0, 1])
[email protected]("content", [b'{"metadata":', b'\xff'])
+def test_compare_reports_invalid_input(tmp_path, argument, content):
+ valid = tmp_path / "valid.json"
+ valid.write_text("{}", encoding="utf-8")
+ invalid = tmp_path / "invalid.json"
+ invalid.write_bytes(content)
+ arguments = [str(valid), str(valid)]
+ arguments[argument] = str(invalid)
+
+ result = CliRunner().invoke(compare, arguments)
+
+ assert result.exit_code == 1
+ assert "Error: Could not read scorecard" in result.output
+ assert repr(str(invalid)) in result.output
+ assert "Traceback" not in result.output
+
+
[email protected]("argument", [0, 1])
+def test_compare_rejects_directories(tmp_path, argument):
+ valid = tmp_path / "valid.json"
+ valid.write_text("{}", encoding="utf-8")
+ arguments = [str(valid), str(valid)]
+ arguments[argument] = str(tmp_path)
+
+ result = CliRunner().invoke(compare, arguments)
+
+ assert result.exit_code == 2
+ assert "is a directory" in result.output
+
+
+def test_compare_reports_read_failure(tmp_path, monkeypatch):
+ valid = tmp_path / "valid.json"
+ valid.write_text("{}", encoding="utf-8")
+
+ def deny_read(*args, **kwargs):
+ raise PermissionError("Permission denied")
+
+ monkeypatch.setattr("builtins.open", deny_read)
+ result = CliRunner().invoke(compare, [str(valid), str(valid)])
+
+ assert result.exit_code == 1
+ assert "Error: Could not read scorecard" in result.output
+ assert "Permission denied" in result.output