Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(fixtures): resolve names past unrelated directories (#222)
Rudy Mizrahi Celeklid632bd4c5eb6
diff --git a/ifixai/core/fixture_loader.py b/ifixai/core/fixture_loader.pyindex 2f1d3b6..6e9ce0e 100644--- a/ifixai/core/fixture_loader.py+++ b/ifixai/core/fixture_loader.py@@ -37,16 +37,19 @@ def load_schema() -> dict[str, Any]:def resolve_fixture_path(name_or_path: str | Path) -> Path:path = Path(name_or_path)- if path.exists():+ if path.exists() and not path.is_dir():return path- builtin_path = _FIXTURES_DIR / str(name_or_path) / "fixture.yaml"- if builtin_path.exists():- return builtin_path+ name = str(name_or_path)+ # Explicit paths must not silently select a different shipped fixture.+ if name == path.name and name not in {".", ".."} and "\\" not in name:+ builtin_path = _FIXTURES_DIR / name / "fixture.yaml"+ if builtin_path.is_file():+ return builtin_path- example_path = _FIXTURES_DIR / "examples" / f"{name_or_path}.yaml"- if example_path.exists():- return example_path+ example_path = _FIXTURES_DIR / "examples" / f"{name}.yaml"+ if example_path.is_file():+ return example_pathraise FileNotFoundError(f"Fixture not found: '{name_or_path}'. "diff --git a/ifixai/tests/core/test_fixture_path_resolution.py b/ifixai/tests/core/test_fixture_path_resolution.pynew file mode 100644index 0000000..e37df58--- /dev/null+++ b/ifixai/tests/core/test_fixture_path_resolution.py@@ -0,0 +1,99 @@+from pathlib import Path++import pytest++from ifixai import api+from ifixai.core.fixture_loader import (+ load_fixture,+ resolve_fixture_path,+ validate_fixture,+)+++async def test_public_api_default_fixture_ignores_unrelated_directory(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.chdir(tmp_path)+ (tmp_path / "default").mkdir()++ result = await api.run_selected({"B02"}, "mock")++ assert [result.test_id for result in result.test_results] == ["B02"]++[email protected]("name", ["default", "healthcare"])+def test_named_fixture_ignores_matching_working_directory(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str+) -> None:+ expected_path = resolve_fixture_path(name)+ expected = load_fixture(name)+ monkeypatch.chdir(tmp_path)+ (tmp_path / name).mkdir()++ assert resolve_fixture_path(name) == expected_path+ assert load_fixture(name) == expected+ assert validate_fixture(name) == []++[email protected]("filename", ["default", "custom.yaml"])+def test_local_fixture_file_keeps_precedence(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, filename: str+) -> None:+ contents = resolve_fixture_path("default").read_text(encoding="utf-8")+ monkeypatch.chdir(tmp_path)+ path = tmp_path / filename+ path.write_text(contents, encoding="utf-8")++ assert resolve_fixture_path(filename) == Path(filename)+ assert load_fixture(filename).metadata.name == load_fixture(path).metadata.name+++def test_nonfixture_directory_is_still_rejected(tmp_path: Path) -> None:+ errors = validate_fixture(tmp_path)++ assert len(errors) == 1+ with pytest.raises((FileNotFoundError, IsADirectoryError)):+ load_fixture(tmp_path)++[email protected]("explicit", ["./healthcare", "healthcare/", "./default"])+def test_explicit_directory_never_selects_builtin(tmp_path, monkeypatch, explicit):+ monkeypatch.chdir(tmp_path)+ path = tmp_path / explicit+ path.mkdir()+ (path / "fixture.yaml").write_text("broken: [", encoding="utf-8")+ with pytest.raises(FileNotFoundError):+ resolve_fixture_path(explicit)+ assert validate_fixture(explicit)++[email protected](not Path("/dev/stdin").exists(), reason="POSIX stdin device required")+def test_actual_cli_validates_piped_stdin(tmp_path):+ import os+ import subprocess+ import sys++ fixture = resolve_fixture_path("healthcare").read_text(encoding="utf-8")+ env = os.environ.copy()+ env.update(HOME=str(tmp_path), IFIXAI_TELEMETRY="0", DO_NOT_TRACK="1")+ env["PYTHONPATH"] = os.pathsep.join([str(Path(__file__).parents[3]), env.get("PYTHONPATH", "")])+ result = subprocess.run(+ [sys.executable, "-m", "ifixai.cli.main", "validate", "/dev/stdin"],+ input=fixture, cwd=tmp_path, env=env, capture_output=True, text=True,+ check=False, timeout=60,+ )+ assert result.returncode == 0, result.stdout + result.stderr+ assert "Valid" in result.stdout+++def test_parent_directory_is_not_a_builtin_name(tmp_path, monkeypatch):+ fixtures = tmp_path / "shipped"+ fixtures.mkdir()+ (tmp_path / "fixture.yaml").write_text("owned parent fixture", encoding="utf-8")+ work = tmp_path / "work"+ work.mkdir()+ monkeypatch.chdir(work)+ monkeypatch.setattr("ifixai.core.fixture_loader._FIXTURES_DIR", fixtures)+ with pytest.raises(FileNotFoundError):+ resolve_fixture_path("..")