Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(runner): reject unknown API inspection selections before probes (#220)
Rudy Mizrahi Celekli3c5dd9fa464a
diff --git a/ifixai/core/runner.py b/ifixai/core/runner.pyindex 602b344..4067dc5 100644--- a/ifixai/core/runner.py+++ b/ifixai/core/runner.py@@ -270,6 +270,13 @@ async def run_selected(computed via _build_result, so it may be capped when a mandatory-minimuminspection is absent from the selection. Per-test results are exact."""+ unknown_ids = sorted(set(test_ids) - INSPECTION_REGISTRY.keys())+ if unknown_ids:+ raise ValueError(+ f"Unknown test ID(s): {', '.join(unknown_ids)}. "+ f"Available: {sorted(INSPECTION_REGISTRY.keys())}"+ )+if capabilities is None:capabilities = await detect_capabilities(provider, config)@@ -347,6 +354,12 @@ async def run_single(capabilities: ProviderCapabilities | None = None,) -> TestResult:+ inspection = INSPECTION_REGISTRY.get(test_id)+ if inspection is None:+ raise ValueError(+ f"Unknown test: {test_id}. Available: {sorted(INSPECTION_REGISTRY.keys())}"+ )+if capabilities is None:capabilities = await detect_capabilities(provider, config)@@ -354,11 +367,6 @@ async def run_single(pipeline = _build_pipeline(pipeline_config, judge, sut_model=config.model)- inspection = INSPECTION_REGISTRY.get(test_id)- if inspection is None:- raise ValueError(- f"Unknown test: {test_id}. Available: {sorted(INSPECTION_REGISTRY.keys())}"- )try:return await inspection.execute(provider,diff --git a/ifixai/tests/core/test_api_selection_validation.py b/ifixai/tests/core/test_api_selection_validation.pynew file mode 100644index 0000000..8b60c0e--- /dev/null+++ b/ifixai/tests/core/test_api_selection_validation.py@@ -0,0 +1,74 @@+"""Selection mistakes must not silently narrow a public API audit."""++import pytest++from ifixai import api+from ifixai.core.fixture_loader import load_fixture+from ifixai.providers.base import ChatProvider+++class SelectionProvider(ChatProvider):+ def __init__(self):+ self.capability_calls = 0+ self.message_calls = 0+ self.closed = False++ async def list_tools(self, config):+ self.capability_calls += 1+ return []++ async def send_message(self, messages, config):+ self.message_calls += 1+ return "ACK"++ async def aclose(self):+ self.closed = True+++def fixture_without_governance():+ # Exercise the provided agent's actual hooks instead of fixture composition.+ return load_fixture("default").model_copy(update={"governance": None})++[email protected]("test_ids", [{"B999"}, {"B02", "B999"}, ["B02", "B999"]])+async def test_unknown_selection_rejected_before_agent_probes(+ test_ids, fixture_without_governance+):+ provider = SelectionProvider()+ with pytest.raises(ValueError, match="B999"):+ await api.run_selected(test_ids, provider, fixture=fixture_without_governance)+ assert provider.capability_calls == 0+ assert provider.message_calls == 0+ assert provider.closed+++async def test_unknown_single_rejected_before_agent_probes(fixture_without_governance):+ provider = SelectionProvider()+ with pytest.raises(ValueError, match="Unknown test: B999"):+ await api.run_single("B999", provider, fixture=fixture_without_governance)+ assert provider.capability_calls == 0+ assert provider.message_calls == 0+ assert provider.closed++[email protected]("mode", ["single", "selected", "selected_list"])+async def test_valid_selection_keeps_native_inspection_and_teardown(+ mode, fixture_without_governance+):+ provider = SelectionProvider()+ if mode == "single":+ result = await api.run_single("B02", provider, fixture=fixture_without_governance)+ ids = [result.test_id]+ else:+ result = await api.run_selected(+ {"B02"} if mode == "selected" else ["B02"],+ provider, fixture=fixture_without_governance+ )+ ids = [inspection.test_id for inspection in result.test_results]+ assert ids == ["B02"]+ assert provider.capability_calls == 1+ assert provider.closed