Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix: reject invalid inspection folder names during validation (#139)
Rudy Mizrahi Celeklifa556869d28a
diff --git a/ifixai/harness/validator.py b/ifixai/harness/validator.pyindex 54ab9c8..3498a6e 100644--- a/ifixai/harness/validator.py+++ b/ifixai/harness/validator.py@@ -35,6 +35,7 @@ _FOLDER_NAME_PATTERN = re.compile(r"^([bp](0[1-9]|[12][0-9]|3[0-2])|c(0[1-9]|1[0-6])|s(0[1-8])"r"|x(0[1-9]|1[0-1])|m(0[1-9]|1[0-2])|v(0[1-9]|10))_[a-z0-9_]+$")+_INSPECTION_LIKE_FOLDER_PATTERN = re.compile(r"^[bpcsmxv]\d", re.IGNORECASE)# P19/P27/P32/S02 ship domain-neutral corpora (honeypot restraint / privilege# creep / systemic-harm / stakeholder-conflict); their folders are already# admitted by _FOLDER_NAME_PATTERN above.@@ -104,11 +105,20 @@ class LayoutValidationError(Exception):def _iter_test_folders(tests_dir: Path) -> list[Path]:if not tests_dir.is_dir():raise LayoutValidationError(f"inspections directory missing: {tests_dir}")- return sorted(- p- for p in tests_dir.iterdir()- if p.is_dir() and _FOLDER_NAME_PATTERN.match(p.name)- )+ folders: list[Path] = []+ for path in sorted(tests_dir.iterdir()):+ if not path.is_dir():+ continue+ if _FOLDER_NAME_PATTERN.match(path.name):+ folders.append(path)+ elif _INSPECTION_LIKE_FOLDER_PATTERN.match(path.name):+ # A typo in a new inspection folder must fail validation instead+ # of silently excluding the entire inspection from the suite.+ raise LayoutValidationError(+ f"invalid inspection folder name {path.name!r}; expected "+ "a supported lowercase series/id followed by an underscore and slug"+ )+ return foldersdef _validate_folder(folder: Path) -> str:diff --git a/ifixai/tests/test_validator_folder_names.py b/ifixai/tests/test_validator_folder_names.pynew file mode 100644index 0000000..4ca2402--- /dev/null+++ b/ifixai/tests/test_validator_folder_names.py@@ -0,0 +1,23 @@+import pytest++from ifixai.harness.validator import LayoutValidationError, validate_layout+++ "folder_name",+ ["b33_new_inspection", "v11_new_inspection", "b01-wrong-separator", "B01_uppercase"],+)+def test_validator_rejects_inspection_like_folders_with_invalid_names(+ tmp_path, folder_name+):+ (tmp_path / folder_name).mkdir()++ with pytest.raises(LayoutValidationError, match="invalid inspection folder name"):+ validate_layout(tmp_path)+++def test_validator_ignores_python_cache_and_support_folders(tmp_path):+ (tmp_path / "__pycache__").mkdir()+ (tmp_path / "_shared").mkdir()++ assert validate_layout(tmp_path) == []