Independent Auditing of AI Agents. Run by human or the agent itself, to answer the most crucial question in the AI Agent Economy. Is the agent doing what is supposed to do? With iFixAi you can have this answer in less than 120 seconds.
Python agent-evaluationaiai-alignmentai-evaluationai-governanceai-safetyclidiagnostic-tooleu-ai-acthallucination-detectioniso-42001llm-evaluationllm-securitynist-ai-rmfowasp-llmprompt-injectionpythonresponsible-airisk-assessmentrisk-management
fix(loaders): decode fixture and rule artifacts as UTF-8 (#165)
Rudy Mizrahi Celeklia5c9051aeaa6
diff --git a/ifixai/core/fixture_loader.py b/ifixai/core/fixture_loader.pyindex cadc4dc..7e678e0 100644--- a/ifixai/core/fixture_loader.py+++ b/ifixai/core/fixture_loader.py@@ -32,7 +32,7 @@ class FixtureValidationError(Exception):self.errors = errors or []def load_schema() -> dict[str, Any]:- with open(_SCHEMA_PATH) as fh:+ with open(_SCHEMA_PATH, encoding="utf-8") as fh:return json.load(fh) # type: ignore[no-any-return]def resolve_fixture_path(name_or_path: str | Path) -> Path:@@ -300,7 +300,7 @@ def _normalize_fixture_format(raw: dict[str, Any]) -> dict[str, Any]:return normalizeddef _read_fixture_file(path: Path) -> dict[str, Any]:- with open(path) as fh:+ with open(path, encoding="utf-8") as fh:text = fh.read()if path.suffix in (".yaml", ".yml"):diff --git a/ifixai/rules/loader.py b/ifixai/rules/loader.pyindex 75ea43c..c7e5a42 100644--- a/ifixai/rules/loader.py+++ b/ifixai/rules/loader.py@@ -54,7 +54,7 @@ class RuleLoader:def _read_yaml(self, path: Path) -> dict[str, Any]:try:- with open(path) as fh:+ with open(path, encoding="utf-8") as fh:data = yaml.safe_load(fh)if not isinstance(data, dict):raise RuleLoadError(f"Rule file {path} must be a mapping")@@ -114,7 +114,7 @@ def load_inspection_definition(test_id: str) -> ConversationPlan | None:yaml_path = _DEFAULT_LOADER._find_rule_file(test_id)if yaml_path is not None and _SCHEMA_PATH.exists():raw = _DEFAULT_LOADER._read_yaml(yaml_path)- with open(_SCHEMA_PATH) as fh:+ with open(_SCHEMA_PATH, encoding="utf-8") as fh:schema = json.load(fh)try:jsonschema.validate(instance=raw, schema=schema)diff --git a/ifixai/tests/core/test_utf8_artifact_readers.py b/ifixai/tests/core/test_utf8_artifact_readers.pynew file mode 100644index 0000000..09b4ec0--- /dev/null+++ b/ifixai/tests/core/test_utf8_artifact_readers.py@@ -0,0 +1,83 @@+import json+import os+import subprocess+import sys+from pathlib import Path++import pytest+import yaml++from ifixai.core.fixture_loader import load_fixture, resolve_fixture_path+++def _read_in_non_utf8_process(operation, argument):+ script = """+import json, locale, sys+from ifixai.core.fixture_loader import load_fixture, validate_fixture+from ifixai.rules.loader import RuleLoader, load_inspection_definition+encoding = locale.getpreferredencoding(False)+if encoding.lower().replace('-', '') == 'utf8':+ print(json.dumps({'utf8_locale': True}))+ raise SystemExit(0)+operation, argument = sys.argv[1:]+if operation == 'fixture':+ fixture = load_fixture(argument)+ assert not validate_fixture(argument)+ result = fixture.metadata.name+elif operation == 'plan':+ result = RuleLoader().load_rules(argument).model_dump(mode='json')+else:+ result = load_inspection_definition(argument).model_dump(mode='json')+print(json.dumps({'encoding': encoding, 'result': result}, ensure_ascii=True))+"""+ result = subprocess.run(+ [sys.executable, "-c", script, operation, argument],+ cwd=Path(__file__).parents[3],+ env={+ **os.environ,+ "LC_ALL": "C",+ "PYTHONUTF8": "0",+ "PYTHONCOERCECLOCALE": "0",+ "IFIXAI_TELEMETRY": "0",+ },+ capture_output=True,+ text=True,+ encoding="utf-8",+ timeout=15,+ )+ assert result.returncode == 0, result.stderr+ data = json.loads(result.stdout)+ if data.get("utf8_locale"):+ pytest.skip("this platform cannot select a non-UTF-8 locale")+ return data["result"]++[email protected]("name", ["healthcare", "openclaw_strict"])+def test_bundled_utf8_fixtures_load_outside_utf8_locale(name):+ expected = load_fixture(name).metadata.name+ assert (+ _read_in_non_utf8_process("fixture", str(resolve_fixture_path(name)))+ == expected+ )++[email protected]("suffix", [".yaml", ".json"])+def test_utf8_fixture_text_is_preserved_outside_utf8_locale(tmp_path, suffix):+ raw = yaml.safe_load(resolve_fixture_path("healthcare").read_text(encoding="utf-8"))+ raw["metadata"]["name"] = "Café — 医療"+ path = tmp_path / f"fixture{suffix}"+ text = (+ yaml.safe_dump(raw, allow_unicode=True)+ if suffix == ".yaml"+ else json.dumps(raw, ensure_ascii=False)+ )+ path.write_text(text, encoding="utf-8")+ assert _read_in_non_utf8_process("fixture", str(path)) == raw["metadata"]["name"]++[email protected]("operation", ["plan", "definition"])+def test_bundled_utf8_rule_text_is_preserved_outside_utf8_locale(operation):+ from ifixai.rules.loader import RuleLoader++ expected = RuleLoader().load_rules("B12").model_dump(mode="json")+ assert _read_in_non_utf8_process(operation, "B12") == expected