Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

fix(rag): decode local json files with utf-8 bom (#8023)

エイカク53c355e0b36a
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
index e14bf8584..41c019f3e 100644
--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
@@ -25,7 +25,7 @@ class JSONLoader(BaseLoader):
@staticmethod
def _load_from_file(path: str) -> str:
- with open(path, encoding="utf-8") as file:
+ with open(path, encoding="utf-8-sig") as file:
return file.read()
def _parse_json(self, content: str, source_ref: str) -> LoaderResult:
diff --git a/lib/crewai-tools/tests/rag/test_json_loader_bom.py b/lib/crewai-tools/tests/rag/test_json_loader_bom.py
new file mode 100644
index 000000000..0802d9ce2
--- /dev/null
+++ b/lib/crewai-tools/tests/rag/test_json_loader_bom.py
@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from crewai_tools.rag.base_loader import LoaderResult
+from crewai_tools.rag.loaders.json_loader import JSONLoader
+from crewai_tools.rag.source_content import SourceContent
+
+
+def _load(tmp_path: Path, content: bytes) -> LoaderResult:
+ """Load exact local bytes through the public loader entry point."""
+ path = tmp_path / "data.json"
+ path.write_bytes(content)
+ return JSONLoader().load(SourceContent(path))
+
+
+def test_json_loader_parses_utf8_bom_object(tmp_path: Path) -> None:
+ """A local UTF-8 BOM must not turn valid JSON into raw parse-error text."""
+ result = _load(tmp_path, b'\xef\xbb\xbf{"message":"hello"}')
+ assert result.metadata == {"format": "json", "type": "dict", "size": 1}
+ assert result.content == 'message: "hello"'
+ assert result.source == str(tmp_path / "data.json")
+ assert result.doc_id == JSONLoader.generate_doc_id(result.source, result.content)
+
+
+def test_json_loader_parses_utf8_bom_array(tmp_path: Path) -> None:
+ """BOM-prefixed arrays retain the same structured output as plain JSON."""
+ result = _load(tmp_path, b"\xef\xbb\xbf[1, 2]")
+ assert result.metadata == {"format": "json", "type": "list", "size": 2}
+ assert result.content == "1\n2"
+
+
+def test_json_loader_preserves_internal_bom_and_malformed_fallback(
+ tmp_path: Path,
+) -> None:
+ """Only a leading BOM is removed; content and malformed fallback stay intact."""
+ internal = _load(tmp_path, b'{"value":"a\xef\xbb\xbf b"}')
+ malformed = _load(tmp_path, b"\xef\xbb\xbf{invalid json}")
+ assert "a\\ufeff b" in internal.content
+ assert "parse_error" in malformed.metadata
+ assert malformed.content == "{invalid json}"