Skip to content

Framework for orchestrating role-playing, autonomous AI agents. By fostering collaborative intelligence, CrewAI empowers agents to work together seamlessly, tackling complex tasks.

Fix JSONLoader escaping non-ASCII text when rendering JSON documents (#8278)

Ashh.2282ab658b79
diff --git a/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py b/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
index 41c019f3e..8d8c98f69 100644
--- a/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
+++ b/lib/crewai-tools/src/crewai_tools/rag/loaders/json_loader.py
@@ -33,12 +33,18 @@ class JSONLoader(BaseLoader):
data = json.loads(content)
if isinstance(data, dict):
text = "\n".join(
- f"{k}: {json.dumps(v, indent=0)}" for k, v in data.items()
+ f"{k}: {json.dumps(v, indent=0, ensure_ascii=False)}"
+ for k, v in data.items()
)
elif isinstance(data, list):
- text = "\n".join(json.dumps(item, indent=0) for item in data)
+ text = "\n".join(
+ json.dumps(item, indent=0, ensure_ascii=False) for item in data
+ )
else:
- text = json.dumps(data, indent=0)
+ text = json.dumps(data, indent=0, ensure_ascii=False)
+
+ # Keep lone surrogates escaped without replacing readable Unicode.
+ text = text.encode("utf-8", errors="backslashreplace").decode("utf-8")
metadata = {
"format": "json",
diff --git a/lib/crewai-tools/tests/rag/test_json_loader_bom.py b/lib/crewai-tools/tests/rag/test_json_loader_bom.py
index 0802d9ce2..419a3bf3f 100644
--- a/lib/crewai-tools/tests/rag/test_json_loader_bom.py
+++ b/lib/crewai-tools/tests/rag/test_json_loader_bom.py
@@ -34,6 +34,6 @@ def test_json_loader_preserves_internal_bom_and_malformed_fallback(
"""Only a leading BOM is removed; content and malformed fallback stay intact."""
internal = _load(tmp_path, b'{"value":"a\xef\xbb\xbf b"}')
malformed = _load(tmp_path, b"\xef\xbb\xbf{invalid json}")
- assert "a\\ufeff b" in internal.content
+ assert "a\ufeff b" in internal.content
assert "parse_error" in malformed.metadata
assert malformed.content == "{invalid json}"
diff --git a/lib/crewai-tools/tests/rag/test_json_loader_unicode.py b/lib/crewai-tools/tests/rag/test_json_loader_unicode.py
new file mode 100644
index 000000000..5d65e165d
--- /dev/null
+++ b/lib/crewai-tools/tests/rag/test_json_loader_unicode.py
@@ -0,0 +1,56 @@
+import pytest
+
+from crewai_tools.rag.loaders.json_loader import JSONLoader
+from crewai_tools.rag.source_content import SourceContent
+
+
+def test_json_loader_keeps_non_ascii_text_readable() -> None:
+ """Object values keep readable Unicode characters."""
+ raw = '{"city": "Málaga", "note": "日本語"}'
+
+ result = JSONLoader().load(SourceContent(raw))
+
+ assert "Málaga" in result.content
+ assert "日本語" in result.content
+ assert "\\u" not in result.content
+
+
+def test_json_loader_list_keeps_non_ascii_text_readable() -> None:
+ """List items keep readable Unicode characters."""
+ result = JSONLoader().load(SourceContent('["café", "naïve"]'))
+
+ assert "café" in result.content
+ assert "\\u" not in result.content
+
+
+def test_json_loader_scalar_keeps_non_ascii_text_readable() -> None:
+ """Scalar strings keep readable Unicode characters."""
+ result = JSONLoader().load(SourceContent('"Málaga"'))
+
+ assert result.content == '"Málaga"'
+
+
[email protected]("surrogate", [r"\ud800", r"\udfff"])
[email protected]("shape", ["object", "list", "scalar"])
+def test_json_loader_lone_surrogate_stays_escaped(surrogate: str, shape: str) -> None:
+ """Lone surrogate escapes remain ingestible without losing their value."""
+ value = '"café ' + surrogate + '"'
+ raw = {
+ "object": '{"value": ' + value + '}',
+ "list": '[' + value + ']',
+ "scalar": value,
+ }[shape]
+
+ result = JSONLoader().load(SourceContent(raw))
+
+ assert surrogate in result.content
+ assert "café" in result.content
+ assert result.content.encode("utf-8")
+ assert result.doc_id
+
+
+def test_json_loader_paired_surrogate_keeps_emoji_readable() -> None:
+ """A valid surrogate pair is rendered as its Unicode character."""
+ result = JSONLoader().load(SourceContent(r'"\ud83d\ude00"'))
+
+ assert result.content == '"😀"'