Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): split JSONL records only on newline characters (#20029)

tri4tana2418baa03702
diff --git a/deepdoc/parser/json_parser.py b/deepdoc/parser/json_parser.py
index d82760efbd..560cfd48fe 100644
--- a/deepdoc/parser/json_parser.py
+++ b/deepdoc/parser/json_parser.py
@@ -150,7 +150,7 @@ class RAGFlowJsonParser:
return sections
def _parse_jsonl(self, content: str) -> list[str]:
- lines = content.strip().splitlines()
+ lines = content.strip().split("\n")
all_chunks = []
for line in lines:
if not line.strip():
@@ -164,7 +164,7 @@ class RAGFlowJsonParser:
return all_chunks
def is_jsonl_format(self, txt: str, sample_limit: int = 10, threshold: float = 0.8) -> bool:
- lines = [line.strip() for line in txt.strip().splitlines() if line.strip()]
+ lines = [line.strip() for line in txt.strip().split("\n") if line.strip()]
if not lines:
return False
diff --git a/test/unit_test/deepdoc/parser/test_json_parser.py b/test/unit_test/deepdoc/parser/test_json_parser.py
index 1c15b6154a..63fcc87aec 100644
--- a/test/unit_test/deepdoc/parser/test_json_parser.py
+++ b/test/unit_test/deepdoc/parser/test_json_parser.py
@@ -31,6 +31,8 @@ import os
import sys
import types
+import pytest
+
# Load json_parser by file path so we don't trigger deepdoc/parser/__init__.py
# (which pulls in heavy parsers). json_parser imports ``decode_text`` from
# rag.nlp; stub rag.nlp only while loading, then restore sys.modules so later
@@ -95,6 +97,18 @@ def test_top_level_scalars_do_not_crash():
assert parser._parse_json("false") == ["false"]
[email protected]("separator", ["\u0085", "\u2028", "\u2029"])
[email protected]("newline", ["\n", "\r\n"])
+def test_jsonl_preserves_unicode_separators_in_strings(separator, newline):
+ records = [{"text": f"first{separator}second"}, {"text": "next record"}]
+ content = newline.join(json.dumps(record, ensure_ascii=False) for record in records)
+ parser = RAGFlowJsonParser()
+
+ assert parser.is_jsonl_format(content)
+ assert [json.loads(chunk) for chunk in parser._parse_jsonl(content)] == records
+ assert [json.loads(chunk) for chunk in parser(content.encode("utf-8"))] == records
+
+
def test_top_level_null_yields_no_chunk():
# null carries no content; it should be dropped, not crash.
parser = RAGFlowJsonParser()