RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): split JSONL records only on newline characters (#20029)
tri4tana2418baa03702
diff --git a/deepdoc/parser/json_parser.py b/deepdoc/parser/json_parser.pyindex d82760efbd..560cfd48fe 100644--- a/deepdoc/parser/json_parser.py+++ b/deepdoc/parser/json_parser.py@@ -150,7 +150,7 @@ class RAGFlowJsonParser:return sectionsdef _parse_jsonl(self, content: str) -> list[str]:- lines = content.strip().splitlines()+ lines = content.strip().split("\n")all_chunks = []for line in lines:if not line.strip():@@ -164,7 +164,7 @@ class RAGFlowJsonParser:return all_chunksdef is_jsonl_format(self, txt: str, sample_limit: int = 10, threshold: float = 0.8) -> bool:- lines = [line.strip() for line in txt.strip().splitlines() if line.strip()]+ lines = [line.strip() for line in txt.strip().split("\n") if line.strip()]if not lines:return Falsediff --git a/test/unit_test/deepdoc/parser/test_json_parser.py b/test/unit_test/deepdoc/parser/test_json_parser.pyindex 1c15b6154a..63fcc87aec 100644--- a/test/unit_test/deepdoc/parser/test_json_parser.py+++ b/test/unit_test/deepdoc/parser/test_json_parser.py@@ -31,6 +31,8 @@ import osimport sysimport types+import pytest+# Load json_parser by file path so we don't trigger deepdoc/parser/__init__.py# (which pulls in heavy parsers). json_parser imports ``decode_text`` from# rag.nlp; stub rag.nlp only while loading, then restore sys.modules so later@@ -95,6 +97,18 @@ def test_top_level_scalars_do_not_crash():assert parser._parse_json("false") == ["false"][email protected]("separator", ["\u0085", "\u2028", "\u2029"])[email protected]("newline", ["\n", "\r\n"])+def test_jsonl_preserves_unicode_separators_in_strings(separator, newline):+ records = [{"text": f"first{separator}second"}, {"text": "next record"}]+ content = newline.join(json.dumps(record, ensure_ascii=False) for record in records)+ parser = RAGFlowJsonParser()++ assert parser.is_jsonl_format(content)+ assert [json.loads(chunk) for chunk in parser._parse_jsonl(content)] == records+ assert [json.loads(chunk) for chunk in parser(content.encode("utf-8"))] == records++def test_top_level_null_yields_no_chunk():# null carries no content; it should be dropped, not crash.parser = RAGFlowJsonParser()