Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(rag): preserve newlines in recursive character fallback (#44179)
lorenzozane743be5c6382a
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.pyindex e6b0e80d4f1..8963c30c721 100644--- a/api/core/rag/splitter/fixed_text_splitter.py+++ b/api/core/rag/splitter/fixed_text_splitter.py@@ -76,7 +76,9 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)splits = [s + separator for s in splits[:-1]] + splits[-1:]else:splits = list(text)- splits = [s for s in splits if s != ""] if separator == "\n" else [s for s in splits if s not in {"", "\n"}]+ splits = (+ [s for s in splits if s != ""] if separator in {"", "\n"} else [s for s in splits if s not in {"", "\n"}]+ )_good_splits = []_good_splits_lengths = [] # cache the lengths of the splits_separator = "" if self._keep_separator else separatordiff --git a/api/core/rag/splitter/text_splitter.py b/api/core/rag/splitter/text_splitter.pyindex ca03c517b86..1c051677107 100644--- a/api/core/rag/splitter/text_splitter.py+++ b/api/core/rag/splitter/text_splitter.py@@ -27,7 +27,7 @@ def _split_text_with_regex(text: str, separator: str, keep_separator: bool) -> lsplits = re.split(separator, text)else:splits = list(text)- return [s for s in splits if (s not in {"", "\n"})]+ return [s for s in splits if s != "" and (separator == "" or s != "\n")]class TextSplitter(BaseDocumentTransformer, ABC):diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.pyindex eaf084ee25a..06c27b08b9c 100644--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py@@ -817,6 +817,19 @@ class TestEnhanceRecursiveCharacterTextSplitter:assert splitter.split_text("") == []mock_model.get_text_embedding_num_tokens.assert_not_called()+ def test_character_fallback_preserves_single_newlines(self):+ text = "\n".join(f"SKU{i:04d}" for i in range(20))+ splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(+ embedding_model_instance=None,+ separators=["\n\n", "。", ". ", " ", ""],+ chunk_size=50,+ chunk_overlap=0,+ )++ chunks = splitter.split_text(text)++ assert chunks[0] == "SKU0000\nSKU0001\nSKU0002\nSKU0003\nSKU0004\nSKU0005\nSK"+# ============================================================================# Test FixedRecursiveCharacterTextSplitter@@ -935,6 +948,18 @@ class TestFixedRecursiveCharacterTextSplitter:for chunk in result:assert len(chunk) <= 12 # chunk_size + some tolerance for overlap+ def test_character_fallback_preserves_single_newlines(self):+ text = "\n".join(f"SKU{i:04d}" for i in range(20))+ splitter = FixedRecursiveCharacterTextSplitter(+ separators=["\n\n", "。", ". ", " ", ""],+ chunk_size=50,+ chunk_overlap=0,+ )++ chunks = splitter.split_text(text)++ assert chunks[0] == "SKU0000\nSKU0001\nSKU0002\nSKU0003\nSKU0004\nSKU0005\nSK"+def test_overlap_in_character_splitting(self):"""Test that overlap is correctly applied in character-level splitting."""text = string.ascii_uppercase@@ -1040,6 +1065,7 @@ class TestFixedRecursiveCharacterTextSplitter:assert chunksassert all(chunk != "" for chunk in chunks)+ assert chunks == ["line1\n\nline2\n\nline3"]assert "line1" in "".join(chunks)assert "line2" in "".join(chunks)assert "line3" in "".join(chunks)