Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag): preserve newlines in recursive character fallback (#44179)

lorenzozane743be5c6382a
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py
index e6b0e80d4f1..8963c30c721 100644
--- a/api/core/rag/splitter/fixed_text_splitter.py
+++ b/api/core/rag/splitter/fixed_text_splitter.py
@@ -76,7 +76,9 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
splits = [s + separator for s in splits[:-1]] + splits[-1:]
else:
splits = list(text)
- splits = [s for s in splits if s != ""] if separator == "\n" else [s for s in splits if s not in {"", "\n"}]
+ splits = (
+ [s for s in splits if s != ""] if separator in {"", "\n"} else [s for s in splits if s not in {"", "\n"}]
+ )
_good_splits = []
_good_splits_lengths = [] # cache the lengths of the splits
_separator = "" if self._keep_separator else separator
diff --git a/api/core/rag/splitter/text_splitter.py b/api/core/rag/splitter/text_splitter.py
index ca03c517b86..1c051677107 100644
--- a/api/core/rag/splitter/text_splitter.py
+++ b/api/core/rag/splitter/text_splitter.py
@@ -27,7 +27,7 @@ def _split_text_with_regex(text: str, separator: str, keep_separator: bool) -> l
splits = re.split(separator, text)
else:
splits = list(text)
- return [s for s in splits if (s not in {"", "\n"})]
+ return [s for s in splits if s != "" and (separator == "" or s != "\n")]
class TextSplitter(BaseDocumentTransformer, ABC):
diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
index eaf084ee25a..06c27b08b9c 100644
--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
@@ -817,6 +817,19 @@ class TestEnhanceRecursiveCharacterTextSplitter:
assert splitter.split_text("") == []
mock_model.get_text_embedding_num_tokens.assert_not_called()
+ def test_character_fallback_preserves_single_newlines(self):
+ text = "\n".join(f"SKU{i:04d}" for i in range(20))
+ splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(
+ embedding_model_instance=None,
+ separators=["\n\n", "。", ". ", " ", ""],
+ chunk_size=50,
+ chunk_overlap=0,
+ )
+
+ chunks = splitter.split_text(text)
+
+ assert chunks[0] == "SKU0000\nSKU0001\nSKU0002\nSKU0003\nSKU0004\nSKU0005\nSK"
+
# ============================================================================
# Test FixedRecursiveCharacterTextSplitter
@@ -935,6 +948,18 @@ class TestFixedRecursiveCharacterTextSplitter:
for chunk in result:
assert len(chunk) <= 12 # chunk_size + some tolerance for overlap
+ def test_character_fallback_preserves_single_newlines(self):
+ text = "\n".join(f"SKU{i:04d}" for i in range(20))
+ splitter = FixedRecursiveCharacterTextSplitter(
+ separators=["\n\n", "。", ". ", " ", ""],
+ chunk_size=50,
+ chunk_overlap=0,
+ )
+
+ chunks = splitter.split_text(text)
+
+ assert chunks[0] == "SKU0000\nSKU0001\nSKU0002\nSKU0003\nSKU0004\nSKU0005\nSK"
+
def test_overlap_in_character_splitting(self):
"""Test that overlap is correctly applied in character-level splitting."""
text = string.ascii_uppercase
@@ -1040,6 +1065,7 @@ class TestFixedRecursiveCharacterTextSplitter:
assert chunks
assert all(chunk != "" for chunk in chunks)
+ assert chunks == ["line1\n\nline2\n\nline3"]
assert "line1" in "".join(chunks)
assert "line2" in "".join(chunks)
assert "line3" in "".join(chunks)