Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: preserve spaces during recursive text splitting (#39498)

CYJ1226e4e99d198b54
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py
index 98ba4d7fcc7..687f544e874 100644
--- a/api/core/rag/splitter/fixed_text_splitter.py
+++ b/api/core/rag/splitter/fixed_text_splitter.py
@@ -91,8 +91,8 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
splits = re.split(r" +", text)
else:
splits = text.split(separator)
- if self._keep_separator:
- splits = [s + separator for s in splits[:-1]] + splits[-1:]
+ if self._keep_separator:
+ splits = [s + separator for s in splits[:-1]] + splits[-1:]
else:
splits = list(text)
if separator == "\n":
diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
index 12117241b5d..7a0726d3a9c 100644
--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
@@ -952,6 +952,21 @@ class TestFixedRecursiveCharacterTextSplitter:
assert "word1" in combined
assert "word2" in combined
+ def test_preserves_spaces_when_recursively_splitting_long_paragraph(self):
+ """Ensure recursive space splitting preserves word boundaries."""
+ text = "여름철에는 항상 기상상황에 주목하며 주변 사람들과 함께 정보를 공유합니다."
+ splitter = FixedRecursiveCharacterTextSplitter(
+ fixed_separator="\n\n",
+ chunk_size=20,
+ chunk_overlap=0,
+ keep_separator=True,
+ )
+
+ result = splitter.split_text(text)
+
+ assert len(result) > 1
+ assert " ".join(result) == text
+
def test_character_level_splitting(self):
"""Test character-level splitting when no separator works."""
text = "verylongwordwithoutspaces"