Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix: non-ASCII separators broken in FixedRecursiveCharacterTextSplitter (#33119)

fisherOne1fe6d3a9176bd
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.py
index 8963c30c721..ba1cbcedc27 100644
--- a/api/core/rag/splitter/fixed_text_splitter.py
+++ b/api/core/rag/splitter/fixed_text_splitter.py
@@ -2,7 +2,6 @@
from __future__ import annotations
-import codecs
import re
from collections.abc import Set as AbstractSet
from typing import Any, Literal, override
@@ -35,7 +34,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)
def __init__(self, fixed_separator: str = "\n\n", separators: list[str] | None = None, **kwargs: Any):
"""Create a new TextSplitter."""
super().__init__(**kwargs)
- self._fixed_separator = codecs.decode(fixed_separator, "unicode_escape")
+ self._fixed_separator = fixed_separator.encode("raw_unicode_escape").decode("unicode_escape")
self._separators = separators or ["\n\n", "\n", "。", ". ", " ", ""]
@override
diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
index 06c27b08b9c..63450d5773d 100644
--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py
@@ -901,8 +901,9 @@ class TestFixedRecursiveCharacterTextSplitter:
result = splitter.split_text(text)
- assert len(result) > 0
+ assert len(result) > 1, "Chinese period separator should split text into multiple chunks"
assert all(isinstance(chunk, str) for chunk in result)
+ assert result == ["这是第一句", "这是第二句", "这是第三句", ""]
def test_space_separator_handling(self):
"""Test special handling of space separator."""