Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix: non-ASCII separators broken in FixedRecursiveCharacterTextSplitter (#33119)
fisherOne1fe6d3a9176bd
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.pyindex 8963c30c721..ba1cbcedc27 100644--- a/api/core/rag/splitter/fixed_text_splitter.py+++ b/api/core/rag/splitter/fixed_text_splitter.py@@ -2,7 +2,6 @@from __future__ import annotations-import codecsimport refrom collections.abc import Set as AbstractSetfrom typing import Any, Literal, override@@ -35,7 +34,7 @@ class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter)def __init__(self, fixed_separator: str = "\n\n", separators: list[str] | None = None, **kwargs: Any):"""Create a new TextSplitter."""super().__init__(**kwargs)- self._fixed_separator = codecs.decode(fixed_separator, "unicode_escape")+ self._fixed_separator = fixed_separator.encode("raw_unicode_escape").decode("unicode_escape")self._separators = separators or ["\n\n", "\n", "。", ". ", " ", ""]@overridediff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.pyindex 06c27b08b9c..63450d5773d 100644--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py@@ -901,8 +901,9 @@ class TestFixedRecursiveCharacterTextSplitter:result = splitter.split_text(text)- assert len(result) > 0+ assert len(result) > 1, "Chinese period separator should split text into multiple chunks"assert all(isinstance(chunk, str) for chunk in result)+ assert result == ["这是第一句", "这是第二句", "这是第三句", ""]def test_space_separator_handling(self):"""Test special handling of space separator."""