Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
refactor(rag): remove unused splitter tokenization (#42804)
-LAN-3113343c4dd3
diff --git a/api/core/rag/splitter/fixed_text_splitter.py b/api/core/rag/splitter/fixed_text_splitter.pyindex 687f544e874..ad850541512 100644--- a/api/core/rag/splitter/fixed_text_splitter.py+++ b/api/core/rag/splitter/fixed_text_splitter.py@@ -9,13 +9,10 @@ from typing import Any, Literal, overridefrom core.model_manager import ModelInstancefrom core.rag.splitter.text_splitter import RecursiveCharacterTextSplitter-from graphon.model_runtime.model_providers.base.tokenizers.gpt2_tokenizer import GPT2Tokenizerclass EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):- """- This class is used to implement from_gpt2_encoder, to prevent using of tiktoken- """+ """Create recursive splitters that measure chunk limits in characters."""@classmethoddef from_encoder[T: EnhanceRecursiveCharacterTextSplitter](@@ -25,22 +22,12 @@ class EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):disallowed_special: Literal["all"] | AbstractSet[str] = "all",**kwargs: Any,) -> T:- def _token_encoder(texts: list[str]) -> list[int]:- if not texts:- return []-- if embedding_model_instance:- return embedding_model_instance.get_text_embedding_num_tokens(texts=texts)- else:- return [GPT2Tokenizer.get_num_tokens(text) for text in texts]-def _character_encoder(texts: list[str]) -> list[int]:if not texts:return []return [len(text) for text in texts]- _ = _token_encoder # kept for future token-length wiringreturn cls(length_function=_character_encoder, **kwargs)diff --git a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.pyindex 7a0726d3a9c..eaf084ee25a 100644--- a/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py+++ b/api/tests/unit_tests/core/rag/splitter/test_text_splitter.py@@ -15,8 +15,8 @@ and semantic meaning.2. **TokenTextSplitter**: Splits text based on token count using tiktoken library,useful for LLM context window management.-3. **EnhanceRecursiveCharacterTextSplitter**: Enhanced version with custom token- counting support via embedding models or GPT2 tokenizer.+3. **EnhanceRecursiveCharacterTextSplitter**: Factory that uses character counts+ for chunk limits, with or without an embedding model.4. **FixedRecursiveCharacterTextSplitter**: Prioritizes a fixed separator beforefalling back to recursive splitting, useful for structured documents.@@ -130,7 +130,6 @@ import loggingimport stringimport sysimport types-from inspect import currentframefrom unittest.mock import Mock, patchimport pytest@@ -803,65 +802,20 @@ class TestEnhanceRecursiveCharacterTextSplitter:# Should split into multiple chunksassert len(result) >= 2- def test_with_embedding_model_token_counting(self):- """Test token counting with embedding model."""+ @pytest.mark.parametrize("use_model", [False, True])+ def test_from_encoder_uses_character_lengths(self, use_model):+ """Chunk limits count Unicode characters without invoking tokenization."""mock_model = Mock()- # Mock returns token counts for input texts- mock_model.get_text_embedding_num_tokens = Mock(side_effect=lambda texts: [len(t) // 2 for t in texts])-splitter = EnhanceRecursiveCharacterTextSplitter.from_encoder(- embedding_model_instance=mock_model, chunk_size=50, chunk_overlap=5+ embedding_model_instance=mock_model if use_model else None,+ chunk_size=4,+ chunk_overlap=0,+ separators=[""],)- text = "This is a test text that should be split"- result = splitter.split_text(text)-- assert len(result) > 0- assert all(isinstance(chunk, str) for chunk in result)-- def test_from_encoder_internal_token_encoder_paths(self):- """- Test internal _token_encoder branches by capturing local closure from frame.-- This validates:- - empty texts path- - embedding model path- - GPT2Tokenizer fallback path- - _character_encoder empty-path branch- """-- class _SpySplitter(EnhanceRecursiveCharacterTextSplitter):- captured_token_encoder = None- captured_character_encoder = None-- def __init__(self, **kwargs):- frame = currentframe()- if frame and frame.f_back:- _SpySplitter.captured_token_encoder = frame.f_back.f_locals.get("_token_encoder")- _SpySplitter.captured_character_encoder = frame.f_back.f_locals.get("_character_encoder")- super().__init__(**kwargs)-- mock_model = Mock()- mock_model.get_text_embedding_num_tokens.return_value = [3, 5]-- _SpySplitter.from_encoder(embedding_model_instance=mock_model, chunk_size=10, chunk_overlap=1)- token_encoder = _SpySplitter.captured_token_encoder- character_encoder = _SpySplitter.captured_character_encoder-- assert token_encoder is not None- assert character_encoder is not None- assert token_encoder([]) == []- assert token_encoder(["abc", "defgh"]) == [3, 5]- assert character_encoder([]) == []-- with patch(- "core.rag.splitter.fixed_text_splitter.GPT2Tokenizer.get_num_tokens",- side_effect=lambda text: len(text) + 1,- ):- _SpySplitter.from_encoder(embedding_model_instance=None, chunk_size=10, chunk_overlap=1)- token_encoder_without_model = _SpySplitter.captured_token_encoder- assert token_encoder_without_model is not None- assert token_encoder_without_model(["ab", "cdef"]) == [3, 5]+ assert splitter.split_text("aé中🙂b") == ["aé中🙂", "b"]+ assert splitter.split_text("") == []+ mock_model.get_text_embedding_num_tokens.assert_not_called()# ============================================================================