Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
"""Functionality for splitting text."""
from __future__ import annotations
import re
from collections.abc import Set as AbstractSet
from typing import Any, Literal, override
from core.model_manager import ModelInstance
from core.rag.splitter.text_splitter import RecursiveCharacterTextSplitter
class EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):
"""Create recursive splitters that measure chunk limits in characters."""
@classmethod
def from_encoder[T: EnhanceRecursiveCharacterTextSplitter](
cls: type[T],
embedding_model_instance: ModelInstance | None,
allowed_special: Literal["all"] | AbstractSet[str] = frozenset(),
disallowed_special: Literal["all"] | AbstractSet[str] = "all",
**kwargs: Any,
) -> T:
def _character_encoder(texts: list[str]) -> list[int]:
if not texts:
return []
return [len(text) for text in texts]
return cls(length_function=_character_encoder, **kwargs)
class FixedRecursiveCharacterTextSplitter(EnhanceRecursiveCharacterTextSplitter):
def __init__(self, fixed_separator: str = "\n\n", separators: list[str] | None = None, **kwargs: Any):
"""Create a new TextSplitter."""
super().__init__(**kwargs)
self._fixed_separator = fixed_separator.encode("raw_unicode_escape").decode("unicode_escape")
self._separators = separators or ["\n\n", "\n", "。", ". ", " ", ""]
@override
def split_text(self, text: str) -> list[str]:
"""Split incoming text and return chunks."""
chunks = text.split(self._fixed_separator) if self._fixed_separator else [text]
final_chunks = []
chunks_lengths = self._length_function(chunks)
for chunk, chunk_length in zip(chunks, chunks_lengths, strict=False):
if chunk_length > self._chunk_size:
final_chunks.extend(self.recursive_split_text(chunk))
else:
final_chunks.append(chunk)
return final_chunks
def recursive_split_text(self, text: str) -> list[str]:
"""Split incoming text and return chunks."""
final_chunks = []
separator = self._separators[-1]
new_separators = []
for i, _s in enumerate(self._separators):
if _s == "":
separator = _s
break
if _s in text:
separator = _s
new_separators = self._separators[i + 1 :]
break
# Now that we have the separator, split the text
if separator:
splits = re.split(r" +", text) if separator == " " else text.split(separator)
if self._keep_separator:
splits = [s + separator for s in splits[:-1]] + splits[-1:]
else:
splits = list(text)
splits = (
[s for s in splits if s != ""] if separator in {"", "\n"} else [s for s in splits if s not in {"", "\n"}]
)
_good_splits = []
_good_splits_lengths = [] # cache the lengths of the splits
_separator = "" if self._keep_separator else separator
s_lens = self._length_function(splits)
if separator != "":
for s, s_len in zip(splits, s_lens, strict=False):
if s_len < self._chunk_size:
_good_splits.append(s)
_good_splits_lengths.append(s_len)
else:
if _good_splits:
merged_text = self._merge_splits(_good_splits, _separator, _good_splits_lengths)
final_chunks.extend(merged_text)
_good_splits = []
_good_splits_lengths = []
if not new_separators:
final_chunks.append(s)
else:
other_info = self._split_text(s, new_separators)
final_chunks.extend(other_info)
if _good_splits:
merged_text = self._merge_splits(_good_splits, _separator, _good_splits_lengths)
final_chunks.extend(merged_text)
else:
current_part = ""
current_length = 0
overlap_part = ""
overlap_part_length = 0
for s, s_len in zip(splits, s_lens, strict=False):
if current_length + s_len <= self._chunk_size - self._chunk_overlap:
current_part += s
current_length += s_len
elif current_length + s_len <= self._chunk_size:
current_part += s
current_length += s_len
overlap_part += s
overlap_part_length += s_len
else:
final_chunks.append(current_part)
current_part = overlap_part + s
current_length = s_len + overlap_part_length
overlap_part = ""
overlap_part_length = 0
if current_part:
final_chunks.append(current_part)
return final_chunks