RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(title-chunker): lossless hard-split on multibyte + re-split logging (#18476)
Jackfad48b13a189
diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.pyindex ffa6b09744..fcdbfdae01 100644--- a/rag/flow/chunker/title_chunker/common.py+++ b/rag/flow/chunker/title_chunker/common.py@@ -132,6 +132,12 @@ class BaseTitleChunker(ABC):# Tokenizer unavailable/failed; fall back to a character prefix# so the ceiling still holds instead of raising.head = rest[:cap]+ # truncate decodes the first `cap` tokens; when that lands mid+ # multibyte character it emits a U+FFFD, which is not a true prefix+ # of `rest`. Trim it so the rest[len(head):] advance stays lossless+ # and no character is dropped at the split boundary.+ if head and not rest.startswith(head):+ head = head[:-1]if not head:out.append(rest)break@@ -219,6 +225,13 @@ class BaseTitleChunker(ABC):out.append(chunk)continueout.extend(self._split_text_chunk_by_cap(chunk, cap))+ if len(out) != len(chunks):+ logger.info(+ "title chunker token cap enforced: cap=%s chunks %d -> %d",+ cap,+ len(chunks),+ len(out),+ )return outdef extract_line_records(self):diff --git a/rag/flow/tests/test_title_chunker_token_cap.py b/rag/flow/tests/test_title_chunker_token_cap.pyindex 08c9e9516e..c925ab2e61 100644--- a/rag/flow/tests/test_title_chunker_token_cap.py+++ b/rag/flow/tests/test_title_chunker_token_cap.py@@ -498,4 +498,55 @@ def test_tokenizer_failure_falls_back_to_char_count():chunks = process._outputs.get("chunks", [])assert len(chunks) > 1, "cap must still apply when the tokenizer reports 0"+ for ck in chunks:+ # The character-count fallback must still honour the ceiling, not just+ # split somewhere: every emitted piece stays within the cap.+ assert len(ck["text"]) <= 20, f"fallback chunk exceeds cap: {ck['text']!r}"assert "".join(ck["text"] for ck in chunks) == "x" * 100 + "\n"+++# --------------------------------------------------------------------------- #+# 11. hard split is lossless with the REAL tiktoken-backed truncate #+# --------------------------------------------------------------------------- #+def test_hard_split_lossless_with_real_truncate():+ # _hard_split_by_tokens advances with rest[len(head):], which is only+ # lossless if head is a true prefix of rest. The real truncate+ # (encoder.decode(encoder.encode(s)[:n])) can decode a mid-multibyte token+ # boundary as U+FFFD, which is NOT a prefix. This exercises the hard-split+ # path with the real tokenizer on multibyte, boundary-less text and asserts+ # the concatenated sub-chunks reproduce the original text exactly (no+ # character dropped at the split boundary).+ import importlib.util++ root = Path(__file__).resolve().parents[3]+ spec = importlib.util.spec_from_file_location("ragflow_real_token_utils", str(root / "common" / "token_utils.py"))+ real_token_utils = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(real_token_utils)++ # Multibyte, no sentence boundary -> forces the hard-split fallback path.+ body = "中文混合English文本再混入一些标点符号\uff0c" * 30+ items = [{"text": body, "doc_type_kwd": "text"}]++ with _load_title_chunker_with_stubs() as (common_module, hierarchy_module, _g):+ # Swap the 1-token-per-char stub for the REAL tiktoken-backed truncate+ # AND the real token counter, so the cap is enforced and verified in+ # real token units (not characters).+ common_module.truncate = real_token_utils.truncate+ common_module.num_tokens_from_string = real_token_utils.num_tokens_from_string++ param = common_module.TitleChunkerParam()+ param.method = "hierarchy"+ param.chunk_token_cap = 20+ process = common_module.ProcessBase(None, "title_chunker", param)+ process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")+ process._outputs = {}+ chunker = hierarchy_module.HierarchyTitleChunker(process, _json_upstream(items))+ asyncio.run(chunker.invoke())+ chunks = process._outputs.get("chunks", [])++ assert len(chunks) > 1, "expected the oversized chunk to be hard-split"+ for ck in chunks:+ # Every emitted piece must honour the 20-token cap under the real+ # tokenizer, not just be a lossless concatenation.+ assert real_token_utils.num_tokens_from_string(ck["text"]) <= 20, f"hard-split chunk exceeds token cap: {ck['text']!r}"+ assert "".join(ck["text"] for ck in chunks) == body + "\n"