RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(title-chunker): configurable token-count cap with sentence-boundary re-split (#18455)
Jackebd6be09c7c2
diff --git a/rag/flow/chunker/_sentence_boundary.py b/rag/flow/chunker/_sentence_boundary.pynew file mode 100644index 0000000000..50fd0cd5e2--- /dev/null+++ b/rag/flow/chunker/_sentence_boundary.py@@ -0,0 +1,29 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++"""Sentence-boundary definition shared by the title and token chunkers.++Kept dependency-free so both chunkers can reuse the exact same split without+pulling in heavier parser/tokenizer modules.+"""++import re++# Boundaries used to re-split an oversized chunk into sentences. Covers+# Chinese/English period, exclamation mark, question mark, the Chinese+# variants, and the newline. The capturing group keeps the delimiter attached+# to the preceding sentence so re-merged text preserves original boundaries.+SENTENCE_BOUNDARY_PATTERN = r"([。!??;!\n]|\. )"++SENTENCE_BOUNDARY_RE = re.compile(SENTENCE_BOUNDARY_PATTERN, re.DOTALL)diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.pyindex 0848ac0ca7..ffa6b09744 100644--- a/rag/flow/chunker/title_chunker/common.py+++ b/rag/flow/chunker/title_chunker/common.py@@ -3,7 +3,6 @@## Licensed under the Apache License, Version 2.0 (the "License");# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at## http://www.apache.org/licenses/LICENSE-2.0#@@ -13,6 +12,14 @@# See the License for the specific language governing permissions and# limitations under the License.+"""Title-based chunking shared by the hierarchy and group strategies.++Both strategies build raw chunks from upstream line records, then a single+post-build pass (`BaseTitleChunker._enforce_token_cap`) guarantees no text+chunk exceeds the configured token ceiling.+"""++import loggingimport randomimport reimport sys@@ -20,9 +27,11 @@ from abc import ABC, abstractmethodfrom collections import Counterfrom copy import deepcopy+from common.token_utils import num_tokens_from_string, truncatefrom deepdoc.parser.pdf_parser import RAGFlowPdfParserfrom deepdoc.parser.utils import extract_pdf_outlinesfrom rag.flow.base import ProcessBase, ProcessParamBase+from rag.flow.chunker._sentence_boundary import SENTENCE_BOUNDARY_REfrom rag.flow.parser.pdf_chunk_metadata import (PDF_POSITIONS_KEY,extract_pdf_positions,@@ -32,28 +41,45 @@ from rag.flow.parser.pdf_chunk_metadata import ()from rag.nlp import not_bullet, not_title+logger = logging.getLogger(__name__)+BODY_LEVEL = sys.maxsize - 1class TitleChunkerParam(ProcessParamBase):+ """Parameters for the title-based chunkers (hierarchy and group)."""+def __init__(self):super().__init__()self.levels = []self.hierarchy = Noneself.include_heading_content = Falseself.root_chunk_as_heading = False+ # Hard ceiling on each text chunk's token count. A built chunk that+ # exceeds it is re-split on sentence boundaries into <= cap sub-chunks+ # (see BaseTitleChunker._enforce_token_cap). 0/None disables the+ # ceiling.+ self.chunk_token_cap = 512def check(self):+ """Validate the configured parameters before chunking runs."""if self.method in {"hierarchy", "group"}:self.check_empty(self.levels, "Hierarchical setups.")if self.method == "hierarchy":self.check_empty(self.hierarchy, "Hierarchy number.")+ if self.chunk_token_cap:+ self.check_positive_integer(self.chunk_token_cap, "Chunk token cap.")+ if not (128 <= int(self.chunk_token_cap) <= 8000):+ raise ValueError("Chunk token cap must be between 128 and 8000.")def get_input_form(self) -> dict[str, dict]:+ """Return the canvas-visible parameter form (empty for this chunker)."""return {}class BaseTitleChunker(ABC):+ """Shared base for the hierarchy and group title chunkers."""+start_message = "Start to chunk by title."def __init__(self, process: ProcessBase, from_upstream):@@ -62,26 +88,145 @@ class BaseTitleChunker(ABC):self.from_upstream = from_upstreamasync def invoke(self):+ """Run the full chunking pipeline for one upstream payload."""self.process.set_output("output_format", "chunks")self.process.callback(random.randint(1, 5) / 100.0, self.start_message)line_records = self.extract_line_records()resolved = self.resolve_levels(line_records)chunks = self.build_chunks(line_records, resolved)+ chunks = self._enforce_token_cap(chunks)await self.set_chunks(chunks)self.process.callback(1, "Done.")- def extract_line_records(self):+ @staticmethod+ def _split_text_by_sentences(text):+ """Split text on sentence boundaries, keeping each delimiter attached+ to the preceding sentence so re-merged text preserves punctuation.+ """+ if not text:+ return []+ raw = re.split(SENTENCE_BOUNDARY_RE, text)+ sentences = []+ for i in range(0, len(raw), 2):+ piece = raw[i]+ if i + 1 < len(raw):+ piece += raw[i + 1]+ if piece:+ sentences.append(piece)+ return sentences++ @staticmethod+ def _hard_split_by_tokens(text, cap):+ """Hard-split a boundary-less run into <= cap-token pieces.++ Mirrors common.token_utils.truncate semantics (prefix bounded by+ ``cap`` tokens). Falls back to a character prefix if the tokenizer is+ unavailable so the ceiling still holds.+ """+ out = []+ rest = text or ""+ while rest:+ try:+ head = truncate(rest, cap)+ except (ValueError, TypeError, UnicodeError):+ # Tokenizer unavailable/failed; fall back to a character prefix+ # so the ceiling still holds instead of raising.+ head = rest[:cap]+ if not head:+ out.append(rest)+ break+ out.append(head)+ if head == rest:+ break+ rest = rest[len(head) :]+ return out++ def _split_text_chunk_by_cap(self, chunk, cap):+ """Re-split one oversized text chunk into <= cap sub-chunks.++ Sentence boundaries are tried first; any remaining over-cap segment is+ hard-split. PDF coordinates follow plan A: the original (coarse,+ page-level) positions attach to the first sub-chunk only."""- Normalize all upstream input payloads into a unified ordered record stream.- All level resolution and chunk construction logic operates on this standard stream,- decoupling downstream chunking strategies from different upstream output formats.+ text = chunk.get("text") or ""+ sentences = self._split_text_by_sentences(text)+ if not sentences:+ return [chunk]++ groups = []+ current = ""+ for sentence in sentences:+ candidate = current + sentence if current else sentence+ if current and self._token_count(candidate) > cap:+ groups.append(current)+ current = sentence+ else:+ current = candidate+ if current:+ groups.append(current)++ final_groups = []+ for group in groups:+ if self._token_count(group) <= cap:+ final_groups.append(group)+ else:+ final_groups.extend(self._hard_split_by_tokens(group, cap))+ if not final_groups:+ return [chunk]++ has_positions = PDF_POSITIONS_KEY in chunk+ orig_positions = chunk.get(PDF_POSITIONS_KEY)+ out = []+ for i, group in enumerate(final_groups):+ sub = dict(chunk)+ sub["text"] = group+ # Plan A: the original (coarse, page-level) coordinates attach to the+ # first sub-chunk only; subsequent sub-chunks carry none.+ if has_positions:+ sub[PDF_POSITIONS_KEY] = orig_positions if i == 0 else []+ out.append(sub)+ return out++ def _token_count(self, text):+ """Count tokens for ``text``.++ ``num_tokens_from_string`` returns 0 when the encoder is unavailable;+ in that case fall back to the character count so the cap is still+ enforced rather than silently skipped."""- import logging+ n = num_tokens_from_string(text or "")+ if n == 0 and text:+ logger.warning("tokenizer returned 0 tokens for non-empty text; falling back to character count for the token cap")+ return len(text or "")+ return n- logger = logging.getLogger(__name__)+ def _enforce_token_cap(self, chunks):+ """Apply the hard token ceiling to every text chunk after build_chunks.+ Both hierarchy and group methods honour ``chunk_token_cap``. Table and+ image chunks are atomic and skipped. ``cap`` of 0/None disables it.+ """+ cap = self.param.chunk_token_cap+ if not cap or cap <= 0:+ return chunks++ out = []+ for chunk in chunks:+ if chunk.get("doc_type_kwd", "text") != "text":+ out.append(chunk)+ continue+ if self._token_count(chunk.get("text") or "") <= cap:+ out.append(chunk)+ continue+ out.extend(self._split_text_chunk_by_cap(chunk, cap))+ return out++ def extract_line_records(self):+ """Normalize all upstream input payloads into a unified ordered record+ stream. All level resolution and chunk construction operate on this+ standard stream, decoupling strategies from upstream output formats.+ """payload = None- # Extract raw content payload based on upstream output format typeif self.from_upstream.output_format == "markdown":payload = self.from_upstream.markdown_result or ""elif self.from_upstream.output_format == "text":@@ -89,24 +234,17 @@ class BaseTitleChunker(ABC):elif self.from_upstream.output_format == "html":payload = self.from_upstream.html_result or ""- # Boundary robustness fix: explicit None check to distinguish `None` and empty string ""- # Prevents empty payload from unexpectedly falling through to structured chunk branchif payload is not None:lines = payload.split("\n")input_line_count = len(lines)-- # Format-branched text processing to preserve original document semantics- # Plain text: perform full whitespace stripping and invalid empty line filtering+ # Plain text: full whitespace strip + drop blank lines. Markdown &+ # HTML: keep original spacing, drop only pure blank lines.if self.from_upstream.output_format == "text":clean_lines = [line.strip() for line in lines if line.strip()]- # Markdown & HTML: retain original indentation/spacing, only filter pure blank lineselse:clean_lines = [line for line in lines if line.strip()]-output_line_count = len(clean_lines)- # Production observability log: added format dimension per project coding guidelineslogger.info(f"payload filter: format={self.from_upstream.output_format} before={input_line_count} after={output_line_count}")-return [{"text": line, "doc_type_kwd": "text", "img_id": None, "layout": "", PDF_POSITIONS_KEY: []} for line in clean_lines]items = self.from_upstream.chunks if self.from_upstream.output_format == "chunks" else self.from_upstream.json_resultreturn [@@ -121,6 +259,7 @@ class BaseTitleChunker(ABC):]def extract_outlines(self):+ """Extract PDF bookmarks/outlines used for outline-based leveling."""file = self.from_upstream.file or {}source = file.get("blob") or file.get("binary") or file.get("path") or file.get("name")if not source:@@ -129,6 +268,7 @@ class BaseTitleChunker(ABC):@staticmethoddef match_regex_level(text, level_group):+ """Return the 1-based level whose regex matches ``text``, else None."""stripped = text.strip()for level, pattern in enumerate(level_group, start=1):if re.match(pattern, stripped) and not not_bullet(stripped):@@ -137,12 +277,13 @@ class BaseTitleChunker(ABC):@staticmethoddef select_level_group(lines, raw_levels):+ """Pick the single regex family that best matches ``lines``.++ Mixing families would make level numbers ambiguous and break downstream+ comparisons, so only the most frequently matching family is kept.+ """if not raw_levels:return []-- # Select one regex family before assigning numeric levels. Mixing- # patterns across families would make the level numbers ambiguous and- # break downstream comparisons.hits = [0] * len(raw_levels)for i, group in enumerate(raw_levels):for sec in lines:@@ -153,7 +294,6 @@ class BaseTitleChunker(ABC):if re.match(pattern, sec) and not not_bullet(sec):hits[i] += 1break-maximum = 0selected = -1for i, hit in enumerate(hits):@@ -161,28 +301,29 @@ class BaseTitleChunker(ABC):continueselected = imaximum = hit-if selected < 0:return []return [pattern for pattern in raw_levels[selected] if pattern]@staticmethoddef match_layout_level(text, layout, fallback_level):- if re.search(r"(section|title|head)", layout, re.I) and not not_title(text.split("@")[0].strip()):+ """Treat layout-tagged title-like lines as ``fallback_level``."""+ if re.search(r"(section|title|head)", layout, re.IGNORECASE) and not not_title(text.split("@")[0].strip()):return fallback_levelreturn BODY_LEVEL@staticmethoddef _outline_similarity(left, right):+ """Jaccard similarity of adjacent outline-text bigrams."""left_pairs = {left[i] + left[i + 1] for i in range(len(left) - 1)}right_pairs = {right[i] + right[i + 1] for i in range(min(len(left), len(right) - 1))}return len(left_pairs & right_pairs) / max(len(left_pairs), len(right_pairs), 1)def resolve_outline_levels(self, line_records):+ """Resolve levels from PDF outlines when they cover enough of the doc."""outlines = self.extract_outlines()if not line_records or len(outlines) / len(line_records) <= 0.03:return None-max_level = max(level for _, level, _ in outlines) + 1levels = []for record in line_records:@@ -196,7 +337,6 @@ class BaseTitleChunker(ABC):breakelse:levels.append(BODY_LEVEL)-return {"levels": levels,"most_level": max(1, max_level - 1),@@ -204,6 +344,7 @@ class BaseTitleChunker(ABC):}def resolve_frequency_levels(self, line_records):+ """Resolve levels by regex family and layout tagging."""level_group = self.select_level_group([record["text"] for record in line_records],self.param.levels,@@ -225,13 +366,11 @@ class BaseTitleChunker(ABC):fallback_level,))-most_level = Nonefor level, _ in Counter(levels).most_common():if level < BODY_LEVEL:most_level = levelbreak-return {"levels": levels,"most_level": most_level,@@ -239,13 +378,15 @@ class BaseTitleChunker(ABC):}def resolve_title_levels(self, line_records):+ """Resolve levels, preferring outlines then falling back to frequency."""return self.resolve_outline_levels(line_records) or self.resolve_frequency_levels(line_records)def build_chunks_from_record_groups(self, record_groups):- # Strategy code decides record grouping. This method materializes each- # group into the output chunk representation. For PDF-like inputs, the- # chunk box is defined by merged source positions and the text payload- # is normalized by removing parser tags.+ """Materialize each record group into the output chunk representation.++ For PDF-like inputs the chunk box is defined by merged source positions+ and the text payload is normalized by removing parser tags.+ """if self.from_upstream.output_format in ["markdown", "text", "html"]:chunks = [{"text": "".join(record["text"] + "\n" for record in records)} for records in record_groups if records]else:@@ -280,6 +421,7 @@ class BaseTitleChunker(ABC):return chunksasync def set_chunks(self, chunks):+ """Finalize and emit chunks, enriching PDF positions when needed."""if self.from_upstream.output_format in ["markdown", "text", "html"]:self.process.set_output("chunks", chunks)return@@ -291,14 +433,17 @@ class BaseTitleChunker(ABC):@abstractmethoddef resolve_levels(self, line_records):+ """Resolve title levels for the concrete chunker strategy."""raise NotImplementedError()@abstractmethoddef build_chunks(self, line_records, resolved):+ """Build raw chunks from records and resolved levels."""raise NotImplementedError()def resolve_target_level(levels, hierarchy):+ """Pick the title level used as the chunking target for ``hierarchy``."""title_levels = sorted({level for level in levels if 0 < level < BODY_LEVEL})if not title_levels:return Nonediff --git a/rag/flow/chunker/title_chunker/group_chunker.py b/rag/flow/chunker/title_chunker/group_chunker.pyindex bb4d5e9777..2026565fcb 100644--- a/rag/flow/chunker/title_chunker/group_chunker.py+++ b/rag/flow/chunker/title_chunker/group_chunker.py@@ -20,10 +20,11 @@ from rag.flow.chunker.title_chunker.common import ()MIN_GROUP_TOKENS = 32-MAX_GROUP_TOKENS = 1024def _build_section_ids(levels, target_level):+ """Assign a stable section id that increments whenever a title at or above+ ``target_level`` starts a new section."""sec_ids = []sid = 0for i, level in enumerate(levels):@@ -34,18 +35,30 @@ def _build_section_ids(levels, target_level):def _resolve_group_target_level(levels, hierarchy, most_level):+ """Pick the level used as the grouping target for the group method."""if hierarchy and int(hierarchy) > 0:return resolve_target_level(levels, hierarchy)return most_levelclass GroupTitleChunker(BaseTitleChunker):+ """Group consecutive records under the same title into one chunk."""+start_message = "Start to group by title levels."def resolve_levels(self, line_records):+ """Resolve title levels via the shared outline/frequency strategy."""return self.resolve_title_levels(line_records)def build_chunks(self, line_records, resolved):+ """Build chunks by merging records inside the same logical section.++ The merge ceiling uses the configurable ``chunk_token_cap`` (0/None+ means "no ceiling"). The post-build ``_enforce_token_cap`` in+ BaseTitleChunker is the single hard guarantee, so any residual over-cap+ chunk (e.g. a single record bigger than the cap) is still re-split+ there.+ """target_level = _resolve_group_target_level(resolved["levels"],self.param.hierarchy,@@ -56,6 +69,8 @@ class GroupTitleChunker(BaseTitleChunker):tk_cnt = 0last_sid = -2+ cap = self.param.chunk_token_cap or 0+# The merge state is driven by (current section id, current token size).# A chunk stays open while records remain in the same logical section,# except that very small chunks are allowed to absorb the next record@@ -72,7 +87,8 @@ class GroupTitleChunker(BaseTitleChunker):continuetoken_count = num_tokens_from_string(text)- should_merge = record_groups and record_groups[-1][0]["doc_type_kwd"] == "text" and (tk_cnt < MIN_GROUP_TOKENS or (tk_cnt < MAX_GROUP_TOKENS and sec_id == last_sid))+ merge_ceiling_ok = cap <= 0 or tk_cnt < cap+ should_merge = record_groups and record_groups[-1][0]["doc_type_kwd"] == "text" and (tk_cnt < MIN_GROUP_TOKENS or (merge_ceiling_ok and sec_id == last_sid))if should_merge:record_groups[-1].append(record)diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.pyindex cc6e296798..2038817053 100644--- a/rag/flow/chunker/token_chunker.py+++ b/rag/flow/chunker/token_chunker.py@@ -19,6 +19,7 @@ from copy import deepcopyfrom common.float_utils import normalize_overlapped_percentfrom common.token_utils import num_tokens_from_stringfrom rag.flow.base import ProcessBase, ProcessParamBase+from rag.flow.chunker._sentence_boundary import SENTENCE_BOUNDARY_PATTERNfrom rag.flow.chunker.schema import TokenChunkerFromUpstreamfrom rag.flow.parser.pdf_chunk_metadata import (PDF_POSITIONS_KEY,@@ -173,8 +174,7 @@ def _build_json_chunks(json_result, delimiter_pattern):def _take_sentences(text, need_tokens, from_end=False):# Take text from one side until the target token budget is reached.- split_pat = r"([。!??;!\n]|\. )"- texts = re.split(split_pat, text or "", flags=re.DOTALL)+ texts = re.split(SENTENCE_BOUNDARY_PATTERN, text or "", flags=re.DOTALL)sentences = []for i in range(0, len(texts), 2):sentences.append(texts[i] + (texts[i + 1] if i + 1 < len(texts) else ""))diff --git a/rag/flow/tests/test_title_chunker_position_int.py b/rag/flow/tests/test_title_chunker_position_int.pyindex 2edd07224f..5ea8827c8c 100644--- a/rag/flow/tests/test_title_chunker_position_int.py+++ b/rag/flow/tests/test_title_chunker_position_int.py@@ -60,6 +60,7 @@ def _load_title_chunker_with_stubs():common_token_utils = types.ModuleType("common.token_utils")common_token_utils.num_tokens_from_string = lambda text: 1+ common_token_utils.truncate = lambda text, max_len: (text or "")[:max_len]_install("common.token_utils", common_token_utils)rag_nlp = types.ModuleType("rag.nlp")@@ -247,6 +248,7 @@ def test_title_chunker_preserves_position_int_from_deepdoc_json():param.levels = [] # no headings -> all body -> single merged chunkparam.include_heading_content = Falseparam.root_chunk_as_heading = False+ param.chunk_token_cap = 0 # keep this a pure position regression (no re-split)process = common_module.ProcessBase(None, "title_chunker", param)process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")diff --git a/rag/flow/tests/test_title_chunker_token_cap.py b/rag/flow/tests/test_title_chunker_token_cap.pynew file mode 100644index 0000000000..08c9e9516e--- /dev/null+++ b/rag/flow/tests/test_title_chunker_token_cap.py@@ -0,0 +1,501 @@+import asyncio+import importlib+import importlib.util+import sys+import types+from contextlib import contextmanager+from pathlib import Path++"""TDD tests for the TitleChunker token-count CAP (chunk_token_cap).++The TitleChunker (both ``hierarchy`` and ``group`` methods) historically had+no token-size ceiling: a long section with no sub-heading became one giant+chunk. This suite pins the new behaviour:++ * A configurable ``chunk_token_cap`` (default 512, valid 128..8000) is a+ hard ceiling on every *text* chunk's token count.+ * When a built chunk exceeds the cap, it is re-split on sentence boundaries+ (``。!??;!\\n`` plus the English ``. ``) and the pieces are greedily+ merged into <= cap sub-chunks.+ * A single boundary-less run that still exceeds the cap is hard-split so the+ ceiling always holds (fallback only).+ * Table/image chunks are atomic and never split.+ * ``chunk_token_cap == 0`` disables the ceiling.++Token counting is faked as 1 token == 1 character so the assertions are fully+deterministic (mirrors the stub approach in test_title_chunker_position_int.py,+which also fakes ``num_tokens_from_string``).+"""+++@contextmanager+def _load_title_chunker_with_stubs():+ root = Path(__file__).resolve().parents[3]+ original_modules = {}++ def _install(name: str, module: types.ModuleType):+ original_modules.setdefault(name, sys.modules.get(name))+ sys.modules[name] = module++ try:+ rag_pkg = types.ModuleType("rag")+ rag_pkg.__path__ = [str(root / "rag")]+ _install("rag", rag_pkg)++ rag_flow_pkg = types.ModuleType("rag.flow")+ rag_flow_pkg.__package__ = "rag"+ rag_flow_pkg.__path__ = [str(root / "rag" / "flow")]+ _install("rag.flow", rag_flow_pkg)++ rag_flow_chunker_pkg = types.ModuleType("rag.flow.chunker")+ rag_flow_chunker_pkg.__package__ = "rag.flow"+ rag_flow_chunker_pkg.__path__ = [str(root / "rag" / "flow" / "chunker")]+ _install("rag.flow.chunker", rag_flow_chunker_pkg)++ rag_flow_parser_pkg = types.ModuleType("rag.flow.parser")+ rag_flow_parser_pkg.__package__ = "rag.flow"+ rag_flow_parser_pkg.__path__ = [str(root / "rag" / "flow" / "parser")]+ _install("rag.flow.parser", rag_flow_parser_pkg)++ common_pkg = types.ModuleType("common")+ common_pkg.__path__ = [str(root / "common")]+ _install("common", common_pkg)++ common_float_utils = types.ModuleType("common.float_utils")+ common_float_utils.normalize_overlapped_percent = lambda value: value+ _install("common.float_utils", common_float_utils)++ # Deterministic tokenizer: 1 token per character. ``truncate`` mirrors+ # token_utils.truncate semantics (prefix bounded by max_len tokens).+ common_token_utils = types.ModuleType("common.token_utils")+ common_token_utils.num_tokens_from_string = lambda text: len(text or "")+ common_token_utils.truncate = lambda text, max_len: (text or "")[:max_len]+ _install("common.token_utils", common_token_utils)++ rag_nlp = types.ModuleType("rag.nlp")+ rag_nlp.naive_merge = lambda *args, **kwargs: []+ rag_nlp.not_bullet = lambda text: False+ rag_nlp.not_title = lambda text: True+ _install("rag.nlp", rag_nlp)++ deepdoc_pkg = types.ModuleType("deepdoc")+ deepdoc_pkg.__path__ = [str(root / "deepdoc")]+ _install("deepdoc", deepdoc_pkg)++ deepdoc_parser_pkg = types.ModuleType("deepdoc.parser")+ deepdoc_parser_pkg.__path__ = [str(root / "deepdoc" / "parser")]+ _install("deepdoc.parser", deepdoc_parser_pkg)++ class _RAGFlowPdfParser:+ @staticmethod+ def remove_tag(text):+ return text++ @staticmethod+ def extract_positions(tag):+ return []++ deepdoc_pdf_parser = types.ModuleType("deepdoc.parser.pdf_parser")+ deepdoc_pdf_parser.RAGFlowPdfParser = _RAGFlowPdfParser+ _install("deepdoc.parser.pdf_parser", deepdoc_pdf_parser)++ deepdoc_parser_utils = types.ModuleType("deepdoc.parser.utils")+ deepdoc_parser_utils.extract_pdf_outlines = lambda *args, **kwargs: []+ _install("deepdoc.parser.utils", deepdoc_parser_utils)++ class ProcessParamBase:+ def __init__(self):+ pass++ def check_valid_value(self, value, msg, allowed):+ if value not in allowed:+ raise ValueError(msg)++ def check_positive_integer(self, value, msg):+ if not (isinstance(value, int) and value > 0):+ raise ValueError(msg)++ def check_decimal_float(self, value, msg):+ pass++ def check_nonnegative_number(self, value, msg):+ pass++ def check_empty(self, value, msg):+ pass++ class ProcessBase:+ def __init__(self, _pipeline, _id, param):+ self._pipeline = _pipeline+ self._id = _id+ self._param = param+ self._outputs = {}+ self.callback = lambda *_args, **_kwargs: None++ def set_output(self, key, value):+ self._outputs[key] = value++ rag_flow_base = types.ModuleType("rag.flow.base")+ rag_flow_base.ProcessBase = ProcessBase+ rag_flow_base.ProcessParamBase = ProcessParamBase+ _install("rag.flow.base", rag_flow_base)++ pdf_chunk_metadata = types.ModuleType("rag.flow.parser.pdf_chunk_metadata")+ pdf_chunk_metadata.PDF_POSITIONS_KEY = "pdf_positions"+ pdf_chunk_metadata.extract_pdf_positions = lambda _item: []+ pdf_chunk_metadata.merge_pdf_positions = lambda _records: []+ pdf_chunk_metadata.finalize_pdf_chunk = lambda chunk: chunk++ async def _noop_restore(*_a, **_k):+ return None++ pdf_chunk_metadata.restore_pdf_text_previews = _noop_restore+ _install("rag.flow.parser.pdf_chunk_metadata", pdf_chunk_metadata)++ common_spec = importlib.util.spec_from_file_location(+ "rag.flow.chunker.title_chunker.common",+ root / "rag" / "flow" / "chunker" / "title_chunker" / "common.py",+ )+ common_module = importlib.util.module_from_spec(common_spec)+ _install("rag.flow.chunker.title_chunker.common", common_module)+ common_spec.loader.exec_module(common_module)++ hierarchy_spec = importlib.util.spec_from_file_location(+ "rag.flow.chunker.title_chunker.hierarchy_chunker",+ root / "rag" / "flow" / "chunker" / "title_chunker" / "hierarchy_chunker.py",+ )+ hierarchy_module = importlib.util.module_from_spec(hierarchy_spec)+ _install("rag.flow.chunker.title_chunker.hierarchy_chunker", hierarchy_module)+ hierarchy_spec.loader.exec_module(hierarchy_module)++ group_spec = importlib.util.spec_from_file_location(+ "rag.flow.chunker.title_chunker.group_chunker",+ root / "rag" / "flow" / "chunker" / "title_chunker" / "group_chunker.py",+ )+ group_module = importlib.util.module_from_spec(group_spec)+ _install("rag.flow.chunker.title_chunker.group_chunker", group_module)+ group_spec.loader.exec_module(group_module)++ yield common_module, hierarchy_module, group_module+ finally:+ for module_name, original in original_modules.items():+ if original is None:+ sys.modules.pop(module_name, None)+ else:+ sys.modules[module_name] = original+++def _real_extract_pdf_positions(item):+ if not isinstance(item, dict):+ return []+ positions = item.get("pdf_positions")+ if isinstance(positions, list):+ return [list(p) for p in positions]+ positions = item.get("positions")+ if isinstance(positions, list):+ return [list(p) for p in positions]+ return []+++def _real_merge_pdf_positions(records):+ merged = []+ for rec in records or []:+ if not isinstance(rec, dict):+ continue+ for pos in rec.get("pdf_positions") or []:+ if isinstance(pos, (list, tuple)) and len(pos) >= 5:+ merged.append([pos[0], pos[1], pos[2], pos[3], pos[4]])+ seen = set()+ out = []+ for pos in merged:+ key = tuple(pos[:5])+ if key not in seen:+ seen.add(key)+ out.append(pos)+ return out+++def _real_finalize_pdf_chunk(chunk):+ positions = _real_extract_pdf_positions(chunk)+ if positions:+ chunk["position_int"] = [list(p) for p in positions]+ chunk.pop("pdf_positions", None)+ return chunk+++def _run(method, from_upstream, **param_kwargs):+ with _load_title_chunker_with_stubs() as (common_module, hierarchy_module, group_module):+ if param_kwargs.pop("faithful_positions", False):+ common_module.extract_pdf_positions = _real_extract_pdf_positions+ common_module.merge_pdf_positions = _real_merge_pdf_positions+ common_module.finalize_pdf_chunk = _real_finalize_pdf_chunk++ param = common_module.TitleChunkerParam()+ param.method = method+ param.chunk_token_cap = param_kwargs.pop("chunk_token_cap", 512)+ for key, value in param_kwargs.items():+ setattr(param, key, value)++ process = common_module.ProcessBase(None, "title_chunker", param)+ process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")+ process._outputs = {}++ if method == "hierarchy":+ chunker = hierarchy_module.HierarchyTitleChunker(process, from_upstream)+ else:+ chunker = group_module.GroupTitleChunker(process, from_upstream)+ asyncio.run(chunker.invoke())+ return process._outputs.get("chunks", [])+++def _json_upstream(items, output_format="json"):+ return types.SimpleNamespace(+ output_format=output_format,+ json_result=items if output_format == "json" else None,+ markdown_result=None,+ text_result=None,+ html_result=None,+ chunks=None if output_format == "json" else items,+ file=None,+ name="cap-test",+ )+++def _char_count(text):+ return len(text or "")+++# --------------------------------------------------------------------------- #+# 1. hierarchy: a single oversized chunk is re-split and stays within the cap #+# --------------------------------------------------------------------------- #+def test_hierarchy_oversized_chunk_respects_cap():+ # 12 sentences of 4 chars each ("Saa。") -> 48 tokens, cap 20 -> >=3 chunks.+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)++ assert len(chunks) > 1, "expected the oversized chunk to be split"+ for ck in chunks:+ assert ck.get("doc_type_kwd", "text") == "text"+ assert _char_count(ck["text"]) <= 20, f"chunk exceeds cap: {ck['text']!r}"+ # Content is preserved (single record -> build_chunks appends one trailing+ # newline, which the sentence split keeps as a boundary; rstrip for compare).+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 2. group: same guarantee, different code path #+# --------------------------------------------------------------------------- #+def test_group_oversized_chunk_respects_cap():+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("group", _json_upstream(items), levels=[], hierarchy=0, chunk_token_cap=20)++ assert len(chunks) > 1+ for ck in chunks:+ assert _char_count(ck["text"]) <= 20+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 3. split happens on sentence boundaries, not mid-sentence #+# --------------------------------------------------------------------------- #+def test_split_keeps_sentence_boundaries():+ # Every sentence ends with "。"; after greedy merge each chunk must still+ # end on a boundary (no mid-sentence cut while boundaries are available).+ body = "。".join(f"Sentence number {i} ends here" for i in range(10)) + "。"+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=40)++ assert len(chunks) > 1+ for ck in chunks:+ assert ck["text"].rstrip("\n").endswith("。"), f"chunk cut mid-sentence: {ck['text']!r}"+++# --------------------------------------------------------------------------- #+# 4. chunk_token_cap == 0 disables the ceiling #+# --------------------------------------------------------------------------- #+def test_cap_zero_disables_splitting():+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=0)++ assert len(chunks) == 1, "cap=0 must keep the single-chunk behaviour"+ assert chunks[0]["text"].rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 5. table/image chunks are atomic and never split #+# --------------------------------------------------------------------------- #+def test_non_text_chunk_is_atomic():+ big_table = "x" * 200 # 200 tokens, far above any reasonable cap+ items = [{"text": big_table, "doc_type_kwd": "table"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)++ assert len(chunks) == 1+ assert chunks[0]["doc_type_kwd"] == "table"+ assert chunks[0]["text"] == big_table+++# --------------------------------------------------------------------------- #+# 6. pathological single run with no boundary is hard-split (fallback) #+# --------------------------------------------------------------------------- #+def test_boundaryless_run_is_hard_split():+ # 100 chars, no punctuation -> sentence split yields one giant segment that+ # must be hard-split so the cap still holds.+ body = "x" * 100+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)++ assert len(chunks) > 1+ for ck in chunks:+ assert _char_count(ck["text"]) <= 20+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 7. PDF positions: plan A - first sub-chunk keeps coordinates, rest empty #+# --------------------------------------------------------------------------- #+def test_position_plan_a_first_subchunk_keeps_coordinates():+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ items = [{"text": body, "doc_type_kwd": "text", "positions": [[1, 10, 200, 50, 80]]}]+ chunks = _run(+ "hierarchy",+ _json_upstream(items),+ levels=[],+ chunk_token_cap=20,+ faithful_positions=True,+ )++ assert len(chunks) > 1+ assert chunks[0].get("position_int") == [[1, 10, 200, 50, 80]], "first sub-chunk must keep coordinates"+ for ck in chunks[1:]:+ assert "position_int" not in ck, "only the first sub-chunk should carry coordinates"+++# --------------------------------------------------------------------------- #+# 7b. text output format (no doc_type_kwd) also honours the cap #+# --------------------------------------------------------------------------- #+def test_hierarchy_text_format_respects_cap():+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"+ from_upstream = types.SimpleNamespace(+ output_format="text",+ json_result=None,+ markdown_result=None,+ text_result=body,+ html_result=None,+ chunks=None,+ file=None,+ name="cap-test",+ )+ chunks = _run("hierarchy", from_upstream, levels=[], chunk_token_cap=20)++ assert len(chunks) > 1+ for ck in chunks:+ assert _char_count(ck["text"]) <= 20+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 7c. multiple records merged into one oversized chunk are re-split #+# --------------------------------------------------------------------------- #+def test_hierarchy_multi_record_merged_respects_cap():+ items = [{"text": f"S{i:02d}。", "doc_type_kwd": "text"} for i in range(6)]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=10)++ assert len(chunks) > 1+ for ck in chunks:+ assert _char_count(ck["text"]) <= 10+ expected_built = "".join(f"S{i:02d}。" + "\n" for i in range(6))+ assert "".join(ck["text"] for ck in chunks) == expected_built+++# --------------------------------------------------------------------------- #+# 7d. a chunk already within the cap is left untouched (no spurious split) #+# --------------------------------------------------------------------------- #+def test_chunk_within_cap_not_split():+ body = "S00。S01。"+ items = [{"text": body, "doc_type_kwd": "text"}]+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=512)++ assert len(chunks) == 1+ assert chunks[0]["text"].rstrip("\n") == body+++# --------------------------------------------------------------------------- #+# 8. range validation in TitleChunkerParam.check #+# --------------------------------------------------------------------------- #+def test_chunk_token_cap_range_validation():+ with _load_title_chunker_with_stubs() as (common_module, _h, _g):+ param = common_module.TitleChunkerParam()+ param.method = "hierarchy"+ param.levels = []++ param.chunk_token_cap = 50+ try:+ param.check()+ raise AssertionError("expected ValueError for cap < 128")+ except ValueError:+ pass++ param.chunk_token_cap = 9000+ try:+ param.check()+ raise AssertionError("expected ValueError for cap > 8000")+ except ValueError:+ pass++ # Valid defaults must not raise.+ param.chunk_token_cap = 512+ param.check()+ param.chunk_token_cap = 128+ param.check()+ param.chunk_token_cap = 8000+ param.check()+++# --------------------------------------------------------------------------- #+# 9. hard split is lossless: sub-chunks concatenate back to the original text #+# --------------------------------------------------------------------------- #+def test_hard_split_is_lossless():+ # A boundary-less run is hard-split; concatenating every sub-chunk must+ # reproduce the original built chunk text exactly (no characters dropped+ # or duplicated across the token-prefix truncation).+ body = "x" * 100+ items = [{"text": body, "doc_type_kwd": "text"}]++ disabled = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=0)+ original = disabled[0]["text"]++ capped = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)+ assert len(capped) > 1, "expected the oversized chunk to be hard-split"+ assert "".join(ck["text"] for ck in capped) == original+++# --------------------------------------------------------------------------- #+# 10. tokenizer failure still enforces the cap via character-count fallback #+# --------------------------------------------------------------------------- #+def test_tokenizer_failure_falls_back_to_char_count():+ # Simulate an unavailable tokenizer: num_tokens_from_string returns 0 for+ # every text. The cap must still apply (via the character-count fallback)+ # instead of silently leaving a giant chunk intact.+ body = "x" * 100+ items = [{"text": body, "doc_type_kwd": "text"}]++ with _load_title_chunker_with_stubs() as (common_module, hierarchy_module, _group_module):+ # Override the stubbed tokenizer to always report 0 tokens.+ common_module.num_tokens_from_string = lambda text: 0++ param = common_module.TitleChunkerParam()+ param.method = "hierarchy"+ param.chunk_token_cap = 20+ process = common_module.ProcessBase(None, "title_chunker", param)+ process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")+ process._outputs = {}++ chunker = hierarchy_module.HierarchyTitleChunker(process, _json_upstream(items))+ asyncio.run(chunker.invoke())+ chunks = process._outputs.get("chunks", [])++ assert len(chunks) > 1, "cap must still apply when the tokenizer reports 0"+ assert "".join(ck["text"] for ck in chunks) == "x" * 100 + "\n"diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsxindex e42c264ab8..86f4de16cc 100644--- a/web/src/pages/agent/constant/pipeline.tsx+++ b/web/src/pages/agent/constant/pipeline.tsx@@ -337,6 +337,7 @@ export const initialTitleChunkerValues = {hierarchyGroup: '0',include_heading_content: false,root_chunk_as_heading: false,+ chunk_token_cap: 512,hierarchyRules: cloneDeep(originalRules),groupRules: cloneDeep(originalRules),};diff --git a/web/src/pages/agent/form/title-chunker-form/hook.ts b/web/src/pages/agent/form/title-chunker-form/hook.tsindex 53323c25c7..e94d5eaa46 100644--- a/web/src/pages/agent/form/title-chunker-form/hook.ts+++ b/web/src/pages/agent/form/title-chunker-form/hook.ts@@ -122,12 +122,23 @@ function transformApiResponseToForm(}}+ let chunk_token_cap = apiData.chunk_token_cap;+ if (chunk_token_cap === undefined || chunk_token_cap === null || chunk_token_cap === '') {+ chunk_token_cap = 512;+ } else {+ chunk_token_cap = Number(chunk_token_cap);+ if (!Number.isFinite(chunk_token_cap)) {+ chunk_token_cap = 512;+ }+ }+return {method,hierarchyHierarchy,hierarchyGroup,include_heading_content: Boolean(apiData.include_heading_content),root_chunk_as_heading: Boolean(apiData.root_chunk_as_heading),+ chunk_token_cap,hierarchyRules,groupRules,};diff --git a/web/src/pages/agent/form/title-chunker-form/index.tsx b/web/src/pages/agent/form/title-chunker-form/index.tsxindex 689ffd7194..e8691dfd65 100644--- a/web/src/pages/agent/form/title-chunker-form/index.tsx+++ b/web/src/pages/agent/form/title-chunker-form/index.tsx@@ -1,6 +1,7 @@import { FormFieldType, RenderField } from '@/components/dynamic-form';import { SelectWithSearch } from '@/components/originui/select-with-search';import { RAGFlowFormItem } from '@/components/ragflow-form';+import { SliderInputFormField } from '@/components/slider-input-form-field';import { BlockButton, Button } from '@/components/ui/button';import { Card, CardContent, CardHeader } from '@/components/ui/card';import { Form } from '@/components/ui/form';@@ -63,6 +64,12 @@ export const FormSchema = z.object({root_chunk_as_heading: z.boolean().optional(),hierarchyRules: rulesSchema,groupRules: rulesSchema,+ chunk_token_cap: z+ .number()+ .int()+ .min(128)+ .max(8000)+ .optional(),});export enum TitleChunkerRulesField {@@ -295,6 +302,16 @@ const TitleChunkerForm = ({</div></div></div>+ <SliderInputFormField+ name="chunk_token_cap"+ max={8000}+ min={128}+ label={t('flow.chunkTokenCap', 'Chunk token cap')}+ tooltip={t(+ 'flow.chunkTokenCapTip',+ 'Maximum tokens per chunk. Chunks exceeding the cap are re-split on sentence boundaries (Chinese/English period, exclamation mark, question mark, newline). 0 disables the cap.',+ )}+ /><RAGFlowFormItemname={'hierarchyHierarchy'}label={''}