Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(title-chunker): configurable token-count cap with sentence-boundary re-split (#18455)

Jackebd6be09c7c2
diff --git a/rag/flow/chunker/_sentence_boundary.py b/rag/flow/chunker/_sentence_boundary.py
new file mode 100644
index 0000000000..50fd0cd5e2
--- /dev/null
+++ b/rag/flow/chunker/_sentence_boundary.py
@@ -0,0 +1,29 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Sentence-boundary definition shared by the title and token chunkers.
+
+Kept dependency-free so both chunkers can reuse the exact same split without
+pulling in heavier parser/tokenizer modules.
+"""
+
+import re
+
+# Boundaries used to re-split an oversized chunk into sentences. Covers
+# Chinese/English period, exclamation mark, question mark, the Chinese
+# variants, and the newline. The capturing group keeps the delimiter attached
+# to the preceding sentence so re-merged text preserves original boundaries.
+SENTENCE_BOUNDARY_PATTERN = r"([。!??;!\n]|\. )"
+
+SENTENCE_BOUNDARY_RE = re.compile(SENTENCE_BOUNDARY_PATTERN, re.DOTALL)
diff --git a/rag/flow/chunker/title_chunker/common.py b/rag/flow/chunker/title_chunker/common.py
index 0848ac0ca7..ffa6b09744 100644
--- a/rag/flow/chunker/title_chunker/common.py
+++ b/rag/flow/chunker/title_chunker/common.py
@@ -3,7 +3,6 @@
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@@ -13,6 +12,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+"""Title-based chunking shared by the hierarchy and group strategies.
+
+Both strategies build raw chunks from upstream line records, then a single
+post-build pass (`BaseTitleChunker._enforce_token_cap`) guarantees no text
+chunk exceeds the configured token ceiling.
+"""
+
+import logging
import random
import re
import sys
@@ -20,9 +27,11 @@ from abc import ABC, abstractmethod
from collections import Counter
from copy import deepcopy
+from common.token_utils import num_tokens_from_string, truncate
from deepdoc.parser.pdf_parser import RAGFlowPdfParser
from deepdoc.parser.utils import extract_pdf_outlines
from rag.flow.base import ProcessBase, ProcessParamBase
+from rag.flow.chunker._sentence_boundary import SENTENCE_BOUNDARY_RE
from rag.flow.parser.pdf_chunk_metadata import (
PDF_POSITIONS_KEY,
extract_pdf_positions,
@@ -32,28 +41,45 @@ from rag.flow.parser.pdf_chunk_metadata import (
)
from rag.nlp import not_bullet, not_title
+logger = logging.getLogger(__name__)
+
BODY_LEVEL = sys.maxsize - 1
class TitleChunkerParam(ProcessParamBase):
+ """Parameters for the title-based chunkers (hierarchy and group)."""
+
def __init__(self):
super().__init__()
self.levels = []
self.hierarchy = None
self.include_heading_content = False
self.root_chunk_as_heading = False
+ # Hard ceiling on each text chunk's token count. A built chunk that
+ # exceeds it is re-split on sentence boundaries into <= cap sub-chunks
+ # (see BaseTitleChunker._enforce_token_cap). 0/None disables the
+ # ceiling.
+ self.chunk_token_cap = 512
def check(self):
+ """Validate the configured parameters before chunking runs."""
if self.method in {"hierarchy", "group"}:
self.check_empty(self.levels, "Hierarchical setups.")
if self.method == "hierarchy":
self.check_empty(self.hierarchy, "Hierarchy number.")
+ if self.chunk_token_cap:
+ self.check_positive_integer(self.chunk_token_cap, "Chunk token cap.")
+ if not (128 <= int(self.chunk_token_cap) <= 8000):
+ raise ValueError("Chunk token cap must be between 128 and 8000.")
def get_input_form(self) -> dict[str, dict]:
+ """Return the canvas-visible parameter form (empty for this chunker)."""
return {}
class BaseTitleChunker(ABC):
+ """Shared base for the hierarchy and group title chunkers."""
+
start_message = "Start to chunk by title."
def __init__(self, process: ProcessBase, from_upstream):
@@ -62,26 +88,145 @@ class BaseTitleChunker(ABC):
self.from_upstream = from_upstream
async def invoke(self):
+ """Run the full chunking pipeline for one upstream payload."""
self.process.set_output("output_format", "chunks")
self.process.callback(random.randint(1, 5) / 100.0, self.start_message)
line_records = self.extract_line_records()
resolved = self.resolve_levels(line_records)
chunks = self.build_chunks(line_records, resolved)
+ chunks = self._enforce_token_cap(chunks)
await self.set_chunks(chunks)
self.process.callback(1, "Done.")
- def extract_line_records(self):
+ @staticmethod
+ def _split_text_by_sentences(text):
+ """Split text on sentence boundaries, keeping each delimiter attached
+ to the preceding sentence so re-merged text preserves punctuation.
+ """
+ if not text:
+ return []
+ raw = re.split(SENTENCE_BOUNDARY_RE, text)
+ sentences = []
+ for i in range(0, len(raw), 2):
+ piece = raw[i]
+ if i + 1 < len(raw):
+ piece += raw[i + 1]
+ if piece:
+ sentences.append(piece)
+ return sentences
+
+ @staticmethod
+ def _hard_split_by_tokens(text, cap):
+ """Hard-split a boundary-less run into <= cap-token pieces.
+
+ Mirrors common.token_utils.truncate semantics (prefix bounded by
+ ``cap`` tokens). Falls back to a character prefix if the tokenizer is
+ unavailable so the ceiling still holds.
+ """
+ out = []
+ rest = text or ""
+ while rest:
+ try:
+ head = truncate(rest, cap)
+ except (ValueError, TypeError, UnicodeError):
+ # Tokenizer unavailable/failed; fall back to a character prefix
+ # so the ceiling still holds instead of raising.
+ head = rest[:cap]
+ if not head:
+ out.append(rest)
+ break
+ out.append(head)
+ if head == rest:
+ break
+ rest = rest[len(head) :]
+ return out
+
+ def _split_text_chunk_by_cap(self, chunk, cap):
+ """Re-split one oversized text chunk into <= cap sub-chunks.
+
+ Sentence boundaries are tried first; any remaining over-cap segment is
+ hard-split. PDF coordinates follow plan A: the original (coarse,
+ page-level) positions attach to the first sub-chunk only.
"""
- Normalize all upstream input payloads into a unified ordered record stream.
- All level resolution and chunk construction logic operates on this standard stream,
- decoupling downstream chunking strategies from different upstream output formats.
+ text = chunk.get("text") or ""
+ sentences = self._split_text_by_sentences(text)
+ if not sentences:
+ return [chunk]
+
+ groups = []
+ current = ""
+ for sentence in sentences:
+ candidate = current + sentence if current else sentence
+ if current and self._token_count(candidate) > cap:
+ groups.append(current)
+ current = sentence
+ else:
+ current = candidate
+ if current:
+ groups.append(current)
+
+ final_groups = []
+ for group in groups:
+ if self._token_count(group) <= cap:
+ final_groups.append(group)
+ else:
+ final_groups.extend(self._hard_split_by_tokens(group, cap))
+ if not final_groups:
+ return [chunk]
+
+ has_positions = PDF_POSITIONS_KEY in chunk
+ orig_positions = chunk.get(PDF_POSITIONS_KEY)
+ out = []
+ for i, group in enumerate(final_groups):
+ sub = dict(chunk)
+ sub["text"] = group
+ # Plan A: the original (coarse, page-level) coordinates attach to the
+ # first sub-chunk only; subsequent sub-chunks carry none.
+ if has_positions:
+ sub[PDF_POSITIONS_KEY] = orig_positions if i == 0 else []
+ out.append(sub)
+ return out
+
+ def _token_count(self, text):
+ """Count tokens for ``text``.
+
+ ``num_tokens_from_string`` returns 0 when the encoder is unavailable;
+ in that case fall back to the character count so the cap is still
+ enforced rather than silently skipped.
"""
- import logging
+ n = num_tokens_from_string(text or "")
+ if n == 0 and text:
+ logger.warning("tokenizer returned 0 tokens for non-empty text; falling back to character count for the token cap")
+ return len(text or "")
+ return n
- logger = logging.getLogger(__name__)
+ def _enforce_token_cap(self, chunks):
+ """Apply the hard token ceiling to every text chunk after build_chunks.
+ Both hierarchy and group methods honour ``chunk_token_cap``. Table and
+ image chunks are atomic and skipped. ``cap`` of 0/None disables it.
+ """
+ cap = self.param.chunk_token_cap
+ if not cap or cap <= 0:
+ return chunks
+
+ out = []
+ for chunk in chunks:
+ if chunk.get("doc_type_kwd", "text") != "text":
+ out.append(chunk)
+ continue
+ if self._token_count(chunk.get("text") or "") <= cap:
+ out.append(chunk)
+ continue
+ out.extend(self._split_text_chunk_by_cap(chunk, cap))
+ return out
+
+ def extract_line_records(self):
+ """Normalize all upstream input payloads into a unified ordered record
+ stream. All level resolution and chunk construction operate on this
+ standard stream, decoupling strategies from upstream output formats.
+ """
payload = None
- # Extract raw content payload based on upstream output format type
if self.from_upstream.output_format == "markdown":
payload = self.from_upstream.markdown_result or ""
elif self.from_upstream.output_format == "text":
@@ -89,24 +234,17 @@ class BaseTitleChunker(ABC):
elif self.from_upstream.output_format == "html":
payload = self.from_upstream.html_result or ""
- # Boundary robustness fix: explicit None check to distinguish `None` and empty string ""
- # Prevents empty payload from unexpectedly falling through to structured chunk branch
if payload is not None:
lines = payload.split("\n")
input_line_count = len(lines)
-
- # Format-branched text processing to preserve original document semantics
- # Plain text: perform full whitespace stripping and invalid empty line filtering
+ # Plain text: full whitespace strip + drop blank lines. Markdown &
+ # HTML: keep original spacing, drop only pure blank lines.
if self.from_upstream.output_format == "text":
clean_lines = [line.strip() for line in lines if line.strip()]
- # Markdown & HTML: retain original indentation/spacing, only filter pure blank lines
else:
clean_lines = [line for line in lines if line.strip()]
-
output_line_count = len(clean_lines)
- # Production observability log: added format dimension per project coding guidelines
logger.info(f"payload filter: format={self.from_upstream.output_format} before={input_line_count} after={output_line_count}")
-
return [{"text": line, "doc_type_kwd": "text", "img_id": None, "layout": "", PDF_POSITIONS_KEY: []} for line in clean_lines]
items = self.from_upstream.chunks if self.from_upstream.output_format == "chunks" else self.from_upstream.json_result
return [
@@ -121,6 +259,7 @@ class BaseTitleChunker(ABC):
]
def extract_outlines(self):
+ """Extract PDF bookmarks/outlines used for outline-based leveling."""
file = self.from_upstream.file or {}
source = file.get("blob") or file.get("binary") or file.get("path") or file.get("name")
if not source:
@@ -129,6 +268,7 @@ class BaseTitleChunker(ABC):
@staticmethod
def match_regex_level(text, level_group):
+ """Return the 1-based level whose regex matches ``text``, else None."""
stripped = text.strip()
for level, pattern in enumerate(level_group, start=1):
if re.match(pattern, stripped) and not not_bullet(stripped):
@@ -137,12 +277,13 @@ class BaseTitleChunker(ABC):
@staticmethod
def select_level_group(lines, raw_levels):
+ """Pick the single regex family that best matches ``lines``.
+
+ Mixing families would make level numbers ambiguous and break downstream
+ comparisons, so only the most frequently matching family is kept.
+ """
if not raw_levels:
return []
-
- # Select one regex family before assigning numeric levels. Mixing
- # patterns across families would make the level numbers ambiguous and
- # break downstream comparisons.
hits = [0] * len(raw_levels)
for i, group in enumerate(raw_levels):
for sec in lines:
@@ -153,7 +294,6 @@ class BaseTitleChunker(ABC):
if re.match(pattern, sec) and not not_bullet(sec):
hits[i] += 1
break
-
maximum = 0
selected = -1
for i, hit in enumerate(hits):
@@ -161,28 +301,29 @@ class BaseTitleChunker(ABC):
continue
selected = i
maximum = hit
-
if selected < 0:
return []
return [pattern for pattern in raw_levels[selected] if pattern]
@staticmethod
def match_layout_level(text, layout, fallback_level):
- if re.search(r"(section|title|head)", layout, re.I) and not not_title(text.split("@")[0].strip()):
+ """Treat layout-tagged title-like lines as ``fallback_level``."""
+ if re.search(r"(section|title|head)", layout, re.IGNORECASE) and not not_title(text.split("@")[0].strip()):
return fallback_level
return BODY_LEVEL
@staticmethod
def _outline_similarity(left, right):
+ """Jaccard similarity of adjacent outline-text bigrams."""
left_pairs = {left[i] + left[i + 1] for i in range(len(left) - 1)}
right_pairs = {right[i] + right[i + 1] for i in range(min(len(left), len(right) - 1))}
return len(left_pairs & right_pairs) / max(len(left_pairs), len(right_pairs), 1)
def resolve_outline_levels(self, line_records):
+ """Resolve levels from PDF outlines when they cover enough of the doc."""
outlines = self.extract_outlines()
if not line_records or len(outlines) / len(line_records) <= 0.03:
return None
-
max_level = max(level for _, level, _ in outlines) + 1
levels = []
for record in line_records:
@@ -196,7 +337,6 @@ class BaseTitleChunker(ABC):
break
else:
levels.append(BODY_LEVEL)
-
return {
"levels": levels,
"most_level": max(1, max_level - 1),
@@ -204,6 +344,7 @@ class BaseTitleChunker(ABC):
}
def resolve_frequency_levels(self, line_records):
+ """Resolve levels by regex family and layout tagging."""
level_group = self.select_level_group(
[record["text"] for record in line_records],
self.param.levels,
@@ -225,13 +366,11 @@ class BaseTitleChunker(ABC):
fallback_level,
)
)
-
most_level = None
for level, _ in Counter(levels).most_common():
if level < BODY_LEVEL:
most_level = level
break
-
return {
"levels": levels,
"most_level": most_level,
@@ -239,13 +378,15 @@ class BaseTitleChunker(ABC):
}
def resolve_title_levels(self, line_records):
+ """Resolve levels, preferring outlines then falling back to frequency."""
return self.resolve_outline_levels(line_records) or self.resolve_frequency_levels(line_records)
def build_chunks_from_record_groups(self, record_groups):
- # Strategy code decides record grouping. This method materializes each
- # group into the output chunk representation. For PDF-like inputs, the
- # chunk box is defined by merged source positions and the text payload
- # is normalized by removing parser tags.
+ """Materialize each record group into the output chunk representation.
+
+ For PDF-like inputs the chunk box is defined by merged source positions
+ and the text payload is normalized by removing parser tags.
+ """
if self.from_upstream.output_format in ["markdown", "text", "html"]:
chunks = [{"text": "".join(record["text"] + "\n" for record in records)} for records in record_groups if records]
else:
@@ -280,6 +421,7 @@ class BaseTitleChunker(ABC):
return chunks
async def set_chunks(self, chunks):
+ """Finalize and emit chunks, enriching PDF positions when needed."""
if self.from_upstream.output_format in ["markdown", "text", "html"]:
self.process.set_output("chunks", chunks)
return
@@ -291,14 +433,17 @@ class BaseTitleChunker(ABC):
@abstractmethod
def resolve_levels(self, line_records):
+ """Resolve title levels for the concrete chunker strategy."""
raise NotImplementedError()
@abstractmethod
def build_chunks(self, line_records, resolved):
+ """Build raw chunks from records and resolved levels."""
raise NotImplementedError()
def resolve_target_level(levels, hierarchy):
+ """Pick the title level used as the chunking target for ``hierarchy``."""
title_levels = sorted({level for level in levels if 0 < level < BODY_LEVEL})
if not title_levels:
return None
diff --git a/rag/flow/chunker/title_chunker/group_chunker.py b/rag/flow/chunker/title_chunker/group_chunker.py
index bb4d5e9777..2026565fcb 100644
--- a/rag/flow/chunker/title_chunker/group_chunker.py
+++ b/rag/flow/chunker/title_chunker/group_chunker.py
@@ -20,10 +20,11 @@ from rag.flow.chunker.title_chunker.common import (
)
MIN_GROUP_TOKENS = 32
-MAX_GROUP_TOKENS = 1024
def _build_section_ids(levels, target_level):
+ """Assign a stable section id that increments whenever a title at or above
+ ``target_level`` starts a new section."""
sec_ids = []
sid = 0
for i, level in enumerate(levels):
@@ -34,18 +35,30 @@ def _build_section_ids(levels, target_level):
def _resolve_group_target_level(levels, hierarchy, most_level):
+ """Pick the level used as the grouping target for the group method."""
if hierarchy and int(hierarchy) > 0:
return resolve_target_level(levels, hierarchy)
return most_level
class GroupTitleChunker(BaseTitleChunker):
+ """Group consecutive records under the same title into one chunk."""
+
start_message = "Start to group by title levels."
def resolve_levels(self, line_records):
+ """Resolve title levels via the shared outline/frequency strategy."""
return self.resolve_title_levels(line_records)
def build_chunks(self, line_records, resolved):
+ """Build chunks by merging records inside the same logical section.
+
+ The merge ceiling uses the configurable ``chunk_token_cap`` (0/None
+ means "no ceiling"). The post-build ``_enforce_token_cap`` in
+ BaseTitleChunker is the single hard guarantee, so any residual over-cap
+ chunk (e.g. a single record bigger than the cap) is still re-split
+ there.
+ """
target_level = _resolve_group_target_level(
resolved["levels"],
self.param.hierarchy,
@@ -56,6 +69,8 @@ class GroupTitleChunker(BaseTitleChunker):
tk_cnt = 0
last_sid = -2
+ cap = self.param.chunk_token_cap or 0
+
# The merge state is driven by (current section id, current token size).
# A chunk stays open while records remain in the same logical section,
# except that very small chunks are allowed to absorb the next record
@@ -72,7 +87,8 @@ class GroupTitleChunker(BaseTitleChunker):
continue
token_count = num_tokens_from_string(text)
- should_merge = record_groups and record_groups[-1][0]["doc_type_kwd"] == "text" and (tk_cnt < MIN_GROUP_TOKENS or (tk_cnt < MAX_GROUP_TOKENS and sec_id == last_sid))
+ merge_ceiling_ok = cap <= 0 or tk_cnt < cap
+ should_merge = record_groups and record_groups[-1][0]["doc_type_kwd"] == "text" and (tk_cnt < MIN_GROUP_TOKENS or (merge_ceiling_ok and sec_id == last_sid))
if should_merge:
record_groups[-1].append(record)
diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py
index cc6e296798..2038817053 100644
--- a/rag/flow/chunker/token_chunker.py
+++ b/rag/flow/chunker/token_chunker.py
@@ -19,6 +19,7 @@ from copy import deepcopy
from common.float_utils import normalize_overlapped_percent
from common.token_utils import num_tokens_from_string
from rag.flow.base import ProcessBase, ProcessParamBase
+from rag.flow.chunker._sentence_boundary import SENTENCE_BOUNDARY_PATTERN
from rag.flow.chunker.schema import TokenChunkerFromUpstream
from rag.flow.parser.pdf_chunk_metadata import (
PDF_POSITIONS_KEY,
@@ -173,8 +174,7 @@ def _build_json_chunks(json_result, delimiter_pattern):
def _take_sentences(text, need_tokens, from_end=False):
# Take text from one side until the target token budget is reached.
- split_pat = r"([。!??;!\n]|\. )"
- texts = re.split(split_pat, text or "", flags=re.DOTALL)
+ texts = re.split(SENTENCE_BOUNDARY_PATTERN, text or "", flags=re.DOTALL)
sentences = []
for i in range(0, len(texts), 2):
sentences.append(texts[i] + (texts[i + 1] if i + 1 < len(texts) else ""))
diff --git a/rag/flow/tests/test_title_chunker_position_int.py b/rag/flow/tests/test_title_chunker_position_int.py
index 2edd07224f..5ea8827c8c 100644
--- a/rag/flow/tests/test_title_chunker_position_int.py
+++ b/rag/flow/tests/test_title_chunker_position_int.py
@@ -60,6 +60,7 @@ def _load_title_chunker_with_stubs():
common_token_utils = types.ModuleType("common.token_utils")
common_token_utils.num_tokens_from_string = lambda text: 1
+ common_token_utils.truncate = lambda text, max_len: (text or "")[:max_len]
_install("common.token_utils", common_token_utils)
rag_nlp = types.ModuleType("rag.nlp")
@@ -247,6 +248,7 @@ def test_title_chunker_preserves_position_int_from_deepdoc_json():
param.levels = [] # no headings -> all body -> single merged chunk
param.include_heading_content = False
param.root_chunk_as_heading = False
+ param.chunk_token_cap = 0 # keep this a pure position regression (no re-split)
process = common_module.ProcessBase(None, "title_chunker", param)
process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")
diff --git a/rag/flow/tests/test_title_chunker_token_cap.py b/rag/flow/tests/test_title_chunker_token_cap.py
new file mode 100644
index 0000000000..08c9e9516e
--- /dev/null
+++ b/rag/flow/tests/test_title_chunker_token_cap.py
@@ -0,0 +1,501 @@
+import asyncio
+import importlib
+import importlib.util
+import sys
+import types
+from contextlib import contextmanager
+from pathlib import Path
+
+"""TDD tests for the TitleChunker token-count CAP (chunk_token_cap).
+
+The TitleChunker (both ``hierarchy`` and ``group`` methods) historically had
+no token-size ceiling: a long section with no sub-heading became one giant
+chunk. This suite pins the new behaviour:
+
+ * A configurable ``chunk_token_cap`` (default 512, valid 128..8000) is a
+ hard ceiling on every *text* chunk's token count.
+ * When a built chunk exceeds the cap, it is re-split on sentence boundaries
+ (``。!??;!\\n`` plus the English ``. ``) and the pieces are greedily
+ merged into <= cap sub-chunks.
+ * A single boundary-less run that still exceeds the cap is hard-split so the
+ ceiling always holds (fallback only).
+ * Table/image chunks are atomic and never split.
+ * ``chunk_token_cap == 0`` disables the ceiling.
+
+Token counting is faked as 1 token == 1 character so the assertions are fully
+deterministic (mirrors the stub approach in test_title_chunker_position_int.py,
+which also fakes ``num_tokens_from_string``).
+"""
+
+
+@contextmanager
+def _load_title_chunker_with_stubs():
+ root = Path(__file__).resolve().parents[3]
+ original_modules = {}
+
+ def _install(name: str, module: types.ModuleType):
+ original_modules.setdefault(name, sys.modules.get(name))
+ sys.modules[name] = module
+
+ try:
+ rag_pkg = types.ModuleType("rag")
+ rag_pkg.__path__ = [str(root / "rag")]
+ _install("rag", rag_pkg)
+
+ rag_flow_pkg = types.ModuleType("rag.flow")
+ rag_flow_pkg.__package__ = "rag"
+ rag_flow_pkg.__path__ = [str(root / "rag" / "flow")]
+ _install("rag.flow", rag_flow_pkg)
+
+ rag_flow_chunker_pkg = types.ModuleType("rag.flow.chunker")
+ rag_flow_chunker_pkg.__package__ = "rag.flow"
+ rag_flow_chunker_pkg.__path__ = [str(root / "rag" / "flow" / "chunker")]
+ _install("rag.flow.chunker", rag_flow_chunker_pkg)
+
+ rag_flow_parser_pkg = types.ModuleType("rag.flow.parser")
+ rag_flow_parser_pkg.__package__ = "rag.flow"
+ rag_flow_parser_pkg.__path__ = [str(root / "rag" / "flow" / "parser")]
+ _install("rag.flow.parser", rag_flow_parser_pkg)
+
+ common_pkg = types.ModuleType("common")
+ common_pkg.__path__ = [str(root / "common")]
+ _install("common", common_pkg)
+
+ common_float_utils = types.ModuleType("common.float_utils")
+ common_float_utils.normalize_overlapped_percent = lambda value: value
+ _install("common.float_utils", common_float_utils)
+
+ # Deterministic tokenizer: 1 token per character. ``truncate`` mirrors
+ # token_utils.truncate semantics (prefix bounded by max_len tokens).
+ common_token_utils = types.ModuleType("common.token_utils")
+ common_token_utils.num_tokens_from_string = lambda text: len(text or "")
+ common_token_utils.truncate = lambda text, max_len: (text or "")[:max_len]
+ _install("common.token_utils", common_token_utils)
+
+ rag_nlp = types.ModuleType("rag.nlp")
+ rag_nlp.naive_merge = lambda *args, **kwargs: []
+ rag_nlp.not_bullet = lambda text: False
+ rag_nlp.not_title = lambda text: True
+ _install("rag.nlp", rag_nlp)
+
+ deepdoc_pkg = types.ModuleType("deepdoc")
+ deepdoc_pkg.__path__ = [str(root / "deepdoc")]
+ _install("deepdoc", deepdoc_pkg)
+
+ deepdoc_parser_pkg = types.ModuleType("deepdoc.parser")
+ deepdoc_parser_pkg.__path__ = [str(root / "deepdoc" / "parser")]
+ _install("deepdoc.parser", deepdoc_parser_pkg)
+
+ class _RAGFlowPdfParser:
+ @staticmethod
+ def remove_tag(text):
+ return text
+
+ @staticmethod
+ def extract_positions(tag):
+ return []
+
+ deepdoc_pdf_parser = types.ModuleType("deepdoc.parser.pdf_parser")
+ deepdoc_pdf_parser.RAGFlowPdfParser = _RAGFlowPdfParser
+ _install("deepdoc.parser.pdf_parser", deepdoc_pdf_parser)
+
+ deepdoc_parser_utils = types.ModuleType("deepdoc.parser.utils")
+ deepdoc_parser_utils.extract_pdf_outlines = lambda *args, **kwargs: []
+ _install("deepdoc.parser.utils", deepdoc_parser_utils)
+
+ class ProcessParamBase:
+ def __init__(self):
+ pass
+
+ def check_valid_value(self, value, msg, allowed):
+ if value not in allowed:
+ raise ValueError(msg)
+
+ def check_positive_integer(self, value, msg):
+ if not (isinstance(value, int) and value > 0):
+ raise ValueError(msg)
+
+ def check_decimal_float(self, value, msg):
+ pass
+
+ def check_nonnegative_number(self, value, msg):
+ pass
+
+ def check_empty(self, value, msg):
+ pass
+
+ class ProcessBase:
+ def __init__(self, _pipeline, _id, param):
+ self._pipeline = _pipeline
+ self._id = _id
+ self._param = param
+ self._outputs = {}
+ self.callback = lambda *_args, **_kwargs: None
+
+ def set_output(self, key, value):
+ self._outputs[key] = value
+
+ rag_flow_base = types.ModuleType("rag.flow.base")
+ rag_flow_base.ProcessBase = ProcessBase
+ rag_flow_base.ProcessParamBase = ProcessParamBase
+ _install("rag.flow.base", rag_flow_base)
+
+ pdf_chunk_metadata = types.ModuleType("rag.flow.parser.pdf_chunk_metadata")
+ pdf_chunk_metadata.PDF_POSITIONS_KEY = "pdf_positions"
+ pdf_chunk_metadata.extract_pdf_positions = lambda _item: []
+ pdf_chunk_metadata.merge_pdf_positions = lambda _records: []
+ pdf_chunk_metadata.finalize_pdf_chunk = lambda chunk: chunk
+
+ async def _noop_restore(*_a, **_k):
+ return None
+
+ pdf_chunk_metadata.restore_pdf_text_previews = _noop_restore
+ _install("rag.flow.parser.pdf_chunk_metadata", pdf_chunk_metadata)
+
+ common_spec = importlib.util.spec_from_file_location(
+ "rag.flow.chunker.title_chunker.common",
+ root / "rag" / "flow" / "chunker" / "title_chunker" / "common.py",
+ )
+ common_module = importlib.util.module_from_spec(common_spec)
+ _install("rag.flow.chunker.title_chunker.common", common_module)
+ common_spec.loader.exec_module(common_module)
+
+ hierarchy_spec = importlib.util.spec_from_file_location(
+ "rag.flow.chunker.title_chunker.hierarchy_chunker",
+ root / "rag" / "flow" / "chunker" / "title_chunker" / "hierarchy_chunker.py",
+ )
+ hierarchy_module = importlib.util.module_from_spec(hierarchy_spec)
+ _install("rag.flow.chunker.title_chunker.hierarchy_chunker", hierarchy_module)
+ hierarchy_spec.loader.exec_module(hierarchy_module)
+
+ group_spec = importlib.util.spec_from_file_location(
+ "rag.flow.chunker.title_chunker.group_chunker",
+ root / "rag" / "flow" / "chunker" / "title_chunker" / "group_chunker.py",
+ )
+ group_module = importlib.util.module_from_spec(group_spec)
+ _install("rag.flow.chunker.title_chunker.group_chunker", group_module)
+ group_spec.loader.exec_module(group_module)
+
+ yield common_module, hierarchy_module, group_module
+ finally:
+ for module_name, original in original_modules.items():
+ if original is None:
+ sys.modules.pop(module_name, None)
+ else:
+ sys.modules[module_name] = original
+
+
+def _real_extract_pdf_positions(item):
+ if not isinstance(item, dict):
+ return []
+ positions = item.get("pdf_positions")
+ if isinstance(positions, list):
+ return [list(p) for p in positions]
+ positions = item.get("positions")
+ if isinstance(positions, list):
+ return [list(p) for p in positions]
+ return []
+
+
+def _real_merge_pdf_positions(records):
+ merged = []
+ for rec in records or []:
+ if not isinstance(rec, dict):
+ continue
+ for pos in rec.get("pdf_positions") or []:
+ if isinstance(pos, (list, tuple)) and len(pos) >= 5:
+ merged.append([pos[0], pos[1], pos[2], pos[3], pos[4]])
+ seen = set()
+ out = []
+ for pos in merged:
+ key = tuple(pos[:5])
+ if key not in seen:
+ seen.add(key)
+ out.append(pos)
+ return out
+
+
+def _real_finalize_pdf_chunk(chunk):
+ positions = _real_extract_pdf_positions(chunk)
+ if positions:
+ chunk["position_int"] = [list(p) for p in positions]
+ chunk.pop("pdf_positions", None)
+ return chunk
+
+
+def _run(method, from_upstream, **param_kwargs):
+ with _load_title_chunker_with_stubs() as (common_module, hierarchy_module, group_module):
+ if param_kwargs.pop("faithful_positions", False):
+ common_module.extract_pdf_positions = _real_extract_pdf_positions
+ common_module.merge_pdf_positions = _real_merge_pdf_positions
+ common_module.finalize_pdf_chunk = _real_finalize_pdf_chunk
+
+ param = common_module.TitleChunkerParam()
+ param.method = method
+ param.chunk_token_cap = param_kwargs.pop("chunk_token_cap", 512)
+ for key, value in param_kwargs.items():
+ setattr(param, key, value)
+
+ process = common_module.ProcessBase(None, "title_chunker", param)
+ process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")
+ process._outputs = {}
+
+ if method == "hierarchy":
+ chunker = hierarchy_module.HierarchyTitleChunker(process, from_upstream)
+ else:
+ chunker = group_module.GroupTitleChunker(process, from_upstream)
+ asyncio.run(chunker.invoke())
+ return process._outputs.get("chunks", [])
+
+
+def _json_upstream(items, output_format="json"):
+ return types.SimpleNamespace(
+ output_format=output_format,
+ json_result=items if output_format == "json" else None,
+ markdown_result=None,
+ text_result=None,
+ html_result=None,
+ chunks=None if output_format == "json" else items,
+ file=None,
+ name="cap-test",
+ )
+
+
+def _char_count(text):
+ return len(text or "")
+
+
+# --------------------------------------------------------------------------- #
+# 1. hierarchy: a single oversized chunk is re-split and stays within the cap #
+# --------------------------------------------------------------------------- #
+def test_hierarchy_oversized_chunk_respects_cap():
+ # 12 sentences of 4 chars each ("Saa。") -> 48 tokens, cap 20 -> >=3 chunks.
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)
+
+ assert len(chunks) > 1, "expected the oversized chunk to be split"
+ for ck in chunks:
+ assert ck.get("doc_type_kwd", "text") == "text"
+ assert _char_count(ck["text"]) <= 20, f"chunk exceeds cap: {ck['text']!r}"
+ # Content is preserved (single record -> build_chunks appends one trailing
+ # newline, which the sentence split keeps as a boundary; rstrip for compare).
+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 2. group: same guarantee, different code path #
+# --------------------------------------------------------------------------- #
+def test_group_oversized_chunk_respects_cap():
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("group", _json_upstream(items), levels=[], hierarchy=0, chunk_token_cap=20)
+
+ assert len(chunks) > 1
+ for ck in chunks:
+ assert _char_count(ck["text"]) <= 20
+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 3. split happens on sentence boundaries, not mid-sentence #
+# --------------------------------------------------------------------------- #
+def test_split_keeps_sentence_boundaries():
+ # Every sentence ends with "。"; after greedy merge each chunk must still
+ # end on a boundary (no mid-sentence cut while boundaries are available).
+ body = "。".join(f"Sentence number {i} ends here" for i in range(10)) + "。"
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=40)
+
+ assert len(chunks) > 1
+ for ck in chunks:
+ assert ck["text"].rstrip("\n").endswith("。"), f"chunk cut mid-sentence: {ck['text']!r}"
+
+
+# --------------------------------------------------------------------------- #
+# 4. chunk_token_cap == 0 disables the ceiling #
+# --------------------------------------------------------------------------- #
+def test_cap_zero_disables_splitting():
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=0)
+
+ assert len(chunks) == 1, "cap=0 must keep the single-chunk behaviour"
+ assert chunks[0]["text"].rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 5. table/image chunks are atomic and never split #
+# --------------------------------------------------------------------------- #
+def test_non_text_chunk_is_atomic():
+ big_table = "x" * 200 # 200 tokens, far above any reasonable cap
+ items = [{"text": big_table, "doc_type_kwd": "table"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)
+
+ assert len(chunks) == 1
+ assert chunks[0]["doc_type_kwd"] == "table"
+ assert chunks[0]["text"] == big_table
+
+
+# --------------------------------------------------------------------------- #
+# 6. pathological single run with no boundary is hard-split (fallback) #
+# --------------------------------------------------------------------------- #
+def test_boundaryless_run_is_hard_split():
+ # 100 chars, no punctuation -> sentence split yields one giant segment that
+ # must be hard-split so the cap still holds.
+ body = "x" * 100
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)
+
+ assert len(chunks) > 1
+ for ck in chunks:
+ assert _char_count(ck["text"]) <= 20
+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 7. PDF positions: plan A - first sub-chunk keeps coordinates, rest empty #
+# --------------------------------------------------------------------------- #
+def test_position_plan_a_first_subchunk_keeps_coordinates():
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ items = [{"text": body, "doc_type_kwd": "text", "positions": [[1, 10, 200, 50, 80]]}]
+ chunks = _run(
+ "hierarchy",
+ _json_upstream(items),
+ levels=[],
+ chunk_token_cap=20,
+ faithful_positions=True,
+ )
+
+ assert len(chunks) > 1
+ assert chunks[0].get("position_int") == [[1, 10, 200, 50, 80]], "first sub-chunk must keep coordinates"
+ for ck in chunks[1:]:
+ assert "position_int" not in ck, "only the first sub-chunk should carry coordinates"
+
+
+# --------------------------------------------------------------------------- #
+# 7b. text output format (no doc_type_kwd) also honours the cap #
+# --------------------------------------------------------------------------- #
+def test_hierarchy_text_format_respects_cap():
+ body = "。".join(f"S{i:02d}" for i in range(12)) + "。"
+ from_upstream = types.SimpleNamespace(
+ output_format="text",
+ json_result=None,
+ markdown_result=None,
+ text_result=body,
+ html_result=None,
+ chunks=None,
+ file=None,
+ name="cap-test",
+ )
+ chunks = _run("hierarchy", from_upstream, levels=[], chunk_token_cap=20)
+
+ assert len(chunks) > 1
+ for ck in chunks:
+ assert _char_count(ck["text"]) <= 20
+ assert "".join(ck["text"] for ck in chunks).rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 7c. multiple records merged into one oversized chunk are re-split #
+# --------------------------------------------------------------------------- #
+def test_hierarchy_multi_record_merged_respects_cap():
+ items = [{"text": f"S{i:02d}。", "doc_type_kwd": "text"} for i in range(6)]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=10)
+
+ assert len(chunks) > 1
+ for ck in chunks:
+ assert _char_count(ck["text"]) <= 10
+ expected_built = "".join(f"S{i:02d}。" + "\n" for i in range(6))
+ assert "".join(ck["text"] for ck in chunks) == expected_built
+
+
+# --------------------------------------------------------------------------- #
+# 7d. a chunk already within the cap is left untouched (no spurious split) #
+# --------------------------------------------------------------------------- #
+def test_chunk_within_cap_not_split():
+ body = "S00。S01。"
+ items = [{"text": body, "doc_type_kwd": "text"}]
+ chunks = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=512)
+
+ assert len(chunks) == 1
+ assert chunks[0]["text"].rstrip("\n") == body
+
+
+# --------------------------------------------------------------------------- #
+# 8. range validation in TitleChunkerParam.check #
+# --------------------------------------------------------------------------- #
+def test_chunk_token_cap_range_validation():
+ with _load_title_chunker_with_stubs() as (common_module, _h, _g):
+ param = common_module.TitleChunkerParam()
+ param.method = "hierarchy"
+ param.levels = []
+
+ param.chunk_token_cap = 50
+ try:
+ param.check()
+ raise AssertionError("expected ValueError for cap < 128")
+ except ValueError:
+ pass
+
+ param.chunk_token_cap = 9000
+ try:
+ param.check()
+ raise AssertionError("expected ValueError for cap > 8000")
+ except ValueError:
+ pass
+
+ # Valid defaults must not raise.
+ param.chunk_token_cap = 512
+ param.check()
+ param.chunk_token_cap = 128
+ param.check()
+ param.chunk_token_cap = 8000
+ param.check()
+
+
+# --------------------------------------------------------------------------- #
+# 9. hard split is lossless: sub-chunks concatenate back to the original text #
+# --------------------------------------------------------------------------- #
+def test_hard_split_is_lossless():
+ # A boundary-less run is hard-split; concatenating every sub-chunk must
+ # reproduce the original built chunk text exactly (no characters dropped
+ # or duplicated across the token-prefix truncation).
+ body = "x" * 100
+ items = [{"text": body, "doc_type_kwd": "text"}]
+
+ disabled = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=0)
+ original = disabled[0]["text"]
+
+ capped = _run("hierarchy", _json_upstream(items), levels=[], chunk_token_cap=20)
+ assert len(capped) > 1, "expected the oversized chunk to be hard-split"
+ assert "".join(ck["text"] for ck in capped) == original
+
+
+# --------------------------------------------------------------------------- #
+# 10. tokenizer failure still enforces the cap via character-count fallback #
+# --------------------------------------------------------------------------- #
+def test_tokenizer_failure_falls_back_to_char_count():
+ # Simulate an unavailable tokenizer: num_tokens_from_string returns 0 for
+ # every text. The cap must still apply (via the character-count fallback)
+ # instead of silently leaving a giant chunk intact.
+ body = "x" * 100
+ items = [{"text": body, "doc_type_kwd": "text"}]
+
+ with _load_title_chunker_with_stubs() as (common_module, hierarchy_module, _group_module):
+ # Override the stubbed tokenizer to always report 0 tokens.
+ common_module.num_tokens_from_string = lambda text: 0
+
+ param = common_module.TitleChunkerParam()
+ param.method = "hierarchy"
+ param.chunk_token_cap = 20
+ process = common_module.ProcessBase(None, "title_chunker", param)
+ process._canvas = types.SimpleNamespace(_doc_id="doc", _tenant_id="tenant")
+ process._outputs = {}
+
+ chunker = hierarchy_module.HierarchyTitleChunker(process, _json_upstream(items))
+ asyncio.run(chunker.invoke())
+ chunks = process._outputs.get("chunks", [])
+
+ assert len(chunks) > 1, "cap must still apply when the tokenizer reports 0"
+ assert "".join(ck["text"] for ck in chunks) == "x" * 100 + "\n"
diff --git a/web/src/pages/agent/constant/pipeline.tsx b/web/src/pages/agent/constant/pipeline.tsx
index e42c264ab8..86f4de16cc 100644
--- a/web/src/pages/agent/constant/pipeline.tsx
+++ b/web/src/pages/agent/constant/pipeline.tsx
@@ -337,6 +337,7 @@ export const initialTitleChunkerValues = {
hierarchyGroup: '0',
include_heading_content: false,
root_chunk_as_heading: false,
+ chunk_token_cap: 512,
hierarchyRules: cloneDeep(originalRules),
groupRules: cloneDeep(originalRules),
};
diff --git a/web/src/pages/agent/form/title-chunker-form/hook.ts b/web/src/pages/agent/form/title-chunker-form/hook.ts
index 53323c25c7..e94d5eaa46 100644
--- a/web/src/pages/agent/form/title-chunker-form/hook.ts
+++ b/web/src/pages/agent/form/title-chunker-form/hook.ts
@@ -122,12 +122,23 @@ function transformApiResponseToForm(
}
}
+ let chunk_token_cap = apiData.chunk_token_cap;
+ if (chunk_token_cap === undefined || chunk_token_cap === null || chunk_token_cap === '') {
+ chunk_token_cap = 512;
+ } else {
+ chunk_token_cap = Number(chunk_token_cap);
+ if (!Number.isFinite(chunk_token_cap)) {
+ chunk_token_cap = 512;
+ }
+ }
+
return {
method,
hierarchyHierarchy,
hierarchyGroup,
include_heading_content: Boolean(apiData.include_heading_content),
root_chunk_as_heading: Boolean(apiData.root_chunk_as_heading),
+ chunk_token_cap,
hierarchyRules,
groupRules,
};
diff --git a/web/src/pages/agent/form/title-chunker-form/index.tsx b/web/src/pages/agent/form/title-chunker-form/index.tsx
index 689ffd7194..e8691dfd65 100644
--- a/web/src/pages/agent/form/title-chunker-form/index.tsx
+++ b/web/src/pages/agent/form/title-chunker-form/index.tsx
@@ -1,6 +1,7 @@
import { FormFieldType, RenderField } from '@/components/dynamic-form';
import { SelectWithSearch } from '@/components/originui/select-with-search';
import { RAGFlowFormItem } from '@/components/ragflow-form';
+import { SliderInputFormField } from '@/components/slider-input-form-field';
import { BlockButton, Button } from '@/components/ui/button';
import { Card, CardContent, CardHeader } from '@/components/ui/card';
import { Form } from '@/components/ui/form';
@@ -63,6 +64,12 @@ export const FormSchema = z.object({
root_chunk_as_heading: z.boolean().optional(),
hierarchyRules: rulesSchema,
groupRules: rulesSchema,
+ chunk_token_cap: z
+ .number()
+ .int()
+ .min(128)
+ .max(8000)
+ .optional(),
});
export enum TitleChunkerRulesField {
@@ -295,6 +302,16 @@ const TitleChunkerForm = ({
</div>
</div>
</div>
+ <SliderInputFormField
+ name="chunk_token_cap"
+ max={8000}
+ min={128}
+ label={t('flow.chunkTokenCap', 'Chunk token cap')}
+ tooltip={t(
+ 'flow.chunkTokenCapTip',
+ 'Maximum tokens per chunk. Chunks exceeding the cap are re-split on sentence boundaries (Chinese/English period, exclamation mark, question mark, newline). 0 disables the cap.',
+ )}
+ />
<RAGFlowFormItem
name={'hierarchyHierarchy'}
label={''}