Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix: align pipeline delimiter chunking (#17723)

buua4369ccb23e6612b
diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.py
index 9e51ffc961..f6414bfc97 100644
--- a/deepdoc/parser/txt_parser.py
+++ b/deepdoc/parser/txt_parser.py
@@ -19,7 +19,6 @@ import re
from common.token_utils import num_tokens_from_string
from deepdoc.parser.utils import get_text
-from rag.nlp import _split_oversized_unit
from rag.nlp.delim import (
compile_delimiter_pattern,
normalize_text_newlines,
@@ -28,12 +27,12 @@ from rag.nlp.delim import (
class RAGFlowTxtParser:
- def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?"):
+ def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):
txt = get_text(fnm, binary)
- return self.parser_txt(txt, chunk_token_num, delimiter)
+ return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)
@classmethod
- def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?"):
+ def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):
if not isinstance(txt, str):
raise TypeError("txt type should be str!")
cks = [""]
@@ -42,21 +41,15 @@ class RAGFlowTxtParser:
def add_chunk(t):
nonlocal cks, tk_nums
tnum = num_tokens_from_string(t)
-
- if cks[-1] == "":
- cks[-1] = t
- tk_nums[-1] = tnum
- return
-
- merged = cks[-1] + "\n" + t
- merged_tnum = num_tokens_from_string(merged)
- if merged_tnum <= chunk_token_num:
- cks[-1] = merged
- tk_nums[-1] = merged_tnum
- return
-
- cks.append(t)
- tk_nums.append(tnum)
+ if tk_nums[-1] > chunk_token_num:
+ cks.append(t)
+ tk_nums.append(tnum)
+ else:
+ if cks[-1]:
+ cks[-1] += "\n" + t
+ else:
+ cks[-1] += t
+ tk_nums[-1] += tnum
txt = normalize_text_newlines(txt)
parsed_dels = parse_delimiter_field(delimiter)
@@ -67,18 +60,14 @@ class RAGFlowTxtParser:
bool(dels),
)
secs = re.split(r"(%s)" % dels, txt) if dels else [txt]
- for sec in secs:
+ for index, sec in enumerate(secs):
if dels and re.match(f"^{dels}$", sec):
continue
if not sec:
continue
- if num_tokens_from_string(sec) <= chunk_token_num:
- add_chunk(sec)
- continue
- pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)
- logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))
- for piece in pieces:
- add_chunk(piece)
+ if keep_delimiters and index + 1 < len(secs) and re.match(f"^{dels}$", secs[index + 1]):
+ sec += secs[index + 1]
+ add_chunk(sec)
logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)
return [[c, ""] for c in cks]
diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.py
index b7a75c5c3a..ab9063f43c 100644
--- a/rag/flow/chunker/token_chunker.py
+++ b/rag/flow/chunker/token_chunker.py
@@ -77,7 +77,7 @@ def _compile_delimiter_pattern(delimiters):
def _split_text_by_pattern(text, pattern):
- # Split text by the compiled delimiter pattern and keep delimiter text in each chunk.
+ # Split text by the compiled delimiter pattern and discard delimiters.
if not pattern:
return [text or ""]
@@ -85,10 +85,6 @@ def _split_text_by_pattern(text, pattern):
chunks = []
for i in range(0, len(split_texts), 2):
chunk = split_texts[i]
- if not chunk:
- continue
- if i + 1 < len(split_texts):
- chunk += split_texts[i + 1]
if chunk.strip():
chunks.append(chunk)
return chunks
@@ -317,16 +313,17 @@ class TokenChunker(ProcessBase):
self.set_output("chunks", [{"text": payload}] if payload.strip() else [])
self.callback(1, "Done.")
return
- cks = (
- _split_text_by_pattern(payload, delimiter_pattern)
- if delimiter_pattern
- else naive_merge(
+ if self._param.delimiter_mode == "delimiter":
+ cks = _split_text_by_pattern(payload, delimiter_pattern)
+ elif delimiter_pattern:
+ cks = _split_text_by_pattern(payload, delimiter_pattern)
+ else:
+ cks = naive_merge(
payload,
self._param.chunk_token_size,
"".join(self._param.delimiters),
overlapped_percent,
)
- )
if custom_pattern:
docs = []
for c in cks:
@@ -357,11 +354,47 @@ class TokenChunker(ProcessBase):
self.set_output("chunks", [{"text": merged_text}] if merged_text.strip() else [])
self.callback(1, "Done.")
return
+
+ if self._param.delimiter_mode == "delimiter":
+ text_chunks = _build_json_chunks(json_result, "")
+ chunks = []
+ text_buffer = []
+
+ def flush_text_buffer():
+ if not text_buffer:
+ return
+ combined_text = "".join(text_buffer)
+ split_texts = _split_text_by_pattern(combined_text, delimiter_pattern)
+ chunks.extend(
+ {
+ "text": text,
+ "doc_type_kwd": "text",
+ "ck_type": "text",
+ "tk_nums": num_tokens_from_string(text),
+ }
+ for text in split_texts
+ if text.strip()
+ )
+ text_buffer.clear()
+
+ for chunk in text_chunks:
+ if chunk["ck_type"] == "text":
+ text_buffer.append(chunk["text"])
+ else:
+ flush_text_buffer()
+ chunks.append(chunk)
+ flush_text_buffer()
+ _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
+ await restore_pdf_text_previews(chunks, from_upstream, self._canvas)
+ self.set_output("chunks", _finalize_json_chunks(chunks))
+ self.callback(1, "Done.")
+ return
+
# Structured JSON input is normalized first, then optionally enriched with
# media context, and finally merged only when delimiter splitting is inactive.
chunks = _build_json_chunks(json_result, delimiter_pattern)
_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)
- if not delimiter_pattern:
+ if self._param.delimiter_mode == "token_size" and not delimiter_pattern:
chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)
if custom_pattern:
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index cf9048d02a..1d472ab4ff 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -1131,6 +1131,7 @@ class Parser(ProcessBase):
blob,
conf.get("chunk_token_num", 128),
conf.get("delimiter", "\n!?;。;!?"),
+ keep_delimiters=True,
)
if conf.get("output_format") == "json":
self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]])
diff --git a/test/unit_test/deepdoc/parser/test_txt_parser.py b/test/unit_test/deepdoc/parser/test_txt_parser.py
deleted file mode 100644
index 289f77ac8e..0000000000
--- a/test/unit_test/deepdoc/parser/test_txt_parser.py
+++ /dev/null
@@ -1,178 +0,0 @@
-#
-# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour.
-
-The pre-fix ``add_chunk`` fired its size check *after* the append, so each
-chunk could overshoot ``chunk_token_num`` by up to the size of one line. These
-tests assert the proactive projected-total invariant: no produced chunk may
-contain more than ``chunk_token_num`` tokens.
-"""
-
-import importlib.util
-import os
-import sys
-from unittest import mock
-
-_MOCK_MODULES = [
- "xgboost",
- "pdfplumber",
- "huggingface_hub",
- "PIL",
- "PIL.Image",
- "pypdf",
- "sklearn",
- "deepdoc.vision",
- "deepdoc",
- "deepdoc.parser",
- "deepdoc.parser.utils",
-]
-_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES}
-_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None)
-
-try:
- for _m in _MOCK_MODULES:
- if _m not in sys.modules:
- sys.modules[_m] = mock.MagicMock()
-
- # ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by
- # ``parser_txt``. Provide a permissive stub so the module loads.
- sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: ""
-
- def _find_project_root(marker="pyproject.toml"):
- d = os.path.dirname(os.path.abspath(__file__))
- while d != os.path.dirname(d):
- if os.path.exists(os.path.join(d, marker)):
- return d
- d = os.path.dirname(d)
- return None
-
- _PROJECT_ROOT = _find_project_root()
-
- _spec = importlib.util.spec_from_file_location(
- "deepdoc.parser._txt_parser_under_test",
- os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"),
- )
- _mod = importlib.util.module_from_spec(_spec)
- sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod
- _spec.loader.exec_module(_mod)
-
- RAGFlowTxtParser = _mod.RAGFlowTxtParser
-finally:
- for _m, _orig in _orig_modules.items():
- if _orig is None:
- sys.modules.pop(_m, None)
- else:
- sys.modules[_m] = _orig
- if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None:
- _orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text
- if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None:
- _orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"]
-
-
-# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so
-# the assertions below reason in plain words and are independent of tiktoken.
-
-
-def _patch_word_count(monkeypatch_module):
- def fake_num_tokens(s):
- return len((s or "").split())
-
- monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens)
-
-
-def test_no_overshoot_when_packing_short_lines(monkeypatch):
- """Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens."""
- _patch_word_count(monkeypatch)
- txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25)
- chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n")
- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
- assert all(s <= 100 for s in sizes), sizes
- # 75 tokens of content, expected a single 75-token chunk.
- assert sum(sizes) == 75
-
-
-def test_no_overshoot_at_chunk_boundary(monkeypatch):
- """Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens."""
- _patch_word_count(monkeypatch)
- lines = [" ".join([f"w{i}"] * 30) for i in range(10)] # 10 lines, 300 tokens
- chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n")
- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
- assert all(s <= 100 for s in sizes), sizes
-
-
-def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch):
- """A single line that exceeds the budget is split on whitespace atoms."""
- _patch_word_count(monkeypatch)
- huge_line = " ".join(["alpha"] * 80) # 80 tokens, no internal delimiter
- chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n")
- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]
- assert all(s <= 50 for s in sizes), sizes
- assert sum(sizes) == 80
- assert len(chunks) >= 2
-
-
-def test_empty_text_returns_empty(monkeypatch):
- _patch_word_count(monkeypatch)
- # Empty input produces a single empty chunk placeholder (existing
- # behaviour the callers rely on). The hard-cap guarantee is that any
- # chunk carrying content stays within the budget.
- result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n")
- non_empty = [c for c in result if c[0].strip()]
- assert non_empty == []
- result2 = RAGFlowTxtParser.parser_txt(" \n\n ", chunk_token_num=128, delimiter="\n")
- non_empty2 = [c for c in result2 if c[0].strip()]
- assert non_empty2 == []
-
-
-def test_unbroken_token_exceeding_budget_fallback(monkeypatch):
- """A single unbroken non-whitespace string exceeding the budget is split
- via the character-window/token-slicing fallback.
- """
-
- def char_count_tokens(s):
- return len(s or "")
-
- monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
-
- huge_word = "a" * 80 # 80 characters/tokens, no whitespace
- chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n")
-
- non_empty = [c[0] for c in chunks if c[0].strip()]
- assert len(non_empty) >= 3
- assert all(char_count_tokens(c) <= 30 for c in non_empty)
- assert "".join(non_empty) == huge_word
-
-
-def test_newline_join_token_count_strict_cap(monkeypatch):
- """Verify that joining chunks with newline does not overshoot chunk_token_num
- even when individual token counts sum to <= budget but the newline pushes it over.
- """
-
- def char_count_tokens(s):
- return len(s or "")
-
- monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)
-
- # Two lines of 10 chars each. Budget = 20.
- # line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20.
- line1 = "a" * 10
- line2 = "b" * 10
- txt = f"{line1}\n{line2}"
- chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n")
- non_empty = [c[0] for c in chunks if c[0].strip()]
- assert all(char_count_tokens(c) <= 20 for c in non_empty)
- assert len(non_empty) == 2
diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.ts
index f6e7768149..76facd72cc 100644
--- a/web/src/utils/__tests__/delimiter-preview.test.ts
+++ b/web/src/utils/__tests__/delimiter-preview.test.ts
@@ -34,9 +34,7 @@ describe('parseDelimitersForDisplay', () => {
});
it('normalizes CRLF before parsing', () => {
- expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([
- '\n',
- ]);
+ expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual(['\n']);
expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([
'\n',
]);