RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: align pipeline delimiter chunking (#17723)
buua4369ccb23e6612b
diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.pyindex 9e51ffc961..f6414bfc97 100644--- a/deepdoc/parser/txt_parser.py+++ b/deepdoc/parser/txt_parser.py@@ -19,7 +19,6 @@ import refrom common.token_utils import num_tokens_from_stringfrom deepdoc.parser.utils import get_text-from rag.nlp import _split_oversized_unitfrom rag.nlp.delim import (compile_delimiter_pattern,normalize_text_newlines,@@ -28,12 +27,12 @@ from rag.nlp.delim import (class RAGFlowTxtParser:- def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?"):+ def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):txt = get_text(fnm, binary)- return self.parser_txt(txt, chunk_token_num, delimiter)+ return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)@classmethod- def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?"):+ def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):if not isinstance(txt, str):raise TypeError("txt type should be str!")cks = [""]@@ -42,21 +41,15 @@ class RAGFlowTxtParser:def add_chunk(t):nonlocal cks, tk_numstnum = num_tokens_from_string(t)-- if cks[-1] == "":- cks[-1] = t- tk_nums[-1] = tnum- return-- merged = cks[-1] + "\n" + t- merged_tnum = num_tokens_from_string(merged)- if merged_tnum <= chunk_token_num:- cks[-1] = merged- tk_nums[-1] = merged_tnum- return-- cks.append(t)- tk_nums.append(tnum)+ if tk_nums[-1] > chunk_token_num:+ cks.append(t)+ tk_nums.append(tnum)+ else:+ if cks[-1]:+ cks[-1] += "\n" + t+ else:+ cks[-1] += t+ tk_nums[-1] += tnumtxt = normalize_text_newlines(txt)parsed_dels = parse_delimiter_field(delimiter)@@ -67,18 +60,14 @@ class RAGFlowTxtParser:bool(dels),)secs = re.split(r"(%s)" % dels, txt) if dels else [txt]- for sec in secs:+ for index, sec in enumerate(secs):if dels and re.match(f"^{dels}$", sec):continueif not sec:continue- if num_tokens_from_string(sec) <= chunk_token_num:- add_chunk(sec)- continue- pieces = _split_oversized_unit(sec, chunk_token_num, token_count_fn=num_tokens_from_string)- logging.debug("parser_txt: split oversized section (%d tokens) into %d pieces", num_tokens_from_string(sec), len(pieces))- for piece in pieces:- add_chunk(piece)+ if keep_delimiters and index + 1 < len(secs) and re.match(f"^{dels}$", secs[index + 1]):+ sec += secs[index + 1]+ add_chunk(sec)logging.debug("parser_txt: %d sections -> %d chunks (chunk_token_num=%d)", len(secs), len(cks), chunk_token_num)return [[c, ""] for c in cks]diff --git a/rag/flow/chunker/token_chunker.py b/rag/flow/chunker/token_chunker.pyindex b7a75c5c3a..ab9063f43c 100644--- a/rag/flow/chunker/token_chunker.py+++ b/rag/flow/chunker/token_chunker.py@@ -77,7 +77,7 @@ def _compile_delimiter_pattern(delimiters):def _split_text_by_pattern(text, pattern):- # Split text by the compiled delimiter pattern and keep delimiter text in each chunk.+ # Split text by the compiled delimiter pattern and discard delimiters.if not pattern:return [text or ""]@@ -85,10 +85,6 @@ def _split_text_by_pattern(text, pattern):chunks = []for i in range(0, len(split_texts), 2):chunk = split_texts[i]- if not chunk:- continue- if i + 1 < len(split_texts):- chunk += split_texts[i + 1]if chunk.strip():chunks.append(chunk)return chunks@@ -317,16 +313,17 @@ class TokenChunker(ProcessBase):self.set_output("chunks", [{"text": payload}] if payload.strip() else [])self.callback(1, "Done.")return- cks = (- _split_text_by_pattern(payload, delimiter_pattern)- if delimiter_pattern- else naive_merge(+ if self._param.delimiter_mode == "delimiter":+ cks = _split_text_by_pattern(payload, delimiter_pattern)+ elif delimiter_pattern:+ cks = _split_text_by_pattern(payload, delimiter_pattern)+ else:+ cks = naive_merge(payload,self._param.chunk_token_size,"".join(self._param.delimiters),overlapped_percent,)- )if custom_pattern:docs = []for c in cks:@@ -357,11 +354,47 @@ class TokenChunker(ProcessBase):self.set_output("chunks", [{"text": merged_text}] if merged_text.strip() else [])self.callback(1, "Done.")return++ if self._param.delimiter_mode == "delimiter":+ text_chunks = _build_json_chunks(json_result, "")+ chunks = []+ text_buffer = []++ def flush_text_buffer():+ if not text_buffer:+ return+ combined_text = "".join(text_buffer)+ split_texts = _split_text_by_pattern(combined_text, delimiter_pattern)+ chunks.extend(+ {+ "text": text,+ "doc_type_kwd": "text",+ "ck_type": "text",+ "tk_nums": num_tokens_from_string(text),+ }+ for text in split_texts+ if text.strip()+ )+ text_buffer.clear()++ for chunk in text_chunks:+ if chunk["ck_type"] == "text":+ text_buffer.append(chunk["text"])+ else:+ flush_text_buffer()+ chunks.append(chunk)+ flush_text_buffer()+ _attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)+ await restore_pdf_text_previews(chunks, from_upstream, self._canvas)+ self.set_output("chunks", _finalize_json_chunks(chunks))+ self.callback(1, "Done.")+ return+# Structured JSON input is normalized first, then optionally enriched with# media context, and finally merged only when delimiter splitting is inactive.chunks = _build_json_chunks(json_result, delimiter_pattern)_attach_context_to_media_chunks(chunks, self._param.table_context_size, self._param.image_context_size)- if not delimiter_pattern:+ if self._param.delimiter_mode == "token_size" and not delimiter_pattern:chunks = _merge_text_chunks_by_token_size(chunks, self._param.chunk_token_size, overlapped_percent)if custom_pattern:diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.pyindex cf9048d02a..1d472ab4ff 100644--- a/rag/flow/parser/parser.py+++ b/rag/flow/parser/parser.py@@ -1131,6 +1131,7 @@ class Parser(ProcessBase):blob,conf.get("chunk_token_num", 128),conf.get("delimiter", "\n!?;。;!?"),+ keep_delimiters=True,)if conf.get("output_format") == "json":self.set_output("json", [{"text": section[0], "doc_type_kwd": "text"} for section in sections if section[0]])diff --git a/test/unit_test/deepdoc/parser/test_txt_parser.py b/test/unit_test/deepdoc/parser/test_txt_parser.pydeleted file mode 100644index 289f77ac8e..0000000000--- a/test/unit_test/deepdoc/parser/test_txt_parser.py+++ /dev/null@@ -1,178 +0,0 @@-#-# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-#--"""Unit tests for ``RAGFlowTxtParser.parser_txt`` strict-cap behaviour.--The pre-fix ``add_chunk`` fired its size check *after* the append, so each-chunk could overshoot ``chunk_token_num`` by up to the size of one line. These-tests assert the proactive projected-total invariant: no produced chunk may-contain more than ``chunk_token_num`` tokens.-"""--import importlib.util-import os-import sys-from unittest import mock--_MOCK_MODULES = [- "xgboost",- "pdfplumber",- "huggingface_hub",- "PIL",- "PIL.Image",- "pypdf",- "sklearn",- "deepdoc.vision",- "deepdoc",- "deepdoc.parser",- "deepdoc.parser.utils",-]-_orig_modules = {m: sys.modules.get(m) for m in _MOCK_MODULES}-_orig_get_text = getattr(sys.modules.get("deepdoc.parser.utils"), "get_text", None)--try:- for _m in _MOCK_MODULES:- if _m not in sys.modules:- sys.modules[_m] = mock.MagicMock()-- # ``get_text`` is invoked by ``RAGFlowTxtParser.__call__`` only, not by- # ``parser_txt``. Provide a permissive stub so the module loads.- sys.modules["deepdoc.parser.utils"].get_text = lambda *a, **kw: ""-- def _find_project_root(marker="pyproject.toml"):- d = os.path.dirname(os.path.abspath(__file__))- while d != os.path.dirname(d):- if os.path.exists(os.path.join(d, marker)):- return d- d = os.path.dirname(d)- return None-- _PROJECT_ROOT = _find_project_root()-- _spec = importlib.util.spec_from_file_location(- "deepdoc.parser._txt_parser_under_test",- os.path.join(_PROJECT_ROOT, "deepdoc", "parser", "txt_parser.py"),- )- _mod = importlib.util.module_from_spec(_spec)- sys.modules["deepdoc.parser._txt_parser_under_test"] = _mod- _spec.loader.exec_module(_mod)-- RAGFlowTxtParser = _mod.RAGFlowTxtParser-finally:- for _m, _orig in _orig_modules.items():- if _orig is None:- sys.modules.pop(_m, None)- else:- sys.modules[_m] = _orig- if _orig_modules.get("deepdoc.parser.utils") is not None and _orig_get_text is not None:- _orig_modules["deepdoc.parser.utils"].get_text = _orig_get_text- if _orig_modules.get("deepdoc.parser") is not None and _orig_modules.get("deepdoc.parser.utils") is not None:- _orig_modules["deepdoc.parser"].utils = _orig_modules["deepdoc.parser.utils"]---# A deterministic, tokenizer-free stand-in for ``num_tokens_from_string`` so-# the assertions below reason in plain words and are independent of tiktoken.---def _patch_word_count(monkeypatch_module):- def fake_num_tokens(s):- return len((s or "").split())-- monkeypatch_module.setattr(_mod, "num_tokens_from_string", fake_num_tokens)---def test_no_overshoot_when_packing_short_lines(monkeypatch):- """Lines of 25 tokens, budget 100 — every chunk must be <= 100 tokens."""- _patch_word_count(monkeypatch)- txt = " ".join(["alpha"] * 25) + "\n" + " ".join(["beta"] * 25) + "\n" + " ".join(["gamma"] * 25)- chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=100, delimiter="\n")- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]- assert all(s <= 100 for s in sizes), sizes- # 75 tokens of content, expected a single 75-token chunk.- assert sum(sizes) == 75---def test_no_overshoot_at_chunk_boundary(monkeypatch):- """Lines of 30 tokens, budget 100. Pre-fix the boundary chunk was 130 tokens."""- _patch_word_count(monkeypatch)- lines = [" ".join([f"w{i}"] * 30) for i in range(10)] # 10 lines, 300 tokens- chunks = RAGFlowTxtParser.parser_txt("\n".join(lines), chunk_token_num=100, delimiter="\n")- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]- assert all(s <= 100 for s in sizes), sizes---def test_atomic_oversized_line_is_sub_split_on_whitespace(monkeypatch):- """A single line that exceeds the budget is split on whitespace atoms."""- _patch_word_count(monkeypatch)- huge_line = " ".join(["alpha"] * 80) # 80 tokens, no internal delimiter- chunks = RAGFlowTxtParser.parser_txt(huge_line, chunk_token_num=50, delimiter="\n")- sizes = [len(c[0].split()) for c in chunks if c[0].strip()]- assert all(s <= 50 for s in sizes), sizes- assert sum(sizes) == 80- assert len(chunks) >= 2---def test_empty_text_returns_empty(monkeypatch):- _patch_word_count(monkeypatch)- # Empty input produces a single empty chunk placeholder (existing- # behaviour the callers rely on). The hard-cap guarantee is that any- # chunk carrying content stays within the budget.- result = RAGFlowTxtParser.parser_txt("", chunk_token_num=128, delimiter="\n")- non_empty = [c for c in result if c[0].strip()]- assert non_empty == []- result2 = RAGFlowTxtParser.parser_txt(" \n\n ", chunk_token_num=128, delimiter="\n")- non_empty2 = [c for c in result2 if c[0].strip()]- assert non_empty2 == []---def test_unbroken_token_exceeding_budget_fallback(monkeypatch):- """A single unbroken non-whitespace string exceeding the budget is split- via the character-window/token-slicing fallback.- """-- def char_count_tokens(s):- return len(s or "")-- monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)-- huge_word = "a" * 80 # 80 characters/tokens, no whitespace- chunks = RAGFlowTxtParser.parser_txt(huge_word, chunk_token_num=30, delimiter="\n")-- non_empty = [c[0] for c in chunks if c[0].strip()]- assert len(non_empty) >= 3- assert all(char_count_tokens(c) <= 30 for c in non_empty)- assert "".join(non_empty) == huge_word---def test_newline_join_token_count_strict_cap(monkeypatch):- """Verify that joining chunks with newline does not overshoot chunk_token_num- even when individual token counts sum to <= budget but the newline pushes it over.- """-- def char_count_tokens(s):- return len(s or "")-- monkeypatch.setattr(_mod, "num_tokens_from_string", char_count_tokens)-- # Two lines of 10 chars each. Budget = 20.- # line1 + "\n" + line2 = 10 + 1 + 10 = 21 chars/tokens, exceeding budget of 20.- line1 = "a" * 10- line2 = "b" * 10- txt = f"{line1}\n{line2}"- chunks = RAGFlowTxtParser.parser_txt(txt, chunk_token_num=20, delimiter="\n")- non_empty = [c[0] for c in chunks if c[0].strip()]- assert all(char_count_tokens(c) <= 20 for c in non_empty)- assert len(non_empty) == 2diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.tsindex f6e7768149..76facd72cc 100644--- a/web/src/utils/__tests__/delimiter-preview.test.ts+++ b/web/src/utils/__tests__/delimiter-preview.test.ts@@ -34,9 +34,7 @@ describe('parseDelimitersForDisplay', () => {});it('normalizes CRLF before parsing', () => {- expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([- '\n',- ]);+ expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual(['\n']);expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual(['\n',]);