Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(rag): unify parser_config delimiter defaults on a single constant (#19058)

euvre4bb522d11d78
diff --git a/api/db/services/file_service.py b/api/db/services/file_service.py
index 7b3825d48b..245b7852af 100644
--- a/api/db/services/file_service.py
+++ b/api/db/services/file_service.py
@@ -43,6 +43,7 @@ from common.constants import MAXIMUM_PAGE_NUMBER, FileSource, ParserType, TaskSt
from common.misc_utils import get_uuid
from common.ssrf_guard import assert_url_is_safe
from rag.llm.cv_model import GptV4
+from rag.nlp.delim import DEFAULT_DELIMITER
class FileService(CommonService):
@@ -710,7 +711,7 @@ class FileService(CommonService):
pass
FACTORY = {ParserType.PRESENTATION.value: presentation, ParserType.PICTURE.value: picture, ParserType.AUDIO.value: audio, ParserType.EMAIL.value: email}
- parser_config = {"chunk_token_num": 16096, "delimiter": "\n!?;。;!?", "layout_recognize": layout_recognize or "Plain Text"}
+ parser_config = {"chunk_token_num": 16096, "delimiter": DEFAULT_DELIMITER, "layout_recognize": layout_recognize or "Plain Text"}
kwargs = {"lang": "English", "callback": dummy, "parser_config": parser_config, "from_page": 0, "to_page": MAXIMUM_PAGE_NUMBER, "tenant_id": current_user.id if current_user else tenant_id}
file_type = filename_type(filename)
if img_base64 and file_type == FileType.VISUAL.value:
diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.py
index 2f914ffb08..b0dd6556e4 100644
--- a/deepdoc/parser/txt_parser.py
+++ b/deepdoc/parser/txt_parser.py
@@ -20,6 +20,7 @@ import re
from deepdoc.parser.utils import get_text
from rag.nlp import MergeStrategy, merge_paragraphs
from rag.nlp.delim import (
+ DEFAULT_DELIMITER,
compile_delimiter_pattern,
normalize_text_newlines,
parse_delimiter_field,
@@ -27,12 +28,12 @@ from rag.nlp.delim import (
class RAGFlowTxtParser:
- def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):
+ def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):
txt = get_text(fnm, binary)
return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)
@classmethod
- def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):
+ def parser_txt(cls, txt, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):
if not isinstance(txt, str):
raise TypeError("txt type should be str!")
txt = normalize_text_newlines(txt)
diff --git a/internal/ingestion/component/chunker/sentence_boundary.go b/internal/ingestion/component/chunker/sentence_boundary.go
index 1d0e654027..0632ee1e09 100644
--- a/internal/ingestion/component/chunker/sentence_boundary.go
+++ b/internal/ingestion/component/chunker/sentence_boundary.go
@@ -26,7 +26,7 @@ import "regexp"
//
// The token chunker's TEXT path (naive_merge port) uses a different delimiter
// — sentenceDelimiter in token.go, mirroring Python naive_merge's production
-// delimiter "\n!?。;!?" (no ". "). The two constants mirror two distinct
-// Python constants and differ only in that English boundary, so they are
-// intentionally NOT merged into one.
+// default DEFAULT_DELIMITER ("\n!?;。;!?", no ". "). The two constants mirror
+// two distinct Python constants and differ only in that English boundary, so
+// they are intentionally NOT merged into one.
var sentenceBoundaryRe = regexp.MustCompile(`([。!??;!\n]|\. )`)
diff --git a/internal/ingestion/component/chunker/sentence_boundary_test.go b/internal/ingestion/component/chunker/sentence_boundary_test.go
index a8da922107..387ca7ea17 100644
--- a/internal/ingestion/component/chunker/sentence_boundary_test.go
+++ b/internal/ingestion/component/chunker/sentence_boundary_test.go
@@ -39,11 +39,12 @@ func TestSentenceBoundary_SharedRegexMatchesPython(t *testing.T) {
// TestSentenceBoundary_TokenTextPathDelimiterDistinct pins sentenceDelimiter
// (the token-chunker TEXT path / naive_merge port) to Python naive_merge's
-// production delimiter "\n!?。;!?" — deliberately WITHOUT the English ". "
-// boundary. The two constants mirror two distinct Python delimiters; asserting
-// the exact difference documents that this is intentional, not a drift.
+// production default DEFAULT_DELIMITER ("\n!?;。;!?", rag/nlp/delim.py) —
+// deliberately WITHOUT the English ". " boundary. The two constants mirror two
+// distinct Python delimiters; asserting the exact difference documents that
+// this is intentional, not a drift.
func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {
- const want = `(\n|[!?。;!?])`
+ const want = `(\n|[!?;。;!?])`
if got := sentenceDelimiter.String(); got != want {
t.Errorf("sentenceDelimiter = %q, want %q (Python naive_merge delimiter)", got, want)
}
@@ -52,6 +53,11 @@ func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {
if sentenceDelimiter.MatchString("Hello. World") {
t.Error("sentenceDelimiter split on '. ' but naive_merge's delimiter has no '. ' boundary")
}
+ // The text-path delimiter must split on the ASCII semicolon, matching the
+ // unified Python default (issue #18562: the "; " omission was drift).
+ if !sentenceDelimiter.MatchString("one; two") {
+ t.Error("sentenceDelimiter should split on ';': naive_merge's default includes ';'")
+ }
if !sentenceBoundaryRe.MatchString("Hello. World") {
t.Error("sentenceBoundaryRe should split on '. '")
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index ff362f11fc..a355477448 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -364,14 +364,15 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child
// sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the
// naive_merge port). It mirrors the delimiter Python's naive_merge receives in
-// production: rag/app/naive.py passes "\n!?。;!?", which includes ASCII "!"
-// and "?" plus the CJK punctuation "。;!?" but NOT an English ". " boundary.
+// production: rag/app/naive.py passes DEFAULT_DELIMITER ("\n!?;。;!?", defined
+// once in rag/nlp/delim.py), which includes ASCII "!", "?" and ";" plus the CJK
+// punctuation "。;!?" but NOT an English ". " boundary.
//
// The Title chunker and the token chunker's JSON path use the shared
// sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,
// which adds ". "). The two constants mirror two distinct Python delimiters and
// differ only in that English boundary, exactly as Python does.
-var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。;!?])`)
+var sentenceDelimiter = regexp.MustCompile(`(\n|[!?;。;!?])`)
// overlapCut returns the visible-text rune offset where the overlap prefix
// begins, mirroring the cut computed inside computeOverlapPrefix. It is split
diff --git a/rag/app/book.py b/rag/app/book.py
index d866bc431d..d92a6b4d84 100644
--- a/rag/app/book.py
+++ b/rag/app/book.py
@@ -67,6 +67,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
Since a book is long and not all the parts are useful, if it's a PDF,
please set up the page ranges for every book in order eliminate negative effects and save elapsed computing time.
"""
+ # Deliberate carve-out from DEFAULT_DELIMITER (#18562): "book" targets
+ # Chinese prose, so this kwargs dict and the naive_merge fallback below
+ # keep their own Chinese-only delimiter sets, not the unified constant.
parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})
doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])
diff --git a/rag/app/email.py b/rag/app/email.py
index bde97b207c..e0d0a41f0b 100644
--- a/rag/app/email.py
+++ b/rag/app/email.py
@@ -20,7 +20,7 @@ from email.parser import BytesParser
from rag.app.naive import chunk as naive_chunk
from common.constants import MAXIMUM_PAGE_NUMBER
import re
-from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks
+from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks, DEFAULT_DELIMITER
from deepdoc.parser import HtmlParser, TxtParser
from timeit import default_timer as timer
import io
@@ -41,7 +41,7 @@ def chunk(
eng = lang.lower() == "english" # is_english(cks)
parser_config = kwargs.get(
"parser_config",
- {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"},
+ {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"},
)
doc = {
"docnm_kwd": filename,
@@ -99,7 +99,7 @@ def chunk(
chunks = naive_merge(
sections,
int(parser_config.get("chunk_token_num", 128)),
- parser_config.get("delimiter", "\n!?。;!?"),
+ parser_config.get("delimiter", DEFAULT_DELIMITER),
)
main_res.extend(tokenize_chunks(chunks, doc, eng, None, language=lang))
diff --git a/rag/app/laws.py b/rag/app/laws.py
index 83a3837d7f..3265138c64 100644
--- a/rag/app/laws.py
+++ b/rag/app/laws.py
@@ -24,7 +24,7 @@ from docx.text.paragraph import Paragraph
from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
from deepdoc.parser.utils import get_text
-from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge
+from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge, DEFAULT_DELIMITER
from rag.nlp import rag_tokenizer, Node
from deepdoc.parser import PdfParser, DocxParser, HtmlParser
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
@@ -169,7 +169,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
"""
Supported file formats are docx, pdf, txt.
"""
- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})
+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}
doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])
pdf_parser = None
diff --git a/rag/app/manual.py b/rag/app/manual.py
index 65743ecbbd..7c27f0a5f3 100644
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -21,7 +21,7 @@ import re
from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
from io import BytesIO
from deepdoc.parser.utils import extract_pdf_outlines
-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img
+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img, DEFAULT_DELIMITER
from common.token_utils import num_tokens_from_string
from deepdoc.parser import PdfParser, DocxParser
from deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper, vision_figure_parser_docx_wrapper
@@ -139,7 +139,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
"""
Only pdf is supported.
"""
- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})
+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
pdf_parser = None
doc = {"docnm_kwd": filename}
doc["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", doc["docnm_kwd"]))
diff --git a/rag/app/naive.py b/rag/app/naive.py
index 51593887b9..eaf941d3e5 100644
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -52,6 +52,7 @@ from common.parser_config_utils import has_mineru_options, normalize_layout_reco
from common.text_utils import normalize_arabic_presentation_forms
from rag.nlp import (
concat_img,
+ DEFAULT_DELIMITER,
find_codec,
naive_merge,
naive_merge_with_images,
@@ -1067,7 +1068,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
lang = lang or "Chinese"
is_english = lang.lower() == "english" # is_english(cks)
- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC", "analyze_hyperlink": True})
+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC", "analyze_hyperlink": True})
child_deli = (parser_config.get("children_delimiter") or "").encode("utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")
cust_child_deli = re.findall(r"`([^`]+)`", child_deli)
@@ -1133,7 +1134,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
# chunks list[dict]
# images list - index of image chunk in chunks
- chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), table_context_size, image_context_size)
+ chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), table_context_size, image_context_size)
vision_figure_parser_docx_wrapper_naive(
chunks=chunks,
@@ -1261,7 +1262,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
elif re.search(r"\.(txt|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|sql)$", filename, re.IGNORECASE):
callback(0.1, "Start to parse.")
- sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", "\n!?;。;!?"))
+ sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", DEFAULT_DELIMITER))
sections = _normalize_section_text_for_rtl_presentation_forms(sections)
logging.info("TxtParser produced %d sections for %s", len(sections), filename)
callback(0.8, "Finish parsing.")
@@ -1273,7 +1274,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
filename,
binary,
separate_tables=False,
- delimiter=parser_config.get("delimiter", "\n!?;。;!?"),
+ delimiter=parser_config.get("delimiter", DEFAULT_DELIMITER),
return_section_images=True,
)
sections = _normalize_section_text_for_rtl_presentation_forms(sections)
@@ -1428,10 +1429,10 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
section_images = None
if section_images:
- chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), overlapped_percent)
+ chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)
res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli, language=lang))
else:
- chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), overlapped_percent)
+ chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)
res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang))
diff --git a/rag/app/one.py b/rag/app/one.py
index 7cde2485e8..e866a8beb1 100644
--- a/rag/app/one.py
+++ b/rag/app/one.py
@@ -20,7 +20,7 @@ import re
from deepdoc.parser.utils import get_text
from rag.app import naive
-from rag.nlp import rag_tokenizer, tokenize
+from rag.nlp import rag_tokenizer, tokenize, DEFAULT_DELIMITER
from deepdoc.parser import PdfParser, ExcelParser, HtmlParser
from deepdoc.parser.figure_parser import vision_figure_parser_docx_wrapper_naive
from rag.app.naive import by_plaintext, PARSERS
@@ -65,7 +65,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
Supported file formats are docx, pdf, excel, txt.
One file forms a chunk which maintains original text order.
"""
- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})
+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
eng = lang.lower() == "english" # is_english(cks)
if re.search(r"\.docx$", filename, re.IGNORECASE):
diff --git a/rag/app/paper.py b/rag/app/paper.py
index c0ffb29032..97bbc03572 100644
--- a/rag/app/paper.py
+++ b/rag/app/paper.py
@@ -20,7 +20,7 @@ import re
from deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper
from common.constants import ParserType, MAXIMUM_PAGE_NUMBER
-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context
+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context, DEFAULT_DELIMITER
from deepdoc.parser import PdfParser
import numpy as np
from api.db.joint_services.tenant_model_service import get_composite_model_name_by_id
@@ -137,7 +137,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
Only pdf is supported.
The abstract of the paper will be sliced as an entire chunk, and will not be sliced partly.
"""
- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})
+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})
if re.search(r"\.pdf$", filename, re.IGNORECASE):
layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")
tenant_id = kwargs.get("tenant_id")
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index d8742083ed..402669ef76 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -38,6 +38,7 @@ from api.db.joint_services.tenant_model_service import (
)
from api.db.services.tenant_model_instance_service import TenantModelInstanceService
from api.db.services.tenant_model_provider_service import TenantModelProviderService
+from rag.nlp.delim import DEFAULT_DELIMITER
from api.db.services.tenant_model_service import TenantModelService
from common import settings
from common.constants import LLMType
@@ -1148,7 +1149,7 @@ class Parser(ProcessBase):
name,
blob,
conf.get("chunk_token_num", 128),
- conf.get("delimiter", "\n!?;。;!?"),
+ conf.get("delimiter", DEFAULT_DELIMITER),
keep_delimiters=True,
)
if conf.get("output_format") == "json":
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index e6e48d7ce2..ab101218d9 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -32,6 +32,7 @@ from common.token_utils import num_tokens_from_string
# Re-exported below for backwards compatibility; the canonical parser lives
# in ``rag.nlp.delim``.
from rag.nlp.delim import (
+ DEFAULT_DELIMITER,
compile_delimiter_pattern,
has_wrapped_delimiter,
normalize_text_newlines,
@@ -1415,7 +1416,7 @@ def _apply_overlap_unconditional(chunks, overlapped_percent):
return out
-def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
+def naive_merge(sections: str | list, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
"""Split sections into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""
if not sections:
return []
@@ -1478,7 +1479,7 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;
return _apply_overlap_unconditional(cks, overlapped_percent)
-def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
+def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):
"""Split texts (with images) into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""
if not texts or len(texts) != len(images):
return [], []
@@ -1834,7 +1835,7 @@ def _merge_cks(cks, chunk_token_num, has_custom):
def naive_merge_docx(
sections,
chunk_token_num=128,
- delimiter="\n。;!?",
+ delimiter=DEFAULT_DELIMITER,
table_context_size=0,
image_context_size=0,
):
diff --git a/rag/nlp/delim.py b/rag/nlp/delim.py
index 955bc706c6..05f1dbe5b2 100644
--- a/rag/nlp/delim.py
+++ b/rag/nlp/delim.py
@@ -78,6 +78,15 @@ import re
# Match a backtick-wrapped token. Case-sensitive on purpose (see #17384).
_BACKTICK_RE = re.compile(r"`([^`]+)`")
+# The single source of truth for the default value of the
+# ``parser_config.delimiter`` field (issue #18562). Before this constant
+# existed, the per-callback ``parser_config.get("delimiter", ...)`` defaults
+# drifted across several copy-pasted values: txt/markdown shipped this full
+# 8-character set, docx/image/email omitted the ASCII ``;``, and
+# ``rag/app/book.py`` keeps its own deliberate Chinese-only set. Every backend
+# default must reference this constant so the value can never drift again.
+DEFAULT_DELIMITER = "\n!?;。;!?"
+
def normalize_text_newlines(text: str) -> str:
"""Normalize CRLF and standalone CR to LF in source text."""
diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py
index ae5d3fa0f2..dd7bdada6c 100644
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -82,7 +82,7 @@ from api.db.joint_services.tenant_model_service import get_tenant_default_model_
from common.versions import get_ragflow_version
from api.db.db_models import close_connection
from rag.app import laws, paper, presentation, manual, qa, table, book, resume, picture, naive, one, audio, email, tag
-from rag.nlp import search, rag_tokenizer, add_positions
+from rag.nlp import search, rag_tokenizer, add_positions, DEFAULT_DELIMITER
from common.token_utils import num_tokens_from_string, truncate
from rag.utils.redis_conn import REDIS_CONN, RedisDistributedLock
@@ -342,7 +342,7 @@ async def build_chunks(task, progress_callback, on_chunking_start=None):
"parser_id": task["parser_id"],
"chunk_token_num": parser_config_for_chunk.get("chunk_token_num", 128),
"overlapped_percent": normalize_overlapped_percent(parser_config_for_chunk.get("overlapped_percent", 0)),
- "delimiter": parser_config_for_chunk.get("delimiter", "\n!?。;!?"),
+ "delimiter": parser_config_for_chunk.get("delimiter", DEFAULT_DELIMITER),
"from_page": task["from_page"],
"to_page": task["to_page"],
"language": task["language"],
diff --git a/rag/svr/task_executor_refactor/chunk_service.py b/rag/svr/task_executor_refactor/chunk_service.py
index d792c353cd..a76b71e593 100644
--- a/rag/svr/task_executor_refactor/chunk_service.py
+++ b/rag/svr/task_executor_refactor/chunk_service.py
@@ -40,7 +40,7 @@ from common.misc_utils import thread_pool_exec
from common.float_utils import normalize_overlapped_percent
from api.db.services.document_service import DocumentService
from api.db.services.task_service import TaskService
-from rag.nlp import search
+from rag.nlp import search, DEFAULT_DELIMITER
from rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_ID
from rag.svr.task_executor_refactor.task_context import TaskContext
from rag.utils.base64_image import image2id
@@ -171,7 +171,7 @@ class ChunkService:
"parser_id": ctx.parser_id,
"chunk_token_num": ctx.parser_config.get("chunk_token_num", 128),
"overlapped_percent": normalize_overlapped_percent(ctx.parser_config.get("overlapped_percent", 0)),
- "delimiter": ctx.parser_config.get("delimiter", "\n!?。;!?"),
+ "delimiter": ctx.parser_config.get("delimiter", DEFAULT_DELIMITER),
"from_page": ctx.from_page,
"to_page": ctx.to_page,
"language": ctx.language,
diff --git a/test/unit_test/rag/nlp/test_default_delimiter_constant.py b/test/unit_test/rag/nlp/test_default_delimiter_constant.py
new file mode 100644
index 0000000000..0aa4cc3e94
--- /dev/null
+++ b/test/unit_test/rag/nlp/test_default_delimiter_constant.py
@@ -0,0 +1,151 @@
+#
+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Pin the single source of truth for the default ``parser_config.delimiter``.
+
+Before #18562 the backend carried four distinct copy-pasted defaults for the
+same field: txt/markdown merged with ``"\\n!?;。;!?"``, docx/image/email with
+``"\\n!?。;!?"`` (no ASCII ``;``), the ``naive_merge*`` signatures with the
+Chinese-only ``"\\n。;!?"``, and the UI/API layer with ``"\\n"``. The
+docx/image/email ``;`` omission was user-visible: English semicolon prose
+chunked differently (often one oversized chunk) than the same text as
+.txt/.md.
+
+These tests pin the unified state:
+
+* ``rag.nlp.delim.DEFAULT_DELIMITER`` is the 8-character set and parses to
+ exactly those delimiters.
+* Every backend ``parser_config.get("delimiter", ...)`` fallback, kwargs
+ default dict, and ``naive_merge*`` / ``RAGFlowTxtParser`` signature default
+ references the constant (AST-checked, so a future copy-pasted literal fails
+ loudly instead of drifting silently).
+* ``rag/app/book.py`` deliberately keeps its own Chinese-only defaults (the
+ kwargs default dict and the ``naive_merge`` fallback) — a product decision,
+ not drift, pinned here as a carve-out.
+* Behaviourally, semicolon-separated English text splits at ``;`` under the
+ unified default but stays whole under the pre-fix docx default.
+"""
+
+import ast
+import inspect
+from pathlib import Path
+
+from rag.nlp import naive_merge, naive_merge_docx, naive_merge_with_images
+from rag.nlp.delim import DEFAULT_DELIMITER, parse_delimiter_field
+
+REPO_ROOT = Path(__file__).resolve().parents[4]
+
+# Production files whose ``delimiter`` defaults must reference the constant,
+# mapped to the exact number of defaults each file carries. Exact per-file
+# counts (rather than an aggregate floor) make a deleted fallback or a newly
+# copy-pasted literal fail loudly instead of silently re-drifting.
+EXPECTED_DELIMITER_DEFAULTS = {
+ "rag/app/naive.py": 6,
+ "rag/app/email.py": 2,
+ "rag/app/laws.py": 1,
+ "rag/app/one.py": 1,
+ "rag/app/manual.py": 1,
+ "rag/app/paper.py": 1,
+ "rag/svr/task_executor.py": 1,
+ "rag/svr/task_executor_refactor/chunk_service.py": 1,
+ "api/db/services/file_service.py": 1,
+ "rag/flow/parser/parser.py": 1,
+ "deepdoc/parser/txt_parser.py": 2,
+ "rag/nlp/__init__.py": 3,
+}
+
+
+def test_constant_value():
+ assert DEFAULT_DELIMITER == "\n!?;。;!?"
+ # Newline + ASCII !, ?, ; + CJK 。;!? — 8 delimiters, deduplicated.
+ assert parse_delimiter_field(DEFAULT_DELIMITER) == ["\n", "!", "?", ";", "。", ";", "!", "?"]
+
+
+def _delimiter_defaults(tree):
+ """Yield (kind, default_node) for every ``delimiter`` default in the module."""
+ for node in ast.walk(tree):
+ if (
+ isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "get"
+ and node.args
+ and isinstance(node.args[0], ast.Constant)
+ and node.args[0].value == "delimiter"
+ and len(node.args) > 1
+ and isinstance(node.args[1], (ast.Constant, ast.Name))
+ ):
+ yield ".get", node.args[1]
+ if isinstance(node, ast.Dict):
+ for key, value in zip(node.keys, node.values):
+ if isinstance(key, ast.Constant) and key.value == "delimiter" and isinstance(value, (ast.Constant, ast.Name)):
+ yield "dict", value
+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
+ pairs = list(zip(node.args.args[-len(node.args.defaults) :], node.args.defaults)) if node.args.defaults else []
+ pairs += [(a, a.default) for a in node.args.kwonlyargs if a.default is not None]
+ for arg, default in pairs:
+ if arg.arg == "delimiter" and not (isinstance(default, ast.Constant) and default.value is None):
+ yield f"sig:{node.name}", default
+
+
+def test_all_backend_defaults_reference_the_constant():
+ for relpath, expected in EXPECTED_DELIMITER_DEFAULTS.items():
+ tree = ast.parse((REPO_ROOT / relpath).read_text(encoding="utf-8"))
+ defaults = list(_delimiter_defaults(tree))
+ assert len(defaults) == expected, (
+ f"{relpath}: expected exactly {expected} delimiter defaults, found {len(defaults)}; update EXPECTED_DELIMITER_DEFAULTS in the same change so a removed or new default stays loud"
+ )
+ for kind, default in defaults:
+ assert isinstance(default, ast.Name) and default.id == "DEFAULT_DELIMITER", f"{relpath}: '{kind}' delimiter default is {ast.dump(default)}, expected the DEFAULT_DELIMITER constant"
+
+
+def test_book_keeps_its_deliberate_chinese_only_default():
+ """``book`` chunking targets Chinese prose; its defaults are a product choice."""
+ tree = ast.parse((REPO_ROOT / "rag/app/book.py").read_text(encoding="utf-8"))
+ found = list(_delimiter_defaults(tree))
+ kwargs_dict_defaults = [d for kind, d in found if kind == "dict"]
+ merge_defaults = [d for kind, d in found if kind == ".get"]
+ # Two distinct Chinese-only sets, both deliberate carve-outs from
+ # DEFAULT_DELIMITER: the chunk() kwargs default dict ("\n!?。;!?")
+ # and the naive_merge fallback ("\n。;!?").
+ assert any(isinstance(d, ast.Constant) and d.value == "\n!?。;!?" for d in kwargs_dict_defaults)
+ assert any(isinstance(d, ast.Constant) and d.value == "\n。;!?" for d in merge_defaults)
+
+
+def test_merge_signatures_carry_the_constant():
+ for func in (naive_merge, naive_merge_with_images, naive_merge_docx):
+ default = inspect.signature(func).parameters["delimiter"].default
+ assert default == DEFAULT_DELIMITER, f"{func.__name__} signature default diverges from DEFAULT_DELIMITER"
+
+
+def test_semicolon_english_text_splits_under_the_unified_default():
+ """The reported symptom: ``;`` clauses must split like they do for txt/md."""
+ text = "; ".join(
+ [
+ "The rain fell gently over the quiet town",
+ "the sun broke through the grey clouds",
+ "the wind stirred the leaves in the yard",
+ "children ran along the wet stone path",
+ "a distant bell rang twice before noon",
+ ]
+ )
+ sections = [(text, "")]
+ unified = naive_merge(sections, chunk_token_num=32, delimiter=DEFAULT_DELIMITER)
+ assert len(unified) > 1, "semicolon-separated clauses should not land in a single chunk"
+
+ # Pre-fix docx/image/email default (no ASCII ';'): same text, one blob.
+ pre_fix_docx_default = "\n!?。;!?"
+ legacy = naive_merge(sections, chunk_token_num=32, delimiter=pre_fix_docx_default)
+ assert len(legacy) < len(unified), "the unified default must split semicolon prose more than the ';'less drift default"
diff --git a/test/unit_test/rag/test_naive_dispatch.py b/test/unit_test/rag/test_naive_dispatch.py
index f7e0cac2f7..8ea2b43302 100644
--- a/test/unit_test/rag/test_naive_dispatch.py
+++ b/test/unit_test/rag/test_naive_dispatch.py
@@ -213,6 +213,9 @@ def naive_module():
_stub(
"rag.nlp",
+ # naive.py imports the unified delimiter constant (#18562); the
+ # dispatch tests never read the value — only the name must exist.
+ DEFAULT_DELIMITER="\n!?;。;!?",
num_tokens_from_string=lambda s: len((s or "").split()),
find_codec=lambda b: "utf-8",
rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),