RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(rag): unify parser_config delimiter defaults on a single constant (#19058)
euvre4bb522d11d78
diff --git a/api/db/services/file_service.py b/api/db/services/file_service.pyindex 7b3825d48b..245b7852af 100644--- a/api/db/services/file_service.py+++ b/api/db/services/file_service.py@@ -43,6 +43,7 @@ from common.constants import MAXIMUM_PAGE_NUMBER, FileSource, ParserType, TaskStfrom common.misc_utils import get_uuidfrom common.ssrf_guard import assert_url_is_safefrom rag.llm.cv_model import GptV4+from rag.nlp.delim import DEFAULT_DELIMITERclass FileService(CommonService):@@ -710,7 +711,7 @@ class FileService(CommonService):passFACTORY = {ParserType.PRESENTATION.value: presentation, ParserType.PICTURE.value: picture, ParserType.AUDIO.value: audio, ParserType.EMAIL.value: email}- parser_config = {"chunk_token_num": 16096, "delimiter": "\n!?;。;!?", "layout_recognize": layout_recognize or "Plain Text"}+ parser_config = {"chunk_token_num": 16096, "delimiter": DEFAULT_DELIMITER, "layout_recognize": layout_recognize or "Plain Text"}kwargs = {"lang": "English", "callback": dummy, "parser_config": parser_config, "from_page": 0, "to_page": MAXIMUM_PAGE_NUMBER, "tenant_id": current_user.id if current_user else tenant_id}file_type = filename_type(filename)if img_base64 and file_type == FileType.VISUAL.value:diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.pyindex 2f914ffb08..b0dd6556e4 100644--- a/deepdoc/parser/txt_parser.py+++ b/deepdoc/parser/txt_parser.py@@ -20,6 +20,7 @@ import refrom deepdoc.parser.utils import get_textfrom rag.nlp import MergeStrategy, merge_paragraphsfrom rag.nlp.delim import (+ DEFAULT_DELIMITER,compile_delimiter_pattern,normalize_text_newlines,parse_delimiter_field,@@ -27,12 +28,12 @@ from rag.nlp.delim import (class RAGFlowTxtParser:- def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):+ def __call__(self, fnm, binary=None, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):txt = get_text(fnm, binary)return self.parser_txt(txt, chunk_token_num, delimiter, keep_delimiters)@classmethod- def parser_txt(cls, txt, chunk_token_num=128, delimiter="\n!?;。;!?", keep_delimiters=False):+ def parser_txt(cls, txt, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, keep_delimiters=False):if not isinstance(txt, str):raise TypeError("txt type should be str!")txt = normalize_text_newlines(txt)diff --git a/internal/ingestion/component/chunker/sentence_boundary.go b/internal/ingestion/component/chunker/sentence_boundary.goindex 1d0e654027..0632ee1e09 100644--- a/internal/ingestion/component/chunker/sentence_boundary.go+++ b/internal/ingestion/component/chunker/sentence_boundary.go@@ -26,7 +26,7 @@ import "regexp"//// The token chunker's TEXT path (naive_merge port) uses a different delimiter// — sentenceDelimiter in token.go, mirroring Python naive_merge's production-// delimiter "\n!?。;!?" (no ". "). The two constants mirror two distinct-// Python constants and differ only in that English boundary, so they are-// intentionally NOT merged into one.+// default DEFAULT_DELIMITER ("\n!?;。;!?", no ". "). The two constants mirror+// two distinct Python constants and differ only in that English boundary, so+// they are intentionally NOT merged into one.var sentenceBoundaryRe = regexp.MustCompile(`([。!??;!\n]|\. )`)diff --git a/internal/ingestion/component/chunker/sentence_boundary_test.go b/internal/ingestion/component/chunker/sentence_boundary_test.goindex a8da922107..387ca7ea17 100644--- a/internal/ingestion/component/chunker/sentence_boundary_test.go+++ b/internal/ingestion/component/chunker/sentence_boundary_test.go@@ -39,11 +39,12 @@ func TestSentenceBoundary_SharedRegexMatchesPython(t *testing.T) {// TestSentenceBoundary_TokenTextPathDelimiterDistinct pins sentenceDelimiter// (the token-chunker TEXT path / naive_merge port) to Python naive_merge's-// production delimiter "\n!?。;!?" — deliberately WITHOUT the English ". "-// boundary. The two constants mirror two distinct Python delimiters; asserting-// the exact difference documents that this is intentional, not a drift.+// production default DEFAULT_DELIMITER ("\n!?;。;!?", rag/nlp/delim.py) —+// deliberately WITHOUT the English ". " boundary. The two constants mirror two+// distinct Python delimiters; asserting the exact difference documents that+// this is intentional, not a drift.func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {- const want = `(\n|[!?。;!?])`+ const want = `(\n|[!?;。;!?])`if got := sentenceDelimiter.String(); got != want {t.Errorf("sentenceDelimiter = %q, want %q (Python naive_merge delimiter)", got, want)}@@ -52,6 +53,11 @@ func TestSentenceBoundary_TokenTextPathDelimiterDistinct(t *testing.T) {if sentenceDelimiter.MatchString("Hello. World") {t.Error("sentenceDelimiter split on '. ' but naive_merge's delimiter has no '. ' boundary")}+ // The text-path delimiter must split on the ASCII semicolon, matching the+ // unified Python default (issue #18562: the "; " omission was drift).+ if !sentenceDelimiter.MatchString("one; two") {+ t.Error("sentenceDelimiter should split on ';': naive_merge's default includes ';'")+ }if !sentenceBoundaryRe.MatchString("Hello. World") {t.Error("sentenceBoundaryRe should split on '. '")}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex ff362f11fc..a355477448 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -364,14 +364,15 @@ func (c *TokenChunkerComponent) chunkPerSegment(text string, delimPattern, child// sentenceDelimiter is the token-chunker TEXT-path sentence delimiter (the// naive_merge port). It mirrors the delimiter Python's naive_merge receives in-// production: rag/app/naive.py passes "\n!?。;!?", which includes ASCII "!"-// and "?" plus the CJK punctuation "。;!?" but NOT an English ". " boundary.+// production: rag/app/naive.py passes DEFAULT_DELIMITER ("\n!?;。;!?", defined+// once in rag/nlp/delim.py), which includes ASCII "!", "?" and ";" plus the CJK+// punctuation "。;!?" but NOT an English ". " boundary.//// The Title chunker and the token chunker's JSON path use the shared// sentenceBoundaryRe instead (Python _sentence_boundary.py SENTENCE_BOUNDARY_RE,// which adds ". "). The two constants mirror two distinct Python delimiters and// differ only in that English boundary, exactly as Python does.-var sentenceDelimiter = regexp.MustCompile(`(\n|[!?。;!?])`)+var sentenceDelimiter = regexp.MustCompile(`(\n|[!?;。;!?])`)// overlapCut returns the visible-text rune offset where the overlap prefix// begins, mirroring the cut computed inside computeOverlapPrefix. It is splitdiff --git a/rag/app/book.py b/rag/app/book.pyindex d866bc431d..d92a6b4d84 100644--- a/rag/app/book.py+++ b/rag/app/book.py@@ -67,6 +67,9 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=Since a book is long and not all the parts are useful, if it's a PDF,please set up the page ranges for every book in order eliminate negative effects and save elapsed computing time."""+ # Deliberate carve-out from DEFAULT_DELIMITER (#18562): "book" targets+ # Chinese prose, so this kwargs dict and the naive_merge fallback below+ # keep their own Chinese-only delimiter sets, not the unified constant.parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])diff --git a/rag/app/email.py b/rag/app/email.pyindex bde97b207c..e0d0a41f0b 100644--- a/rag/app/email.py+++ b/rag/app/email.py@@ -20,7 +20,7 @@ from email.parser import BytesParserfrom rag.app.naive import chunk as naive_chunkfrom common.constants import MAXIMUM_PAGE_NUMBERimport re-from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks+from rag.nlp import rag_tokenizer, naive_merge, tokenize_chunks, DEFAULT_DELIMITERfrom deepdoc.parser import HtmlParser, TxtParserfrom timeit import default_timer as timerimport io@@ -41,7 +41,7 @@ def chunk(eng = lang.lower() == "english" # is_english(cks)parser_config = kwargs.get("parser_config",- {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"},+ {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"},)doc = {"docnm_kwd": filename,@@ -99,7 +99,7 @@ def chunk(chunks = naive_merge(sections,int(parser_config.get("chunk_token_num", 128)),- parser_config.get("delimiter", "\n!?。;!?"),+ parser_config.get("delimiter", DEFAULT_DELIMITER),)main_res.extend(tokenize_chunks(chunks, doc, eng, None, language=lang))diff --git a/rag/app/laws.py b/rag/app/laws.pyindex 83a3837d7f..3265138c64 100644--- a/rag/app/laws.py+++ b/rag/app/laws.py@@ -24,7 +24,7 @@ from docx.text.paragraph import Paragraphfrom common.constants import ParserType, MAXIMUM_PAGE_NUMBERfrom deepdoc.parser.utils import get_text-from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge+from rag.nlp import bullets_category, remove_contents_table, make_colon_as_title, tokenize_chunks, docx_question_level, tree_merge, DEFAULT_DELIMITERfrom rag.nlp import rag_tokenizer, Nodefrom deepdoc.parser import PdfParser, DocxParser, HtmlParserfrom api.db.joint_services.tenant_model_service import get_composite_model_name_by_id@@ -169,7 +169,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang="""Supported file formats are docx, pdf, txt."""- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})doc = {"docnm_kwd": filename, "title_tks": rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", filename))}doc["title_sm_tks"] = rag_tokenizer.fine_grained_tokenize(doc["title_tks"])pdf_parser = Nonediff --git a/rag/app/manual.py b/rag/app/manual.pyindex 65743ecbbd..7c27f0a5f3 100644--- a/rag/app/manual.py+++ b/rag/app/manual.py@@ -21,7 +21,7 @@ import refrom common.constants import ParserType, MAXIMUM_PAGE_NUMBERfrom io import BytesIOfrom deepdoc.parser.utils import extract_pdf_outlines-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, bullets_category, title_frequency, tokenize_chunks, docx_question_level, attach_media_context, concat_img, DEFAULT_DELIMITERfrom common.token_utils import num_tokens_from_stringfrom deepdoc.parser import PdfParser, DocxParserfrom deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapper, vision_figure_parser_docx_wrapper@@ -139,7 +139,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang="""Only pdf is supported."""- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})pdf_parser = Nonedoc = {"docnm_kwd": filename}doc["title_tks"] = rag_tokenizer.tokenize(re.sub(r"\.[a-zA-Z]+$", "", doc["docnm_kwd"]))diff --git a/rag/app/naive.py b/rag/app/naive.pyindex 51593887b9..eaf941d3e5 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -52,6 +52,7 @@ from common.parser_config_utils import has_mineru_options, normalize_layout_recofrom common.text_utils import normalize_arabic_presentation_formsfrom rag.nlp import (concat_img,+ DEFAULT_DELIMITER,find_codec,naive_merge,naive_merge_with_images,@@ -1067,7 +1068,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=lang = lang or "Chinese"is_english = lang.lower() == "english" # is_english(cks)- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC", "analyze_hyperlink": True})+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC", "analyze_hyperlink": True})child_deli = (parser_config.get("children_delimiter") or "").encode("utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")cust_child_deli = re.findall(r"`([^`]+)`", child_deli)@@ -1133,7 +1134,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=# chunks list[dict]# images list - index of image chunk in chunks- chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), table_context_size, image_context_size)+ chunks, images = naive_merge_docx(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), table_context_size, image_context_size)vision_figure_parser_docx_wrapper_naive(chunks=chunks,@@ -1261,7 +1262,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=elif re.search(r"\.(txt|py|js|java|c|cpp|h|php|go|ts|sh|cs|kt|sql)$", filename, re.IGNORECASE):callback(0.1, "Start to parse.")- sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", "\n!?;。;!?"))+ sections = TxtParser()(filename, binary, parser_config.get("chunk_token_num", 128), parser_config.get("delimiter", DEFAULT_DELIMITER))sections = _normalize_section_text_for_rtl_presentation_forms(sections)logging.info("TxtParser produced %d sections for %s", len(sections), filename)callback(0.8, "Finish parsing.")@@ -1273,7 +1274,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=filename,binary,separate_tables=False,- delimiter=parser_config.get("delimiter", "\n!?;。;!?"),+ delimiter=parser_config.get("delimiter", DEFAULT_DELIMITER),return_section_images=True,)sections = _normalize_section_text_for_rtl_presentation_forms(sections)@@ -1428,10 +1429,10 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=section_images = Noneif section_images:- chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), overlapped_percent)+ chunks, images = naive_merge_with_images(sections, section_images, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)res.extend(tokenize_chunks_with_images(chunks, doc, is_english, images, child_delimiters_pattern=child_deli, language=lang))else:- chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", "\n!?。;!?"), overlapped_percent)+ chunks = naive_merge(sections, int(parser_config.get("chunk_token_num", 128)), parser_config.get("delimiter", DEFAULT_DELIMITER), overlapped_percent)res.extend(tokenize_chunks(chunks, doc, is_english, pdf_parser, child_delimiters_pattern=child_deli, language=lang))diff --git a/rag/app/one.py b/rag/app/one.pyindex 7cde2485e8..e866a8beb1 100644--- a/rag/app/one.py+++ b/rag/app/one.py@@ -20,7 +20,7 @@ import refrom deepdoc.parser.utils import get_textfrom rag.app import naive-from rag.nlp import rag_tokenizer, tokenize+from rag.nlp import rag_tokenizer, tokenize, DEFAULT_DELIMITERfrom deepdoc.parser import PdfParser, ExcelParser, HtmlParserfrom deepdoc.parser.figure_parser import vision_figure_parser_docx_wrapper_naivefrom rag.app.naive import by_plaintext, PARSERS@@ -65,7 +65,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=Supported file formats are docx, pdf, excel, txt.One file forms a chunk which maintains original text order."""- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})eng = lang.lower() == "english" # is_english(cks)if re.search(r"\.docx$", filename, re.IGNORECASE):diff --git a/rag/app/paper.py b/rag/app/paper.pyindex c0ffb29032..97bbc03572 100644--- a/rag/app/paper.py+++ b/rag/app/paper.py@@ -20,7 +20,7 @@ import refrom deepdoc.parser.figure_parser import vision_figure_parser_pdf_wrapperfrom common.constants import ParserType, MAXIMUM_PAGE_NUMBER-from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context+from rag.nlp import rag_tokenizer, tokenize, tokenize_table, add_positions, bullets_category, title_frequency, tokenize_chunks, attach_media_context, DEFAULT_DELIMITERfrom deepdoc.parser import PdfParserimport numpy as npfrom api.db.joint_services.tenant_model_service import get_composite_model_name_by_id@@ -137,7 +137,7 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=Only pdf is supported.The abstract of the paper will be sliced as an entire chunk, and will not be sliced partly."""- parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": "\n!?。;!?", "layout_recognize": "DeepDOC"})+ parser_config = kwargs.get("parser_config", {"chunk_token_num": 512, "delimiter": DEFAULT_DELIMITER, "layout_recognize": "DeepDOC"})if re.search(r"\.pdf$", filename, re.IGNORECASE):layout_recognize_raw = parser_config.get("layout_recognize", "DeepDOC")tenant_id = kwargs.get("tenant_id")diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.pyindex d8742083ed..402669ef76 100644--- a/rag/flow/parser/parser.py+++ b/rag/flow/parser/parser.py@@ -38,6 +38,7 @@ from api.db.joint_services.tenant_model_service import ()from api.db.services.tenant_model_instance_service import TenantModelInstanceServicefrom api.db.services.tenant_model_provider_service import TenantModelProviderService+from rag.nlp.delim import DEFAULT_DELIMITERfrom api.db.services.tenant_model_service import TenantModelServicefrom common import settingsfrom common.constants import LLMType@@ -1148,7 +1149,7 @@ class Parser(ProcessBase):name,blob,conf.get("chunk_token_num", 128),- conf.get("delimiter", "\n!?;。;!?"),+ conf.get("delimiter", DEFAULT_DELIMITER),keep_delimiters=True,)if conf.get("output_format") == "json":diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex e6e48d7ce2..ab101218d9 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -32,6 +32,7 @@ from common.token_utils import num_tokens_from_string# Re-exported below for backwards compatibility; the canonical parser lives# in ``rag.nlp.delim``.from rag.nlp.delim import (+ DEFAULT_DELIMITER,compile_delimiter_pattern,has_wrapped_delimiter,normalize_text_newlines,@@ -1415,7 +1416,7 @@ def _apply_overlap_unconditional(chunks, overlapped_percent):return out-def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):+def naive_merge(sections: str | list, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):"""Split sections into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""if not sections:return []@@ -1478,7 +1479,7 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;return _apply_overlap_unconditional(cks, overlapped_percent)-def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。;!?", overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):+def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter=DEFAULT_DELIMITER, overlapped_percent=0, strategy=MergeStrategy.OVER_CAP):"""Split texts (with images) into chunks. Chunking contract: see ``merge_paragraphs`` (refs #17799)."""if not texts or len(texts) != len(images):return [], []@@ -1834,7 +1835,7 @@ def _merge_cks(cks, chunk_token_num, has_custom):def naive_merge_docx(sections,chunk_token_num=128,- delimiter="\n。;!?",+ delimiter=DEFAULT_DELIMITER,table_context_size=0,image_context_size=0,):diff --git a/rag/nlp/delim.py b/rag/nlp/delim.pyindex 955bc706c6..05f1dbe5b2 100644--- a/rag/nlp/delim.py+++ b/rag/nlp/delim.py@@ -78,6 +78,15 @@ import re# Match a backtick-wrapped token. Case-sensitive on purpose (see #17384)._BACKTICK_RE = re.compile(r"`([^`]+)`")+# The single source of truth for the default value of the+# ``parser_config.delimiter`` field (issue #18562). Before this constant+# existed, the per-callback ``parser_config.get("delimiter", ...)`` defaults+# drifted across several copy-pasted values: txt/markdown shipped this full+# 8-character set, docx/image/email omitted the ASCII ``;``, and+# ``rag/app/book.py`` keeps its own deliberate Chinese-only set. Every backend+# default must reference this constant so the value can never drift again.+DEFAULT_DELIMITER = "\n!?;。;!?"+def normalize_text_newlines(text: str) -> str:"""Normalize CRLF and standalone CR to LF in source text."""diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.pyindex ae5d3fa0f2..dd7bdada6c 100644--- a/rag/svr/task_executor.py+++ b/rag/svr/task_executor.py@@ -82,7 +82,7 @@ from api.db.joint_services.tenant_model_service import get_tenant_default_model_from common.versions import get_ragflow_versionfrom api.db.db_models import close_connectionfrom rag.app import laws, paper, presentation, manual, qa, table, book, resume, picture, naive, one, audio, email, tag-from rag.nlp import search, rag_tokenizer, add_positions+from rag.nlp import search, rag_tokenizer, add_positions, DEFAULT_DELIMITERfrom common.token_utils import num_tokens_from_string, truncatefrom rag.utils.redis_conn import REDIS_CONN, RedisDistributedLock@@ -342,7 +342,7 @@ async def build_chunks(task, progress_callback, on_chunking_start=None):"parser_id": task["parser_id"],"chunk_token_num": parser_config_for_chunk.get("chunk_token_num", 128),"overlapped_percent": normalize_overlapped_percent(parser_config_for_chunk.get("overlapped_percent", 0)),- "delimiter": parser_config_for_chunk.get("delimiter", "\n!?。;!?"),+ "delimiter": parser_config_for_chunk.get("delimiter", DEFAULT_DELIMITER),"from_page": task["from_page"],"to_page": task["to_page"],"language": task["language"],diff --git a/rag/svr/task_executor_refactor/chunk_service.py b/rag/svr/task_executor_refactor/chunk_service.pyindex d792c353cd..a76b71e593 100644--- a/rag/svr/task_executor_refactor/chunk_service.py+++ b/rag/svr/task_executor_refactor/chunk_service.py@@ -40,7 +40,7 @@ from common.misc_utils import thread_pool_execfrom common.float_utils import normalize_overlapped_percentfrom api.db.services.document_service import DocumentServicefrom api.db.services.task_service import TaskService-from rag.nlp import search+from rag.nlp import search, DEFAULT_DELIMITERfrom rag.svr.task_executor_refactor.constants import GRAPH_RAPTOR_FAKE_DOC_IDfrom rag.svr.task_executor_refactor.task_context import TaskContextfrom rag.utils.base64_image import image2id@@ -171,7 +171,7 @@ class ChunkService:"parser_id": ctx.parser_id,"chunk_token_num": ctx.parser_config.get("chunk_token_num", 128),"overlapped_percent": normalize_overlapped_percent(ctx.parser_config.get("overlapped_percent", 0)),- "delimiter": ctx.parser_config.get("delimiter", "\n!?。;!?"),+ "delimiter": ctx.parser_config.get("delimiter", DEFAULT_DELIMITER),"from_page": ctx.from_page,"to_page": ctx.to_page,"language": ctx.language,diff --git a/test/unit_test/rag/nlp/test_default_delimiter_constant.py b/test/unit_test/rag/nlp/test_default_delimiter_constant.pynew file mode 100644index 0000000000..0aa4cc3e94--- /dev/null+++ b/test/unit_test/rag/nlp/test_default_delimiter_constant.py@@ -0,0 +1,151 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Pin the single source of truth for the default ``parser_config.delimiter``.++Before #18562 the backend carried four distinct copy-pasted defaults for the+same field: txt/markdown merged with ``"\\n!?;。;!?"``, docx/image/email with+``"\\n!?。;!?"`` (no ASCII ``;``), the ``naive_merge*`` signatures with the+Chinese-only ``"\\n。;!?"``, and the UI/API layer with ``"\\n"``. The+docx/image/email ``;`` omission was user-visible: English semicolon prose+chunked differently (often one oversized chunk) than the same text as+.txt/.md.++These tests pin the unified state:++* ``rag.nlp.delim.DEFAULT_DELIMITER`` is the 8-character set and parses to+ exactly those delimiters.+* Every backend ``parser_config.get("delimiter", ...)`` fallback, kwargs+ default dict, and ``naive_merge*`` / ``RAGFlowTxtParser`` signature default+ references the constant (AST-checked, so a future copy-pasted literal fails+ loudly instead of drifting silently).+* ``rag/app/book.py`` deliberately keeps its own Chinese-only defaults (the+ kwargs default dict and the ``naive_merge`` fallback) — a product decision,+ not drift, pinned here as a carve-out.+* Behaviourally, semicolon-separated English text splits at ``;`` under the+ unified default but stays whole under the pre-fix docx default.+"""++import ast+import inspect+from pathlib import Path++from rag.nlp import naive_merge, naive_merge_docx, naive_merge_with_images+from rag.nlp.delim import DEFAULT_DELIMITER, parse_delimiter_field++REPO_ROOT = Path(__file__).resolve().parents[4]++# Production files whose ``delimiter`` defaults must reference the constant,+# mapped to the exact number of defaults each file carries. Exact per-file+# counts (rather than an aggregate floor) make a deleted fallback or a newly+# copy-pasted literal fail loudly instead of silently re-drifting.+EXPECTED_DELIMITER_DEFAULTS = {+ "rag/app/naive.py": 6,+ "rag/app/email.py": 2,+ "rag/app/laws.py": 1,+ "rag/app/one.py": 1,+ "rag/app/manual.py": 1,+ "rag/app/paper.py": 1,+ "rag/svr/task_executor.py": 1,+ "rag/svr/task_executor_refactor/chunk_service.py": 1,+ "api/db/services/file_service.py": 1,+ "rag/flow/parser/parser.py": 1,+ "deepdoc/parser/txt_parser.py": 2,+ "rag/nlp/__init__.py": 3,+}+++def test_constant_value():+ assert DEFAULT_DELIMITER == "\n!?;。;!?"+ # Newline + ASCII !, ?, ; + CJK 。;!? — 8 delimiters, deduplicated.+ assert parse_delimiter_field(DEFAULT_DELIMITER) == ["\n", "!", "?", ";", "。", ";", "!", "?"]+++def _delimiter_defaults(tree):+ """Yield (kind, default_node) for every ``delimiter`` default in the module."""+ for node in ast.walk(tree):+ if (+ isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "get"+ and node.args+ and isinstance(node.args[0], ast.Constant)+ and node.args[0].value == "delimiter"+ and len(node.args) > 1+ and isinstance(node.args[1], (ast.Constant, ast.Name))+ ):+ yield ".get", node.args[1]+ if isinstance(node, ast.Dict):+ for key, value in zip(node.keys, node.values):+ if isinstance(key, ast.Constant) and key.value == "delimiter" and isinstance(value, (ast.Constant, ast.Name)):+ yield "dict", value+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):+ pairs = list(zip(node.args.args[-len(node.args.defaults) :], node.args.defaults)) if node.args.defaults else []+ pairs += [(a, a.default) for a in node.args.kwonlyargs if a.default is not None]+ for arg, default in pairs:+ if arg.arg == "delimiter" and not (isinstance(default, ast.Constant) and default.value is None):+ yield f"sig:{node.name}", default+++def test_all_backend_defaults_reference_the_constant():+ for relpath, expected in EXPECTED_DELIMITER_DEFAULTS.items():+ tree = ast.parse((REPO_ROOT / relpath).read_text(encoding="utf-8"))+ defaults = list(_delimiter_defaults(tree))+ assert len(defaults) == expected, (+ f"{relpath}: expected exactly {expected} delimiter defaults, found {len(defaults)}; update EXPECTED_DELIMITER_DEFAULTS in the same change so a removed or new default stays loud"+ )+ for kind, default in defaults:+ assert isinstance(default, ast.Name) and default.id == "DEFAULT_DELIMITER", f"{relpath}: '{kind}' delimiter default is {ast.dump(default)}, expected the DEFAULT_DELIMITER constant"+++def test_book_keeps_its_deliberate_chinese_only_default():+ """``book`` chunking targets Chinese prose; its defaults are a product choice."""+ tree = ast.parse((REPO_ROOT / "rag/app/book.py").read_text(encoding="utf-8"))+ found = list(_delimiter_defaults(tree))+ kwargs_dict_defaults = [d for kind, d in found if kind == "dict"]+ merge_defaults = [d for kind, d in found if kind == ".get"]+ # Two distinct Chinese-only sets, both deliberate carve-outs from+ # DEFAULT_DELIMITER: the chunk() kwargs default dict ("\n!?。;!?")+ # and the naive_merge fallback ("\n。;!?").+ assert any(isinstance(d, ast.Constant) and d.value == "\n!?。;!?" for d in kwargs_dict_defaults)+ assert any(isinstance(d, ast.Constant) and d.value == "\n。;!?" for d in merge_defaults)+++def test_merge_signatures_carry_the_constant():+ for func in (naive_merge, naive_merge_with_images, naive_merge_docx):+ default = inspect.signature(func).parameters["delimiter"].default+ assert default == DEFAULT_DELIMITER, f"{func.__name__} signature default diverges from DEFAULT_DELIMITER"+++def test_semicolon_english_text_splits_under_the_unified_default():+ """The reported symptom: ``;`` clauses must split like they do for txt/md."""+ text = "; ".join(+ [+ "The rain fell gently over the quiet town",+ "the sun broke through the grey clouds",+ "the wind stirred the leaves in the yard",+ "children ran along the wet stone path",+ "a distant bell rang twice before noon",+ ]+ )+ sections = [(text, "")]+ unified = naive_merge(sections, chunk_token_num=32, delimiter=DEFAULT_DELIMITER)+ assert len(unified) > 1, "semicolon-separated clauses should not land in a single chunk"++ # Pre-fix docx/image/email default (no ASCII ';'): same text, one blob.+ pre_fix_docx_default = "\n!?。;!?"+ legacy = naive_merge(sections, chunk_token_num=32, delimiter=pre_fix_docx_default)+ assert len(legacy) < len(unified), "the unified default must split semicolon prose more than the ';'less drift default"diff --git a/test/unit_test/rag/test_naive_dispatch.py b/test/unit_test/rag/test_naive_dispatch.pyindex f7e0cac2f7..8ea2b43302 100644--- a/test/unit_test/rag/test_naive_dispatch.py+++ b/test/unit_test/rag/test_naive_dispatch.py@@ -213,6 +213,9 @@ def naive_module():_stub("rag.nlp",+ # naive.py imports the unified delimiter constant (#18562); the+ # dispatch tests never read the value — only the name must exist.+ DEFAULT_DELIMITER="\n!?;。;!?",num_tokens_from_string=lambda s: len((s or "").split()),find_codec=lambda b: "utf-8",rag_tokenizer=types.SimpleNamespace(tokenize=lambda s: ((s or "").split(), [])),