RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(nlp): collapse 6 delimiter-parser implementations into one canonical helper (#17383) (#17387)
Sd4ceeee4ed5a
diff --git a/deepdoc/parser/markdown_parser.py b/deepdoc/parser/markdown_parser.pyindex b09e5adb13..3f4583d783 100644--- a/deepdoc/parser/markdown_parser.py+++ b/deepdoc/parser/markdown_parser.py@@ -20,6 +20,12 @@ import refrom markdown import markdown+from rag.nlp.delim import (+ compile_delimiter_pattern,+ normalize_text_newlines,+ parse_delimiter_field,+)+logger = logging.getLogger(__name__)@@ -153,13 +159,19 @@ class RAGFlowMarkdownParser:class MarkdownElementExtractor:def __init__(self, markdown_content):- self.markdown_content = markdown_content- self.lines = markdown_content.split("\n")+ # Normalize CRLF/CR so compiled delimiter patterns (which use LF)+ # match Windows-line-ending source the same as Unix source.+ self.markdown_content = normalize_text_newlines(markdown_content)+ self.lines = self.markdown_content.split("\n")def get_delimiters(self, delimiters):- toks = re.findall(r"`([^`]+)`", delimiters)- toks = sorted(set(toks), key=lambda x: -len(x))- return "|".join(re.escape(t) for t in toks if t)+ # Delegate to the canonical parser (#17383). The previous+ # implementation matched only backtick-wrapped tokens and dropped+ # bare characters, which silently made the shipped default+ # delimiter field a no-op for the markdown path. The helper honors+ # both bare chars and wrapped tokens, so the same field produces+ # the same splits in every file type.+ return compile_delimiter_pattern(parse_delimiter_field(delimiters))def _get_fence_marker(self, line):match = re.match(r"^[ \t]{0,3}(?P<fence>`{3,}|~{3,})(?:.*)$", line)diff --git a/deepdoc/parser/txt_parser.py b/deepdoc/parser/txt_parser.pyindex 60b310cd46..9e51ffc961 100644--- a/deepdoc/parser/txt_parser.py+++ b/deepdoc/parser/txt_parser.py@@ -20,6 +20,11 @@ import refrom common.token_utils import num_tokens_from_stringfrom deepdoc.parser.utils import get_textfrom rag.nlp import _split_oversized_unit+from rag.nlp.delim import (+ compile_delimiter_pattern,+ normalize_text_newlines,+ parse_delimiter_field,+)class RAGFlowTxtParser:@@ -33,10 +38,9 @@ class RAGFlowTxtParser:raise TypeError("txt type should be str!")cks = [""]tk_nums = [0]- delimiter = delimiter.encode("utf-8").decode("unicode_escape").encode("latin1").decode("utf-8")def add_chunk(t):- nonlocal cks, tk_nums, delimiter+ nonlocal cks, tk_numstnum = num_tokens_from_string(t)if cks[-1] == "":@@ -54,21 +58,17 @@ class RAGFlowTxtParser:cks.append(t)tk_nums.append(tnum)- dels = []- s = 0- for m in re.finditer(r"`([^`]+)`", delimiter):- f, m_t = m.span()- dels.append(m.group(1))- dels.extend(list(delimiter[s:f]))- s = m_t- if s < len(delimiter):- dels.extend(list(delimiter[s:]))- dels = [re.escape(d) for d in dels if d]- dels = [d for d in dels if d]- dels = "|".join(dels)- secs = re.split(r"(%s)" % dels, txt)+ txt = normalize_text_newlines(txt)+ parsed_dels = parse_delimiter_field(delimiter)+ dels = compile_delimiter_pattern(parsed_dels)+ logging.debug(+ "RAGFlowTxtParser.parser_txt: delimiter_count=%d, splitting=%s",+ len(parsed_dels),+ bool(dels),+ )+ secs = re.split(r"(%s)" % dels, txt) if dels else [txt]for sec in secs:- if re.match(f"^{dels}$", sec):+ if dels and re.match(f"^{dels}$", sec):continueif not sec:continuediff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex 79d2b996aa..0ee111d88a 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -19,11 +19,11 @@ package chunkerimport ("fmt""regexp"- "sort""strings""ragflow/internal/agent/runtime""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/chunk""ragflow/internal/tokenizer")@@ -76,87 +76,14 @@ func stringListFromAny(in []any) []string {// regex / split helpers// ----------------------------------------------------------------------------// backtickDelimRE extracts backtick-wrapped delimiter tokens.-// Case-sensitive on purpose (mirrors Python after #17384 / PR #17386):-// never add (?i) here — delimiter matching must preserve letter casing.-var backtickDelimRE = regexp.MustCompile("`([^`]+)`")--// escapeAndSort QuoteMeta-escapes tokens and sorts longest-first.-// Empty tokens are dropped. When dedup is true, exact duplicate tokens-// are collapsed (first occurrence wins). Matching stays case-sensitive.-func escapeAndSort(tokens []string, dedup bool) []string {- out := make([]string, 0, len(tokens))- var seen map[string]struct{}- if dedup {- seen = make(map[string]struct{}, len(tokens))- }- for _, tok := range tokens {- if tok == "" {- continue- }- if dedup {- if _, ok := seen[tok]; ok {- continue- }- seen[tok] = struct{}{}- }- out = append(out, regexp.QuoteMeta(tok))- }- sort.SliceStable(out, func(i, j int) bool { return len(out[i]) > len(out[j]) })- return out-}--// getDelimiters ports rag.nlp.get_delimiters (rag/nlp/__init__.py).-//-// It walks a single delimiter string, pulling out backtick-wrapped tokens-// and every bare character outside those spans, then returns a-// length-sorted, QuoteMeta-escaped alternation suitable for regexp.Split.-// Matching is case-sensitive: "a" does not match "A", and "`end`" does not-// match "End" / "END".-func getDelimiters(delimiters string) string {- var dels []string- s := 0- for _, m := range backtickDelimRE.FindAllStringSubmatchIndex(delimiters, -1) {- // m = [fullStart, fullEnd, g1Start, g1End]- f, t := m[0], m[1]- dels = append(dels, delimiters[m[2]:m[3]])- for _, r := range delimiters[s:f] {- dels = append(dels, string(r))- }- s = t- }- if s < len(delimiters) {- for _, r := range delimiters[s:] {- dels = append(dels, string(r))- }- }- // Python get_delimiters does not dedup; preserve that behavior.- return strings.Join(escapeAndSort(dels, false), "|")-}--// compileDelimPattern builds an alternation from backtick-wrapped tokens-// across delimiter entries (mirrors Python _compile_delimiter_pattern).-// Each entry is scanned independently so token boundaries cannot span-// adjacent slice elements. Extraction is case-sensitive — see backtickDelimRE.-//-// Plain (non-backtick) delimiters are not compiled here; callers that-// need bare-char splitting use getDelimiters (naive_merge path).+// compileDelimPattern compiles a TokenChunker-style []string delimiter list.+// Only backtick-wrapped entries produce an active pattern (Python+// token_chunker / rag/nlp/delim list helper). Plain entries are ignored here+// and used by mergeByTokenSize for sentence-level splitting when no active+// pattern exists. Canonical single-string parser_config.delimiter parsing+// lives in ragflow/internal/parser/chunk (ParseDelimiterField).func compileDelimPattern(delims []string) *regexp.Regexp {- var tokens []string- for _, d := range delims {- if d == "" {- continue- }- for _, m := range backtickDelimRE.FindAllStringSubmatch(d, -1) {- tokens = append(tokens, m[1])- }- }- // Python _compile_delimiter_pattern dedups via set(...).- custom := escapeAndSort(tokens, true)- if len(custom) == 0 {- return nil- }- return regexp.MustCompile(strings.Join(custom, "|"))+ return chunk.CompileDelimiterListPattern(delims)}// splitKeepingDelim mirrors Python token_chunker._split_text_by_patterndiff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.goindex 16ea51b610..fc097221c1 100644--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go@@ -32,8 +32,18 @@ import ("regexp""strings""testing"++ "ragflow/internal/parser/chunk")+func getDelimiters(delimiters string) string {+ p := chunk.CompileDelimiterPattern(chunk.ParseDelimiterField(delimiters))+ if p == nil {+ return ""+ }+ return p.String()+}+func TestGetDelimiters_BareCharAReturnsLiteralPattern(t *testing.T) {// Bare-char delimiter "a" must produce the pattern "a", not "a|A".if got, want := getDelimiters("a"), "a"; got != want {@@ -219,15 +229,20 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {}}-func TestBacktickDelimRE_HasNoIgnoreCaseFlag(t *testing.T) {- // Structural guard: the shared backtick extractor must stay case-sensitive.- src := backtickDelimRE.String()- if strings.Contains(src, "(?i)") || strings.HasPrefix(src, "(?i)") {- t.Fatalf("backtickDelimRE must not use (?i); got %q", src)+func TestBacktickDelimiterIsCaseSensitive(t *testing.T) {+ // Extraction and compiled pattern must both preserve letter casing.+ parsed := chunk.ParseDelimiterField("`End`")+ if len(parsed) != 1 || parsed[0] != "End" {+ t.Fatalf("ParseDelimiterField(`End`) = %#v, want [End]", parsed)+ }+ p := chunk.CompileDelimiterPattern(parsed)+ if p == nil {+ t.Fatal("CompileDelimiterPattern returned nil")+ }+ if !p.MatchString("End") || p.MatchString("end") || p.MatchString("END") {+ t.Fatalf("pattern %q is not case-sensitive", p.String())}- // Sanity: the pattern still extracts backtick contents.- m := backtickDelimRE.FindStringSubmatch("`End`")- if len(m) < 2 || m[1] != "End" {- t.Fatalf("backtickDelimRE(`End`) = %#v, want group1=End", m)+ if strings.Contains(p.String(), "(?i)") {+ t.Fatalf("compiled pattern must not carry (?i); got %q", p.String())}}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 07ca2e815b..7d86b611be 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -23,10 +23,11 @@// mirroring Python's normalize_overlapped_percent), table_context_size ≥ 0,// image_context_size ≥ 0. enum/range checks live in param.Check.//-// - DELIMITER PARSING mirrors python `_compile_delimiter_pattern`:-// entries wrapped in backticks (e.g. "`\\n\\n`") are treated as-// regex split points; plain strings are regex-escaped and joined-// into the same alternation. Empty entries are filtered.+// - DELIMITER PARSING for the TokenChunker list API mirrors Python+// token_chunker: only entries wrapped in backticks (e.g. "`\\n\\n`")+// produce an active split pattern. Plain list entries are not+// compiled into the pattern. Single-string parser_config.delimiter+// parsing lives in ragflow/internal/parser/chunk (ParseDelimiterField).//// - CHILDREN DELIMITERS (the secondary split) is implemented via the// shared splitKeepingDelim helper; emitted chunks carry the parent@@ -63,12 +64,13 @@ import ("strings""sync"+ "gorm.io/gorm""ragflow/internal/agent/runtime"deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"- "gorm.io/gorm"+ "ragflow/internal/parser/chunk")const ComponentNameTokenChunker = "TokenChunker"@@ -1075,14 +1077,9 @@ func hasActiveDelimiter(p *regexp.Regexp) bool {// hasCustomDelim reports whether any delimiter uses backtick syntax// (`pattern`). Python's naive_merge skips token-size merging when-// custom delimiters are present (naive_merge:1194-1213).+// custom delimiters are present. Delegates to the canonical helper.func hasCustomDelim(delims []string) bool {- for _, d := range delims {- if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {- return true- }- }- return false+ return chunk.HasCustomDelimiterList(delims)}// applyChildrenDelim mirrors token_chunker.py:325-334.diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.gonew file mode 100644index 0000000000..aebadbd60a--- /dev/null+++ b/internal/parser/chunk/delim.go@@ -0,0 +1,194 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunk++import (+ "regexp"+ "sort"+ "strings"+ "unicode/utf8"+)++// Canonical parser for the parser_config.delimiter field.+//+// Mirrors Python rag/nlp/delim.py (#17383). A delimiter field is a string+// with the grammar:+//+// delimiter_field := token*+// token := backtick_wrapped | bare_char+// backtick_wrapped := "`" bare_char+ "`"+// bare_char := any single Unicode character except "`"+//+// Semantics:+// 1. Characters between matching backticks form one multi-character delimiter.+// 2. Any character outside backticks is its own single-character delimiter.+// 3. Results are deduplicated and sorted longest-first (stable for equal length).+// 4. CRLF and standalone CR are normalized to LF before parsing.+// 5. Matching is case-sensitive.++// backtickWrappedRE matches a backtick-wrapped multi-character token.+// Case-sensitive on purpose (see #17384).+var backtickWrappedRE = regexp.MustCompile("`([^`]+)`")++// NormalizeTextNewlines converts CRLF and standalone CR to LF.+func NormalizeTextNewlines(text string) string {+ if text == "" {+ return text+ }+ text = strings.ReplaceAll(text, "\r\n", "\n")+ return strings.ReplaceAll(text, "\r", "\n")+}++// HasWrappedDelimiter reports whether the delimiter field contains at least+// one backtick-wrapped token. Used to decide the historical "custom delimiter"+// mode that bypasses chunk_token_num. Separate from whether any delimiter is+// present after parsing (bare single-character delimiters still split).+func HasWrappedDelimiter(s string) bool {+ if s == "" {+ return false+ }+ return backtickWrappedRE.MatchString(s)+}++// ParseDelimiterField parses the delimiter field into delimiter strings.+//+// Returns nil for an empty field. Whitespace characters are valid+// single-character delimiters. Output is sorted longest-first and+// deduplicated while preserving first-occurrence order for equal-length+// items (the sort is stable). CRLF/CR inside the field are normalized to LF.+func ParseDelimiterField(s string) []string {+ if s == "" {+ return nil+ }+ normalized := NormalizeTextNewlines(s)++ var delimiters []string+ seen := make(map[string]struct{})+ cursor := 0+ for _, loc := range backtickWrappedRE.FindAllStringSubmatchIndex(normalized, -1) {+ // loc: [fullStart, fullEnd, groupStart, groupEnd]+ start, end := loc[0], loc[1]+ for _, ch := range runesOf(normalized[cursor:start]) {+ if _, ok := seen[ch]; ok {+ continue+ }+ seen[ch] = struct{}{}+ delimiters = append(delimiters, ch)+ }+ token := normalized[loc[2]:loc[3]]+ if token != "" {+ if _, ok := seen[token]; !ok {+ seen[token] = struct{}{}+ delimiters = append(delimiters, token)+ }+ }+ cursor = end+ }+ for _, ch := range runesOf(normalized[cursor:]) {+ if _, ok := seen[ch]; ok {+ continue+ }+ seen[ch] = struct{}{}+ delimiters = append(delimiters, ch)+ }++ sort.SliceStable(delimiters, func(i, j int) bool {+ return utf8.RuneCountInString(delimiters[i]) > utf8.RuneCountInString(delimiters[j])+ })+ return delimiters+}++// CompileDelimiterPattern builds an alternation regex from delimiter strings.+//+// Each delimiter is regexp.QuoteMeta'd so whitespace and metacharacters match+// literally. Returns nil when delimiters is empty or yields no non-empty+// entries. The pattern is intended for use with a capturing split (so+// delimiters appear in the split output) or FindAllStringIndex.+func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {+ if len(delimiters) == 0 {+ return nil+ }+ escaped := make([]string, 0, len(delimiters))+ for _, d := range delimiters {+ if d == "" {+ continue+ }+ escaped = append(escaped, regexp.QuoteMeta(d))+ }+ if len(escaped) == 0 {+ return nil+ }+ return regexp.MustCompile(strings.Join(escaped, "|"))+}++// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter+// list. Entries wrapped in backticks contribute their inner content as a+// split pattern (Python token_chunker / historical Go compileDelimPattern).+// Bare list entries are ignored for the active pattern — they are only used+// by merge paths when no custom pattern exists.+//+// Prefer ParseDelimiterField for the single-string parser_config.delimiter+// field. This helper exists for the dataflow list API.+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {+ var custom []string+ for _, d := range delims {+ if d == "" {+ continue+ }+ if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {+ inner := d[1 : len(d)-1]+ if inner == "" {+ continue+ }+ custom = append(custom, inner)+ }+ }+ if len(custom) == 0 {+ return nil+ }+ sort.SliceStable(custom, func(i, j int) bool {+ return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])+ })+ escaped := make([]string, 0, len(custom))+ for _, d := range custom {+ escaped = append(escaped, regexp.QuoteMeta(d))+ }+ return regexp.MustCompile(strings.Join(escaped, "|"))+}++// HasCustomDelimiterList reports whether any entry in a TokenChunker-style+// delimiter list uses backtick syntax.+func HasCustomDelimiterList(delims []string) bool {+ for _, d := range delims {+ if strings.HasPrefix(d, "`") && strings.HasSuffix(d, "`") && len(d) >= 2 {+ return true+ }+ }+ return false+}++// runesOf splits s into individual Unicode characters (as strings).+func runesOf(s string) []string {+ if s == "" {+ return nil+ }+ out := make([]string, 0, utf8.RuneCountInString(s))+ for _, r := range s {+ out = append(out, string(r))+ }+ return out+}diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.gonew file mode 100644index 0000000000..7a1034200b--- /dev/null+++ b/internal/parser/chunk/delim_test.go@@ -0,0 +1,185 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunk++import (+ "reflect"+ "regexp"+ "testing"+)++func TestNormalizeTextNewlines(t *testing.T) {+ tests := []struct {+ in, want string+ }{+ {"", ""},+ {"a\nb", "a\nb"},+ {"a\r\nb", "a\nb"},+ {"a\rb", "a\nb"},+ {"a\r\nb\rc", "a\nb\nc"},+ }+ for _, tc := range tests {+ if got := NormalizeTextNewlines(tc.in); got != tc.want {+ t.Errorf("NormalizeTextNewlines(%q) = %q, want %q", tc.in, got, tc.want)+ }+ }+}++func TestHasWrappedDelimiter(t *testing.T) {+ if HasWrappedDelimiter("") {+ t.Fatal("empty should be false")+ }+ if HasWrappedDelimiter("!;") {+ t.Fatal("bare chars should be false")+ }+ if !HasWrappedDelimiter("`##`") {+ t.Fatal("wrapped should be true")+ }+ if !HasWrappedDelimiter("\n`##`;") {+ t.Fatal("mixed should be true")+ }+ if !HasWrappedDelimiter("`;`") {+ t.Fatal("wrapped single char should be true")+ }+}++func TestParseDelimiterField(t *testing.T) {+ tests := []struct {+ name string+ field string+ want []string+ }{+ {"empty", "", nil},+ {"single bare", "!", []string{"!"}},+ {"bare pair", "!?", []string{"!", "?"}},+ {"space", " ", []string{" "}},+ {"newline", "\n", []string{"\n"}},+ {"crlf field", "\r\n", []string{"\n"}},+ {"bare cr", "\r", []string{"\n"}},+ {"wrapped end", "`end`", []string{"end"}},+ {"wrapped hierarchy", "`###``##``#`", []string{"###", "##", "#"}},+ {"tooltip example", "\n`##`;", []string{"##", "\n", ";"}},+ {"dedupe", "`a`a`a`", []string{"a"}},+ {"wrapped double newline", "`\n\n`", []string{"\n\n"}},+ {"crlf wrapped", "`\r\n`", []string{"\n"}},+ {"shipped default", "\n!?;。;!?", []string{"\n", "!", "?", ";", "。", ";", "!", "?"}},+ {"chinese", "。;", []string{"。", ";"}},+ {"mixed order equal length", "`##`#\n", []string{"##", "#", "\n"}},+ {"empty backticks bare", "``", []string{"`"}},+ {"double space bare dedupe", " ", []string{" "}},+ {"wrapped double space", "` `", []string{" "}},+ }+ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ got := ParseDelimiterField(tc.field)+ if !reflect.DeepEqual(got, tc.want) {+ t.Errorf("ParseDelimiterField(%q) = %#v, want %#v", tc.field, got, tc.want)+ }+ })+ }+}++func TestCompileDelimiterPattern(t *testing.T) {+ if CompileDelimiterPattern(nil) != nil {+ t.Fatal("empty list should yield nil")+ }+ if CompileDelimiterPattern([]string{}) != nil {+ t.Fatal("empty slice should yield nil")+ }++ pat := CompileDelimiterPattern([]string{"##", "#"})+ if pat == nil {+ t.Fatal("expected pattern")+ }+ if got := pat.FindString("###"); got != "##" {+ t.Errorf("longest match: got %q, want ##", got)+ }++ // Metacharacters match literally.+ for _, ch := range []string{".", "(", "?", "+"} {+ p := CompileDelimiterPattern([]string{ch})+ if p == nil || !p.MatchString(ch) {+ t.Errorf("pattern for %q should match itself", ch)+ }+ if ch != "." && p.MatchString("z") {+ t.Errorf("pattern for %q should not match z", ch)+ }+ }+}++func TestCompileDelimiterPatternShippedDefault(t *testing.T) {+ pat := CompileDelimiterPattern(ParseDelimiterField("\n!?;。;!?"))+ if pat == nil {+ t.Fatal("expected pattern")+ }+ for _, ch := range []string{"\n", "!", "?", ";", "。", ";", "!", "?"} {+ if !pat.MatchString(ch) {+ t.Errorf("default pattern must match %q", ch)+ }+ }+ if pat.MatchString("a") {+ t.Error("default pattern must not match a")+ }+}++func TestCompileDelimiterPatternCaseSensitive(t *testing.T) {+ pat := CompileDelimiterPattern(ParseDelimiterField("a"))+ parts := regexp.MustCompile("("+pat.String()+")").Split("AaBb", -1)+ // Split removes matches; "A" + "Bb" with "a" consumed.+ if len(parts) < 2 {+ t.Fatalf("parts=%v", parts)+ }+ // Only lowercase a splits.+ re := regexp.MustCompile("(" + pat.String() + ")")+ got := re.Split("AaBb", -1)+ want := []string{"A", "Bb"}+ // re.Split in Go does not keep delimiters; with capturing group behavior+ // differs — just check case sensitivity via FindAll.+ matches := pat.FindAllString("AaBb", -1)+ if !reflect.DeepEqual(matches, []string{"a"}) {+ t.Errorf("matches=%v want [a]; split parts=%v want-ish %v", matches, got, want)+ }+}++func TestCompileDelimiterListPattern(t *testing.T) {+ // Bare list entries are ignored (TokenChunker list API).+ if CompileDelimiterListPattern([]string{"\n", "!"}) != nil {+ t.Fatal("bare list entries should not produce a pattern")+ }+ pat := CompileDelimiterListPattern([]string{"`##`", "`#`"})+ if pat == nil {+ t.Fatal("expected pattern from wrapped list entries")+ }+ if got := pat.FindString("###"); got != "##" {+ t.Errorf("got %q want ##", got)+ }++ // Verify unescaped rune length sorting order: "a.b" (3 runes) vs "ab" (2 runes with meta chars)+ patMeta := CompileDelimiterListPattern([]string{"`a`", "`a.b`"})+ if patMeta.String() != `a\.b|a` {+ t.Errorf("got pattern %q, want %q", patMeta.String(), `a\.b|a`)+ }+}++func TestHasCustomDelimiterList(t *testing.T) {+ if HasCustomDelimiterList([]string{"\n", "!"}) {+ t.Fatal("bare list should be false")+ }+ if !HasCustomDelimiterList([]string{"\n", "`##`"}) {+ t.Fatal("wrapped list entry should be true")+ }+}diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex a743637c31..b6edcfcd8b 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -28,6 +28,15 @@ from word2number import w2nfrom common.token_utils import num_tokens_from_string+# Re-exported below for backwards compatibility; the canonical parser lives+# in ``rag.nlp.delim``.+from rag.nlp.delim import (+ compile_delimiter_pattern,+ has_wrapped_delimiter,+ normalize_text_newlines,+ parse_delimiter_field,+)+__all__ = ["rag_tokenizer"]all_codecs = [@@ -1290,7 +1299,7 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;if isinstance(sections[0], str):sections = [(s, "") for s in sections]# Normalize line endings so delimiter ``\n`` matches ``\r\n`` and standalone ``\r``.- sections = [(s.replace("\r\n", "\n").replace("\r", "\n"), pos) for s, pos in sections]+ sections = [(normalize_text_newlines(s), pos) for s, pos in sections]cks = [""]tk_nums = [0]@@ -1304,16 +1313,22 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;cks.append(text)tk_nums.append(tk_num)- custom_delimiters = [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter)]- has_custom = bool(custom_delimiters)+ # Parse the delimiter field once, via the canonical helper (#17383).+ # `has_custom` means the field contains a backtick-wrapped token — the+ # historical signal that chunk_token_num should be bypassed. Splitting+ # itself uses every parsed delimiter (bare and wrapped).+ parsed_dels = parse_delimiter_field(delimiter)+ has_custom = has_wrapped_delimiter(delimiter)if has_custom:# Custom delimiters ignore chunk_token_num: each segment is its own chunk.- custom_pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))+ custom_pattern = compile_delimiter_pattern(parsed_dels)cks, tk_nums = [], []for sec, pos in sections:- split_sec = re.split(r"(%s)" % custom_pattern, sec, flags=re.DOTALL)+ split_sec = re.split(r"(%s)" % custom_pattern, sec, flags=re.DOTALL) if custom_pattern else [sec]for sub_sec in split_sec:- if re.fullmatch(custom_pattern, sub_sec or ""):+ if not sub_sec:+ continue+ if custom_pattern and re.fullmatch(custom_pattern, sub_sec):continuetext = "\n" + sub_seclocal_pos = pos@@ -1329,7 +1344,7 @@ def naive_merge(sections: str | list, chunk_token_num=128, delimiter="\n。;# Units that exceed the budget after the regex split (a single long line with# no delimiter, e.g. PDF / .txt runs of unbroken text) are sub-split on# whitespace atoms with a character-window fallback, mirroring the html path.- dels = get_delimiters(delimiter)+ dels = compile_delimiter_pattern(parsed_dels)for sec, pos in sections:sec_text = "\n" + secif num_tokens_from_string(sec_text) <= chunk_token_num:@@ -1386,20 +1401,26 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。result_images.append(image)tk_nums.append(tk_num)- custom_delimiters = [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter)]- has_custom = bool(custom_delimiters)+ # Parse the delimiter field once, via the canonical helper (#17383).+ # See the matching block in ``naive_merge`` for the rationale on+ # `has_custom` (backtick-wrapped tokens opt into chunk-token-num bypass).+ parsed_dels = parse_delimiter_field(delimiter)+ has_custom = has_wrapped_delimiter(delimiter)if has_custom:# Custom delimiters ignore chunk_token_num: each segment is its own chunk.- custom_pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))+ custom_pattern = compile_delimiter_pattern(parsed_dels)cks, result_images, tk_nums = [], [], []for text, image in zip(texts, images):text_str = text[0] if isinstance(text, tuple) else textif text_str is None:text_str = ""+ text_str = normalize_text_newlines(text_str)text_pos = text[1] if isinstance(text, tuple) and len(text) > 1 else ""- split_sec = re.split(r"(%s)" % custom_pattern, text_str)+ split_sec = re.split(r"(%s)" % custom_pattern, text_str) if custom_pattern else [text_str]for sub_sec in split_sec:- if re.fullmatch(custom_pattern, sub_sec or ""):+ if not sub_sec:+ continue+ if custom_pattern and re.fullmatch(custom_pattern, sub_sec):continuetext_seg = "\n" + sub_seclocal_pos = text_pos@@ -1416,7 +1437,7 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。# along on every piece (concat_img dedupes when pieces re-merge into a chunk).# Units still exceeding the budget after the regex split are sub-split on# whitespace atoms so they cannot blow past the token cap.- dels = get_delimiters(delimiter)+ dels = compile_delimiter_pattern(parsed_dels)for text, image in zip(texts, images):# if text is tuple, unpack itif isinstance(text, tuple):@@ -1425,7 +1446,7 @@ def naive_merge_with_images(texts, images, chunk_token_num=128, delimiter="\n。else:text_str = text or ""text_pos = ""-+ text_str = normalize_text_newlines(text_str)text_seg = "\n" + text_strif num_tokens_from_string(text_seg) <= chunk_token_num:add_chunk(text_seg, image, text_pos)@@ -1524,15 +1545,14 @@ def _build_cks(sections, delimiter):tables = []images = []- # extract custom delimiters wrapped by backticks: `##`, `---`, etc.- custom_delimiters = [m.group(1) for m in re.finditer(r"`([^`]+)`", delimiter)]- has_custom = bool(custom_delimiters)-- if has_custom:- # escape delimiters and build alternation pattern, longest first- custom_pattern = "|".join(re.escape(t) for t in sorted(set(custom_delimiters), key=len, reverse=True))- # capture delimiters so they appear in re.split results- pattern = r"(%s)" % custom_pattern+ # Parse the delimiter field once, via the canonical helper (#17383).+ # Split on every parsed delimiter (bare and wrapped). `has_custom`+ # only controls whether _merge_cks bypasses chunk_token_num (wrapped+ # token present in the original field).+ parsed_dels = parse_delimiter_field(delimiter)+ has_custom = has_wrapped_delimiter(delimiter)+ split_pattern = compile_delimiter_pattern(parsed_dels)+ pattern = r"(%s)" % split_pattern if split_pattern else ""seg = ""for text, image, table in sections:@@ -1540,7 +1560,7 @@ def _build_cks(sections, delimiter):if not text:text = ""else:- text = "\n" + str(text)+ text = "\n" + normalize_text_newlines(str(text))if table:# table chunk@@ -1571,12 +1591,16 @@ def _build_cks(sections, delimiter):images.append(idx)continue- # pure text chunk(s)- if has_custom:+ # pure text chunk(s) — split on every parsed delimiter when present+ if split_pattern:split_sec = re.split(pattern, text)for sub_sec in split_sec:- # ① empty or whitespace-only segment → flush current buffer- if not sub_sec or not sub_sec.strip():+ if not sub_sec:+ continue++ # ① matched delimiter (exact capture; do not strip — wrapped+ # whitespace delimiters such as `` ` ` `` or `\n` must match here)+ if re.fullmatch(split_pattern, sub_sec):if seg and seg.strip():s = seg.strip()cks.append(@@ -1590,8 +1614,8 @@ def _build_cks(sections, delimiter):seg = ""continue- # ② matched custom delimiter (allow surrounding whitespace)- if re.fullmatch(custom_pattern, sub_sec.strip()):+ # ② empty or whitespace-only ordinary segment → flush current buffer+ if not sub_sec.strip():if seg and seg.strip():s = seg.strip()cks.append(@@ -1619,8 +1643,8 @@ def _build_cks(sections, delimiter):})- # final flush after loop (only when custom delimiters are used)- if has_custom and seg and seg.strip():+ # final flush after loop (only when delimiters were used for splitting)+ if split_pattern and seg and seg.strip():s = seg.strip()cks.append({@@ -1765,25 +1789,6 @@ def extract_between(text: str, start_tag: str, end_tag: str) -> list[str]:return re.findall(pattern, text, flags=re.DOTALL)-def get_delimiters(delimiters: str):- dels = []- s = 0- for m in re.finditer(r"`([^`]+)`", delimiters):- f, t = m.span()- dels.append(m.group(1))- dels.extend(list(delimiters[s:f]))- s = t- if s < len(delimiters):- dels.extend(list(delimiters[s:]))-- dels.sort(key=lambda x: -len(x))- dels = [re.escape(d) for d in dels if d]- dels = [d for d in dels if d]- dels_pattern = "|".join(dels)-- return dels_pattern--class Node:def __init__(self, level, depth=-1, texts=None):self.level = leveldiff --git a/rag/nlp/delim.py b/rag/nlp/delim.pynew file mode 100644index 0000000000..955bc706c6--- /dev/null+++ b/rag/nlp/delim.py@@ -0,0 +1,170 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Canonical parser for the ``parser_config.delimiter`` field.++Background+----------+The single string field ``parser_config.delimiter`` is consumed by several+parser implementations depending only on the file extension. Before this+module existed, six implementations diverged on:++ * whether bare (non-backtick) characters are honored+ * dedupe behavior+ * sort order+ * CRLF / CR normalization+ * whether ``re.I`` is applied+ * the ``re.escape`` round-trip dance in ``txt_parser``++This module owns the canonical parsing rule. All six implementations now+call :func:`parse_delimiter_field` and :func:`compile_delimiter_pattern`.++Parsing rule+------------+A "delimiter field" is a string with the following grammar::++ delimiter_field := token*+ token := backtick_wrapped | bare_char+ backtick_wrapped := "`" bare_char+ "`"+ bare_char := any single Unicode character except "`"++Semantics:++ 1. Any character(s) between matching backticks is one multi-character+ delimiter.+ 2. Any character outside backticks is its own single-character+ delimiter.+ 3. The two are combined, deduplicated, and sorted longest-first so+ ``##`` matches before ``#``.+ 4. ``\\r\\n`` and standalone ``\\r`` are normalized to ``\\n`` at the+ top of :func:`parse_delimiter_field` so Windows-line-ending+ documents produce identical splits to Unix-line-ending ones.+ 5. No ``re.I`` is used. Delimiter matching is case-sensitive.++Returns+-------+:func:`parse_delimiter_field` returns a ``list[str]`` of raw delimiter+strings (sorted longest-first, deduplicated, CRLF-normalized).+:func:`compile_delimiter_pattern` takes that list and returns a regex+alternation pattern with ``re.escape`` applied, ready for+``re.split(r"(%s)" % pattern, ...)``.++Frontend parity+---------------+The web UI preview in ``web/src/utils/delimiter-preview.ts``+(``parseDelimitersForDisplay``) follows the same parsing rule+(normalization, dedupe, longest-first order) and applies whitespace+glyph substitution only for display.+"""++from __future__ import annotations++import logging+import re++# Match a backtick-wrapped token. Case-sensitive on purpose (see #17384).+_BACKTICK_RE = re.compile(r"`([^`]+)`")+++def normalize_text_newlines(text: str) -> str:+ """Normalize CRLF and standalone CR to LF in source text."""+ if not text:+ return text+ return text.replace("\r\n", "\n").replace("\r", "\n")+++def has_wrapped_delimiter(s: str) -> bool:+ """True when the delimiter field contains at least one backtick-wrapped token.++ Used to decide the historical "custom delimiter" mode that bypasses+ ``chunk_token_num``. Separate from whether any delimiter is present+ after parsing (bare single-character delimiters still split).+ """+ if not s:+ return False+ return _BACKTICK_RE.search(s) is not None+++def parse_delimiter_field(s: str) -> list[str]:+ """Parse the delimiter field into a list of delimiter strings.++ Returns an empty list for an empty field. Whitespace characters are+ treated as valid single-character delimiters.++ The output is sorted longest-first and deduplicated while+ preserving the first-occurrence order for equal-length items (the+ sort is stable). CRLF / CR line endings inside the field are+ normalized to LF so a user typing ``"\\r\\n"`` and a user typing+ ``"\\n"`` get the same effective delimiter (a single newline).+ """+ if not s:+ return []++ # CRLF normalization: \r\n → \n, then standalone \r → \n. We do this+ # before parsing so the parser never sees a \r in either bare-char+ # position or backtick-wrapped content.+ normalized = normalize_text_newlines(s)++ # Insertion-ordered dedupe so equal-length items keep their first-+ # occurrence order, which is then preserved by the stable sort below.+ delimiters: list[str] = []+ seen: set[str] = set()+ cursor = 0+ for match in _BACKTICK_RE.finditer(normalized):+ start, end = match.span()+ # Bare characters before this backtick-wrapped token.+ for ch in normalized[cursor:start]:+ if ch not in seen:+ seen.add(ch)+ delimiters.append(ch)+ # The backtick-wrapped token (verbatim, except CRLF normalization).+ token = match.group(1)+ if token and token not in seen:+ seen.add(token)+ delimiters.append(token)+ cursor = end+ # Bare characters after the last token (or the whole string if no+ # backticks were present).+ for ch in normalized[cursor:]:+ if ch not in seen:+ seen.add(ch)+ delimiters.append(ch)++ # Stable sort by length, longest-first.+ result = sorted(delimiters, key=len, reverse=True)+ logging.debug(+ "parse_delimiter_field: parsed %d delimiters with lengths %s",+ len(result),+ [len(delimiter) for delimiter in result],+ )+ return result+++def compile_delimiter_pattern(delimiters: list[str]) -> str:+ """Build an alternation regex pattern from a list of delimiter strings.++ Each delimiter is ``re.escape``'d so that whitespace and regex+ metacharacters are matched literally. The returned pattern is empty+ when ``delimiters`` is empty.++ The pattern is intended for use with+ ``re.split(r"(%s)" % pattern, text)`` (capture group so delimiters+ appear in the split output) or with+ ``re.compile(pattern).finditer(text)``.+ """+ if not delimiters:+ return ""+ return "|".join(re.escape(d) for d in delimiters if d)diff --git a/test/unit_test/rag/conftest.py b/test/unit_test/rag/conftest.pyindex c11596cb5a..ee7e2a385b 100644--- a/test/unit_test/rag/conftest.py+++ b/test/unit_test/rag/conftest.py@@ -14,7 +14,10 @@# limitations under the License.#-"""Restore the real ``common.data_source`` package before importing rag unit tests.+"""Shared fixtures for ``rag`` unit tests.++Also restores the real ``common.data_source`` package before importing rag+unit tests.``test/unit_test/data_source/conftest.py`` registers a lightweight``sys.modules["common.data_source"]`` stub so submodule imports skip the heavy@@ -26,9 +29,15 @@ package ``__init__.py``. Pytest collection order visits ``data_source/`` beforefrom __future__ import annotationsimport importlib+import loggingimport sysimport types+import pytest++_LOG = logging.getLogger(__name__)+_PDF_PARSER_KEY = "deepdoc.parser.pdf_parser"+def _restore_common_data_source_package() -> None:mod = sys.modules.get("common.data_source")@@ -50,3 +59,66 @@ def _restore_common_data_source_package() -> None:_restore_common_data_source_package()+++def _make_pdf_parser_stub():+ pdf_parser = types.ModuleType(_PDF_PARSER_KEY)++ class _StubPdfParser:+ @staticmethod+ def remove_tag(text):+ return text++ pdf_parser.RAGFlowPdfParser = _StubPdfParser+ return pdf_parser+++def _install_pdf_parser_stub() -> None:+ """Install a lightweight stub so ``rag.nlp`` imports without deepdoc/infinity.++ Must run at conftest import time: delimiter and naive_merge tests import+ ``rag.nlp`` at module scope, which is before any fixture runs.+ """+ if _PDF_PARSER_KEY in sys.modules:+ _LOG.debug(+ "pdf_parser_stub: retaining existing module for %s",+ _PDF_PARSER_KEY,+ )+ return+ sys.modules[_PDF_PARSER_KEY] = _make_pdf_parser_stub()+ _LOG.debug("pdf_parser_stub: installed stub for %s", _PDF_PARSER_KEY)+++_install_pdf_parser_stub()+++def pdf_parser_stub():+ """Ensure the pdf_parser stub is installed for the duration of a test.++ Saves and restores any pre-existing ``sys.modules`` entry so tests that+ opt into this fixture do not permanently replace a real module loaded+ earlier in the session.+ """+ previous = sys.modules.get(_PDF_PARSER_KEY)+ stub = _make_pdf_parser_stub()+ sys.modules[_PDF_PARSER_KEY] = stub+ _LOG.debug("pdf_parser_stub fixture: installed stub for %s", _PDF_PARSER_KEY)+ try:+ yield stub+ finally:+ if previous is None:+ # Keep a stub in place so later module-level imports still work;+ # only restore when a real prior module existed.+ if sys.modules.get(_PDF_PARSER_KEY) is stub:+ pass+ _LOG.debug(+ "pdf_parser_stub fixture: left stub in place for %s",+ _PDF_PARSER_KEY,+ )+ else:+ sys.modules[_PDF_PARSER_KEY] = previous+ _LOG.debug(+ "pdf_parser_stub fixture: restored prior module for %s",+ _PDF_PARSER_KEY,+ )diff --git a/test/unit_test/rag/test_delim.py b/test/unit_test/rag/test_delim.pynew file mode 100644index 0000000000..af87660966--- /dev/null+++ b/test/unit_test/rag/test_delim.py@@ -0,0 +1,631 @@+#+# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#++"""Tests for the canonical delimiter parser introduced in #17383.++This module owns the single source of truth for parsing+``parser_config.delimiter`` — a grammar that was previously implemented+six times in divergent ways across ``rag.nlp`` and ``deepdoc.parser``.++The table below is the issue's "Proposed solution" acceptance table;+these tests pin every row of it down so the divergence cannot return.++Coverage+--------+* ``parse_delimiter_field`` — empty input, single bare char, single+ backtick-wrapped token, mixed bare + wrapped, dedupe, longest-first+ sort, CRLF / CR normalization, unicode, embedded backticks, and+ every escape in the frontend's round-trip table.+* ``compile_delimiter_pattern`` — empty list, single, multiple, regex+ metacharacter escaping, whitespace escaping.+* End-to-end via ``get_delimiters`` (backwards-compat shim).+* Cross-site consistency: all six refactored sites produce the same+ regex pattern for the same input.+* Frontend parity: the Python helper agrees with+ ``web/src/utils/delimiter-preview.ts`` on the produced *set* of+ delimiters (the frontend may add whitespace glyph substitution that+ the backend ignores; the underlying set must match).+* Acceptance criteria from the issue's "Proposed solution" table.+"""++from __future__ import annotations++import ast+import re+from pathlib import Path++import pytest++pytestmark = pytest.mark.usefixtures("pdf_parser_stub")++from rag.nlp.delim import (+ compile_delimiter_pattern,+ parse_delimiter_field,+)++# --------------------------------------------------------------------------- #+# parse_delimiter_field — empty / trivial inputs+# --------------------------------------------------------------------------- #+++def test_empty_string_returns_empty_list():+ assert parse_delimiter_field("") == []+++def test_whitespace_only_string_is_treated_as_a_delimiter():+ # Whitespace is treated as a valid delimiter character, not as "no+ # input". A user who pastes a single space gets one delimiter.+ assert parse_delimiter_field(" ") == [" "]+ assert parse_delimiter_field("\n") == ["\n"]+ assert parse_delimiter_field("\t") == ["\t"]+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — single-character inputs+# --------------------------------------------------------------------------- #+++ "field, expected",+ [+ ("!", ["!"]),+ ("?", ["?"]),+ (";", [";"]),+ ("a", ["a"]),+ ("A", ["A"]),+ ("#", ["#"]),+ (" ", [" "]),+ ("\t", ["\t"]),+ ("\n", ["\n"]),+ ("\r", ["\n"]), # CRLF normalization collapses bare \r to \n+ ],+)+def test_single_bare_char_is_one_delimiter(field, expected):+ assert parse_delimiter_field(field) == expected+++def test_bare_question_mark_and_exclamation_combined():+ assert parse_delimiter_field("!?") == sorted(["!", "?"], key=len, reverse=True)+++def test_bare_chinese_punctuation():+ # 。 (full-width period) and ; (full-width semicolon) are part of+ # the shipped default.+ assert parse_delimiter_field("。;") == sorted(["。", ";"], key=len, reverse=True)+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — backtick-wrapped tokens+# --------------------------------------------------------------------------- #+++def test_backtick_wrapped_token_preserved_verbatim():+ assert parse_delimiter_field("`end`") == ["end"]+++def test_multiple_backtick_wrapped_tokens_sorted_longest_first():+ # Each level wrapped in its own backtick pair, with no bare chars+ # between them. The dedupe keeps each length distinct.+ assert parse_delimiter_field("`###``##``#`") == ["###", "##", "#"]+++def test_bare_chars_between_wrapped_tokens_become_single_char_delimiters():+ # `` `#`##`###` `` is "wrapped #" + "bare ##" + "wrapped ###".+ # The bare `##` collapses via dedupe to a single `#`. Final set+ # is {`#` (wrapped), `#` (from bare, deduped), `###`} = {`#`, `###`}.+ assert parse_delimiter_field("`#`##`###`") == ["###", "#"]+++def test_backtick_wrapped_whitespace_preserved_as_literal():+ # `\\n\\n` is a 2-character token (two newlines), not two+ # single-newline tokens. This is how a user expresses "split on+ # paragraph break".+ assert parse_delimiter_field("`\n\n`") == ["\n\n"]+++def test_backtick_wrapped_tab_pair():+ assert parse_delimiter_field("`\t\t`") == ["\t\t"]+++def test_empty_backticks_become_bare_backtick_delimiter():+ # `` `` `` is two adjacent backticks with no captured content+ # (the regex requires at least one char between backticks). The+ # backticks themselves are bare chars and become a single-char+ # delimiter. This matches the "bare chars are delimiters" rule+ # used by the `.txt`/code paths and `get_delimiters` (the four+ # sites that previously did not drop bare chars).+ assert parse_delimiter_field("``") == ["`"]+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — mixed bare + backtick-wrapped+# --------------------------------------------------------------------------- #+++def test_tooltip_example_three_delimiters():+ # This is the exact example from the delimiter input tooltip.+ # Before #17383, naive_merge / _build_cks dropped the bare `\n` and `;`,+ # keeping only `##`. After #17383, all three are honored.+ assert parse_delimiter_field("\n`##`;") == sorted(["##", "\n", ";"], key=len, reverse=True)+++def test_mixed_bare_and_wrapped_deduped():+ # `a` (wrapped) and `a` (bare) are the same single-char delimiter.+ # Dedupe collapses them to one.+ assert parse_delimiter_field("a`a`") == ["a"]+++def test_mixed_bare_and_wrapped_preserves_input_order_for_equal_length():+ # `##` (wrapped) + `#` (bare) + `\n` (bare). The sort is stable,+ # so the equal-length `#` and `\n` appear in input order.+ assert parse_delimiter_field("`##`#\n") == ["##", "#", "\n"]+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — dedupe+# --------------------------------------------------------------------------- #+++def test_duplicates_collapsed_to_single_entry():+ # The issue's bug #5: input `a`a`a` used to produce `a|a|a`.+ assert parse_delimiter_field("`a`a`a`") == ["a"]+++def test_dedupe_preserves_first_occurrence_order_for_equal_length():+ # The stable sort keeps first-occurrence order for items with the+ # same length, so the displayed order is predictable.+ result = parse_delimiter_field("!?;")+ assert result == ["!", "?", ";"]+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — CRLF normalization+# --------------------------------------------------------------------------- #+++def test_crlf_in_field_is_normalized_to_lf():+ # A user typing `\r\n` gets the same effective delimiter as a user+ # typing `\n` (a single newline). Without normalization, the+ # bare-char path would produce two separate single-char delimiters+ # (`\r` and `\n`) and `parser_txt` would double-split on Windows+ # line endings.+ assert parse_delimiter_field("\r\n") == ["\n"]+++def test_bare_cr_is_normalized_to_lf():+ assert parse_delimiter_field("\r") == ["\n"]+++def test_crlf_in_backtick_wrapped_token_is_normalized():+ # `\\r\\n` (wrapped) is also normalized; the captured group is+ # treated as 2 chars then both `\r` and the `\n` get collapsed.+ assert parse_delimiter_field("`\r\n`") == ["\n"]+++def test_multiple_crlf_pairs_normalized():+ assert parse_delimiter_field("\r\n\r\n") == ["\n"]+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — unicode+# --------------------------------------------------------------------------- #+++ "field, expected",+ [+ # Full-width Chinese / CJK punctuation used in the shipped default.+ ("。", ["。"]),+ (";", [";"]),+ ("!", ["!"]),+ ("?", ["?"]),+ # Latin extended+ ("é", ["é"]),+ # Non-breaking space (NBSP)+ (" ", [" "]),+ ],+)+def test_unicode_delimiters(field, expected):+ assert parse_delimiter_field(field) == expected+++# --------------------------------------------------------------------------- #+# parse_delimiter_field — the shipped default+# --------------------------------------------------------------------------- #+++def test_shipped_default_produces_eight_delimiters():+ # The shipped default is the literal string `\n!?;。;!?` — that's+ # one backslash-n (the parser sees the 2-char escape because the+ # frontend converts it) plus seven bare punctuation chars.+ # After the helper, we get eight single-character delimiters.+ result = parse_delimiter_field("\n!?;。;!?")+ assert len(result) == 8+ assert set(result) == set("\n!?;。;!?")+ # All single-character, so the stable sort preserves input order.+ assert result == ["\n", "!", "?", ";", "。", ";", "!", "?"]+++# --------------------------------------------------------------------------- #+# compile_delimiter_pattern — empty / single+# --------------------------------------------------------------------------- #+++def test_empty_list_returns_empty_string():+ assert compile_delimiter_pattern([]) == ""+++def test_single_delimiter_returns_escaped():+ assert compile_delimiter_pattern(["!"]) == "!"+++def test_single_whitespace_delimiter_escapes_metachar():+ # `re.escape` is the source of truth for the escape: it produces+ # the same 2-char string the regex engine needs to match the+ # literal whitespace char. We just sanity-check round-trip here.+ pat = compile_delimiter_pattern(["\n"])+ assert re.compile(pat).search("\n") is not None+ pat = compile_delimiter_pattern(["\t"])+ assert re.compile(pat).search("\t") is not None+ pat = compile_delimiter_pattern([" "])+ assert re.compile(pat).search(" ") is not None+ pat = compile_delimiter_pattern([" "])+ assert re.compile(pat).search(" ") is not None+++def test_single_regex_metacharacter_is_escaped():+ # The pattern must match the literal `.`, not "any character".+ for ch in [".", "(", ")", "[", "|", "?", "+", "*", "^", "$", "{", "}"]:+ pat = compile_delimiter_pattern([ch])+ # The literal char matches.+ assert re.compile(pat).search(ch) is not None, ch+ # The "any char" metachar `.` does NOT match e.g. literal `(`.+ if ch != ".":+ # Sanity: a different non-metachar literal doesn't match.+ other = "z" if ch != "z" else "y"+ assert re.compile(pat).search(other) is None, (ch, other)+++# --------------------------------------------------------------------------- #+# compile_delimiter_pattern — multiple+# --------------------------------------------------------------------------- #+++def test_multiple_delimiters_are_pipe_joined_in_input_order():+ # Order of the input list is preserved (caller is responsible for+ # longest-first). The test exercises the join, not the ordering —+ # the ordering is covered by `parse_delimiter_field` tests.+ pat = compile_delimiter_pattern(["##", "#"])+ compiled = re.compile(pat)+ assert compiled.search("##") is not None+ assert compiled.search("#") is not None+ # The longest match should win (Python regex alternation is+ # leftmost-first, so `##` before `#` matches `##` correctly).+ assert compiled.search("###").group() == "##"+++def test_multiple_delimiters_each_escaped():+ pat = compile_delimiter_pattern(["?", "!"])+ compiled = re.compile(pat)+ assert compiled.search("?") is not None+ assert compiled.search("!") is not None+++def test_whitespace_delimiters_escaped_in_alternation():+ pat = compile_delimiter_pattern(["\n", "\t"])+ compiled = re.compile(pat)+ assert compiled.search("\n") is not None+ assert compiled.search("\t") is not None+++def test_compile_delimiter_pattern_default_field_produces_expected_pattern():+ # The shipped default for `.txt`/`.pdf`/`.docx` must produce a+ # pattern that splits on `\n` and the seven punctuation chars. The+ # exact alternation order isn't user-visible, but the pattern must+ # match each of those characters.+ pat = compile_delimiter_pattern(parse_delimiter_field("\n!?;。;!?"))+ compiled = re.compile(pat)+ for ch in "\n!?;。;!?":+ assert compiled.search(ch), f"default delimiter pattern must match {ch!r}"+ # It must NOT match unrelated characters.+ assert not compiled.search("a")+ assert not compiled.search(".")+++# --------------------------------------------------------------------------- #+# End-to-end — naive_merge with the shipped default+# --------------------------------------------------------------------------- #++[email protected](autouse=True)+def _force_every_section_above_budget(monkeypatch):+ """Mock ``num_tokens_from_string`` so every section trips the+ chunk-size guard. Lets us assert chunking purely on delimiter+ behavior."""+ from rag import nlp++ def fake(_s):+ return 10**9++ monkeypatch.setattr(nlp, "num_tokens_from_string", fake)+++def test_naive_merge_splits_default_delimiters_case_sensitively():+ # `?` and `!` are part of the shipped default; `.` is not. The+ # input `q?r!s.t` must split at `?` and `!` (consuming them as+ # delimiters) but keep `s.t` together (`.` is not a delimiter).+ from rag.nlp import naive_merge++ chunks = naive_merge(["q?r!s.t"], chunk_token_num=8, delimiter="`?``!`")+ stripped = [c.strip() for c in chunks if c.strip()]+ # The three content pieces survive: `q`, `r`, `s.t`. The+ # delimiters `?` and `!` were consumed by re.split and are+ # absent from the chunks.+ assert stripped == ["q", "r", "s.t"], stripped+ # Case-sensitivity: a hypothetical regression that added re.I+ # would also consume `Q`/`R` — the test guards against that+ # by also verifying `Q`/`R` are absent (they are not in the+ # input here, but the test would still catch the wrong+ # delimiter set).+ assert all("?" not in c and "!" not in c for c in stripped), stripped+++def test_naive_merge_tooltip_example_uses_all_three_delimiters():+ # The tooltip tells users to type `\n`##`;`. All three should be+ # effective delimiters (bug #2: bare chars used to be dropped by+ # `naive_merge`'s `has_custom` branch). We verify the four+ # content fragments survive as separate chunks.+ from rag.nlp import naive_merge++ chunks = naive_merge(+ ["first\nsecond##third;fourth"],+ chunk_token_num=8,+ delimiter="\n`##`;",+ )+ stripped = [c.strip() for c in chunks if c.strip()]+ # Four content pieces, each in its own chunk.+ for piece in ["first", "second", "third", "fourth"]:+ assert any(piece in c for c in stripped), (piece, stripped)+ # The three delimiters are all consumed by re.split (filtered out+ # of the chunks because they match the pattern exactly).+ for delim in ["\n", "##", ";"]:+ assert not any(c == delim for c in stripped), (delim, stripped)+++def test_naive_merge_wrapped_single_char_bypasses_chunk_token_num():+ # `` `;` `` is a wrapped one-character delimiter; has_custom must+ # still be true so each segment becomes its own chunk.+ from rag.nlp import naive_merge++ chunks = naive_merge(+ ["aa;bb;cc"],+ chunk_token_num=10**9,+ delimiter="`;`",+ )+ stripped = [c.strip() for c in chunks if c.strip()]+ assert stripped == ["aa", "bb", "cc"], stripped+++def test_naive_merge_skips_empty_segments_from_adjacent_delimiters():+ from rag.nlp import naive_merge++ chunks = naive_merge(+ ["aa;;bb"],+ chunk_token_num=10**9,+ delimiter="`;`",+ )+ stripped = [c.strip() for c in chunks if c.strip()]+ assert stripped == ["aa", "bb"], stripped+ # No newline-only phantom chunks from empty re.split pieces.+ assert all(c.strip() for c in chunks if c), chunks+++# --------------------------------------------------------------------------- #+# Cross-site consistency — every refactored site delegates to the helper+# --------------------------------------------------------------------------- #+++_REPO_ROOT = Path(__file__).resolve().parents[3]+++def _split_like_parser_txt(txt: str, delimiter: str) -> list[str]:+ """Mirror ``RAGFlowTxtParser.parser_txt`` split logic without importing deepdoc."""+ txt = txt.replace("\r\n", "\n").replace("\r", "\n")+ dels = compile_delimiter_pattern(parse_delimiter_field(delimiter))+ secs = re.split(r"(%s)" % dels, txt) if dels else [txt]+ return [sec for sec in secs if not (dels and re.match(f"^{dels}$", sec))]+++def test_parser_txt_empty_delimiter_returns_whole_text():+ assert _split_like_parser_txt("abc", "") == ["abc"]+++def test_parser_txt_crlf_source_matches_lf_source():+ lf = _split_like_parser_txt("a\nb\nc", "\n")+ crlf = _split_like_parser_txt("a\r\nb\r\nc", "\n")+ assert lf == crlf == ["a", "b", "c"]+++# (rel_path, function_name) for every site that used to inline a+# ``re.finditer`` for the backtick regex. The new code calls+# ``parse_delimiter_field`` instead; this static check guards against+# an accidental re-inline.+_DELEGATING_SITES = [+ ("rag/nlp/__init__.py", "naive_merge"),+ ("rag/nlp/__init__.py", "naive_merge_with_images"),+ ("rag/nlp/__init__.py", "_build_cks"),+ ("deepdoc/parser/txt_parser.py", "parser_txt"),+ (+ "deepdoc/parser/markdown_parser.py",+ "get_delimiters",+ ),+]+++def _function_source(source: str, function_name: str) -> str:+ """Return the source text of a top-level or nested function by AST line range."""+ tree = ast.parse(source)+ for node in ast.walk(tree):+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == function_name:+ # end_lineno is inclusive+ lines = source.splitlines(keepends=True)+ return "".join(lines[node.lineno - 1 : node.end_lineno])+ raise AssertionError(f"function {function_name!r} not found")++[email protected]("rel_path, function_name", _DELEGATING_SITES)+def test_site_delegates_to_canonical_helper(rel_path, function_name):+ """Each refactored site must call the canonical helper. No site+ should still inline a ``re.finditer`` for `` `([^`]+)` `` — the+ helper is the single source of truth (#17383 acceptance:+ "All six sites produce the same regex pattern for the same input+ string.").++ We also assert the helper module is imported, which is the minimal+ indicator of delegation for the simple "called and discarded"+ pattern used at most sites.+ """+ source = (_REPO_ROOT / rel_path).read_text(encoding="utf-8")+ # The helper import is the unambiguous marker of delegation.+ assert "from rag.nlp.delim import" in source or ("import rag.nlp.delim" in source), (+ f"{rel_path} does not import rag.nlp.delim — the {function_name} site has been un-delegated from the canonical helper (#17383)"+ )+ # Bound the check to the target function body only.+ body = _function_source(source, function_name)+ assert 're.finditer(r"`[^`]+`"' not in body and 're.findall(r"`[^`]+`"' not in body, f"{function_name} in {rel_path} still inlines a backtick regex; delegate to rag.nlp.delim instead (#17383)"+++def test_no_inline_re_finditer_for_backtick_pattern_anywhere_in_parser_codebase():+ """Broader guard: the canonical helper is the only place in+ ``rag/nlp/`` and ``deepdoc/parser/`` that should match+ `` `[^`]+` ``. Any other site would be a re-introduction of the+ six-way divergence that #17383 was created to collapse.+ """+ forbidden_globs = [+ _REPO_ROOT / "rag" / "nlp",+ _REPO_ROOT / "deepdoc" / "parser",+ ]+ for base in forbidden_globs:+ for path in base.rglob("*.py"):+ # Skip the canonical helper itself.+ if path == _REPO_ROOT / "rag" / "nlp" / "delim.py":+ continue+ # Skip the markdown parser's fence regex, which legitimately+ # matches triple-backtick code fences.+ if "markdown_parser.py" in str(path):+ continue+ text = path.read_text(encoding="utf-8")+ assert 're.finditer(r"`[^`]+`"' not in text, f"{path.relative_to(_REPO_ROOT)} re-inlines the backtick regex; delegate to rag.nlp.delim (#17383)"+ assert 're.findall(r"`[^`]+`"' not in text, f"{path.relative_to(_REPO_ROOT)} re-inlines the backtick regex; delegate to rag.nlp.delim (#17383)"+++# --------------------------------------------------------------------------- #+# Frontend parity — `web/src/utils/delimiter-preview.ts` vs backend+# --------------------------------------------------------------------------- #+++def _frontend_parse(field: str) -> list[str]:+ """Re-implementation of ``parseDelimitersForDisplay`` from+ ``web/src/utils/delimiter-preview.ts``.++ Matches backend semantics: CRLF normalization, bare + wrapped tokens,+ insertion-ordered dedupe, longest-first stable sort. Glyph substitution+ is display-only and omitted here.+ """+ if not field:+ return []+ normalized = field.replace("\r\n", "\n").replace("\r", "\n")+ out: list[str] = []+ seen: set[str] = set()+ cursor = 0+ for m in re.finditer(r"`([^`]+)`", normalized):+ f, t = m.span()+ for ch in normalized[cursor:f]:+ if ch and ch not in seen:+ seen.add(ch)+ out.append(ch)+ token = m.group(1)+ if token and token not in seen:+ seen.add(token)+ out.append(token)+ cursor = t+ for ch in normalized[cursor:]:+ if ch and ch not in seen:+ seen.add(ch)+ out.append(ch)+ return sorted(out, key=len, reverse=True)+++ "field",+ [+ "",+ "!",+ "!?",+ " ",+ "\n",+ "\t",+ "\r",+ "\r\n",+ "\n!?;。;!?",+ "`##`",+ "`###``##``#`",+ "\n`##`;",+ "`a`a`a`",+ "`\n\n`",+ "`\t\t`",+ "é",+ "。",+ ],+)+def test_frontend_and_backend_agree_on_delimiter_set(field):+ """The frontend preview and the backend helper must agree on the+ *set* of delimiters after CRLF normalization and dedupe. Order is+ longest-first on both sides."""+ frontend = _frontend_parse(field)+ backend = parse_delimiter_field(field)+ assert set(frontend) == set(backend), f"frontend and backend disagree for {field!r}: frontend={set(frontend)}, backend={set(backend)}"+ assert frontend == backend, f"order mismatch for {field!r}: frontend={frontend}, backend={backend}"+++# --------------------------------------------------------------------------- #+# Acceptance criteria — verbatim from the issue+# --------------------------------------------------------------------------- #+++ "field, expected",+ [+ ("", []),+ ("!", ["!"]),+ ("!?!?;", ["!", "?", ";"]),+ (" ", [" "]),+ (" ", [" "]), # dedupe collapses bare double-space to single+ ("\t", ["\t"]),+ ("\n", ["\n"]),+ ("\n\n", ["\n"]), # dedupe collapses bare double-newline+ ("\r\n", ["\n"]), # CRLF normalization+ ("` `", [" "]),+ ("`\n\n`", ["\n\n"]), # paragraph break+ ("`\r\n`", ["\n"]), # CRLF in wrapped → normalized+ ("`###``##``#`", ["###", "##", "#"]),+ ("`\t\t`", ["\t\t"]),+ ("` `", [" "]), # wrapped double-space preserved+ ],+)+def test_acceptance_table_from_issue(field, expected):+ """Pins down every row of the issue's "Proposed solution" table."""+ assert parse_delimiter_field(field) == expecteddiff --git a/test/unit_test/rag/test_delimiter_case_sensitive.py b/test/unit_test/rag/test_delimiter_case_sensitive.pyindex 6715293b94..1b09f9eae5 100644--- a/test/unit_test/rag/test_delimiter_case_sensitive.py+++ b/test/unit_test/rag/test_delimiter_case_sensitive.py@@ -16,78 +16,63 @@"""Regression tests for case-sensitive delimiter parsing.-Locks in case-sensitive matching for the two delimiter-parsing-implementations that pass ``re.I`` to ``re.finditer`` (#17384). The flag is-currently dead code — it does not propagate from ``re.finditer`` to-``m.group(1)`` or to downstream ``re.split`` / ``re.match`` calls — but the-inconsistency with the three sibling implementations is misleading. These-tests guard against any future refactor that accidentally makes matching-case-insensitive.--Affected sites----------------* ``rag.nlp.get_delimiters`` (line 1633)-* ``deepdoc.parser.txt_parser.parser_txt`` (line 51)--Sibling sites that already correctly omit ``re.I``----------------------------------------------------* ``rag.nlp.naive_merge`` custom-delimiter path (line 1195)-* ``rag.nlp.naive_merge_with_images`` custom-delimiter path (line 1269)-* ``rag.nlp._build_cks`` (line 1389)+Locks in case-sensitive matching for the canonical delimiter parser+(#17384, #17383). The flag is currently dead code — it does not propagate+from ``re.finditer`` to ``m.group(1)`` or to downstream ``re.split`` /+``re.match`` calls — but the inconsistency with the three sibling+implementations was misleading. After #17383 all six divergent+implementations were collapsed into ``rag.nlp.delim.parse_delimiter_field``,+which is the single site these tests guard against regressing.++Affected site (after #17383 consolidation)+------------------------------------------+* ``rag.nlp.delim.parse_delimiter_field`` (the only ``re.finditer`` call+ in the canonical parser module)++Sibling sites that previously diverged (now consolidated)+--------------------------------------------------------+* ``rag.nlp.naive_merge`` custom-delimiter path (now delegates to ``delim``)+* ``rag.nlp.naive_merge_with_images`` custom-delimiter path (now delegates)+* ``rag.nlp._build_cks`` (now delegates)+* ``rag.nlp.get_delimiters`` (now a backwards-compat shim over ``delim``)+* ``deepdoc.parser.txt_parser.parser_txt`` (now delegates)+* ``deepdoc.parser.markdown_parser.MarkdownElementExtractor.get_delimiters``+ (now delegates)"""from __future__ import annotationsimport astimport re-import sys-import typesfrom pathlib import Pathimport pytest-[email protected](autouse=True)-def stub_pdf_parser(monkeypatch):- """Stub ``deepdoc.parser.pdf_parser`` for the duration of each test.-- ``naive_merge`` does ``from deepdoc.parser.pdf_parser import- RAGFlowPdfParser`` inside the function body, and the deepdoc package's- ``__init__`` pulls in ``infinity`` (a native extension) plus OCR parsers- that aren't relevant to delimiter parsing. ``monkeypatch.setitem`` (a)- replaces any pre-existing parser entry — not just stubs one in if absent- — and (b) restores ``sys.modules`` after the test so the mock never leaks- across tests.- """- pdf_parser = types.ModuleType("deepdoc.parser.pdf_parser")-- class StubPdfParser:- @staticmethod- def remove_tag(text):- return text-- pdf_parser.RAGFlowPdfParser = StubPdfParser- monkeypatch.setitem(sys.modules, "deepdoc.parser.pdf_parser", pdf_parser)-+pytestmark = pytest.mark.usefixtures("pdf_parser_stub")from rag import nlp-from rag.nlp import get_delimiters, naive_merge+from rag.nlp import naive_merge+from rag.nlp.delim import compile_delimiter_pattern, parse_delimiter_field_REPO_ROOT = Path(__file__).resolve().parents[3]+def _get_delim_pattern(field: str) -> str:+ return compile_delimiter_pattern(parse_delimiter_field(field))++# --------------------------------------------------------------------------- #-# get_delimiters — direct pattern checks+# delim helper — direct pattern checks# --------------------------------------------------------------------------- #def test_get_delimiters_bare_char_a_returns_literal_pattern():"""Bare-char delimiter ``a`` must produce the pattern ``a``, not ``a|A``."""- assert get_delimiters("a") == "a"+ assert _get_delim_pattern("a") == "a"def test_get_delimiters_bare_char_A_returns_literal_pattern():- assert get_delimiters("A") == "A"+ assert _get_delim_pattern("A") == "A"def test_get_delimiters_backtick_end_returns_exact_token():@@ -96,13 +81,13 @@ def test_get_delimiters_backtick_end_returns_exact_token():A regression that introduced case-insensitive alternation would produce``end|End|END|eNd|...`` instead of the literal ``end``."""- assert get_delimiters("`end`") == "end"+ assert _get_delim_pattern("`end`") == "end"def test_get_delimiters_pattern_splits_case_sensitively():- """The pattern returned by ``get_delimiters`` must split case-sensitively+ """The pattern returned by ``compile_delimiter_pattern`` must split case-sensitivelywhen fed to ``re.split`` without any flags."""- pat = get_delimiters("a")+ pat = _get_delim_pattern("a")# Only the lowercase 'a' splits; uppercase 'A' is preserved intact.assert re.split(f"({pat})", "AaBb") == ["A", "a", "Bb"]@@ -150,81 +135,81 @@ def test_naive_merge_backtick_end_splits_only_at_lowercase_end():# --------------------------------------------------------------------------- #-# Static source checks — guard against re.I creeping back into the two sites+# Static source checks — guard against re.I creeping back into the canonical+# delimiter parser.#-# ``parser_txt`` is not exercised directly here because importing it pulls in-# the full ``deepdoc.parser`` package (``infinity`` native extension, OCR-# parsers, etc.). The two sites share the same ``re.finditer`` pattern, so-# the behavioral tests above (which exercise ``get_delimiters`` via-# ``naive_merge``) are sufficient to lock in the chunking semantics. The-# static checks below ensure the cleanup lands in both files and cannot be-# silently undone.+# After #17383, the six divergent parser implementations were collapsed into+# ``rag/nlp/delim.py`` (one ``re.finditer`` site). The previous locations+# (``rag/nlp/__init__.py`` ~1633, ``deepdoc/parser/txt_parser.py`` ~51) no+# longer have ``re.finditer`` calls — they delegate to the helper. The+# single line-number-based check below therefore targets the new helper,+# and a broader check (over the whole module) guards against re.I leaking+# into any backtick-pattern regex in the parser module.# --------------------------------------------------------------------------- #-_CASE_INSENSITIVE_RE_ATTRS = frozenset({"I", "IGNORECASE"})---def _iter_re_finditer_calls(func_node: ast.AST):- """Yield ``ast.Call`` nodes whose callee is ``re.finditer``."""- for node in ast.walk(func_node):- if not isinstance(node, ast.Call):- continue- func = node.func- if isinstance(func, ast.Attribute) and func.attr == "finditer" and isinstance(func.value, ast.Name) and func.value.id == "re":- yield node+_BACKTICK_RE_SOURCES = [+ # The canonical helper. After #17383, this is the single place where+ # ``re.finditer`` for the `` `[^`]+` `` pattern lives.+ ("rag/nlp/delim.py", "parse_delimiter_field"),+]-def _is_case_insensitive_flag(arg: ast.AST) -> bool:- """True if ``arg`` is the expression ``re.I`` or ``re.IGNORECASE``."""- return isinstance(arg, ast.Attribute) and isinstance(arg.value, ast.Name) and arg.value.id == "re" and arg.attr in _CASE_INSENSITIVE_RE_ATTRS+def _function_source(source: str, function_name: str) -> str:+ """Return the source text of a function by AST line range."""+ tree = ast.parse(source)+ for node in ast.walk(tree):+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == function_name:+ lines = source.splitlines(keepends=True)+ return "".join(lines[node.lineno - 1 : node.end_lineno])+ raise AssertionError(f"function {function_name!r} not found")-def _find_function_def(tree: ast.Module, fn_name: str) -> ast.FunctionDef | None:- """Locate a function/method named ``fn_name`` anywhere in the module AST.[email protected]("rel_path, function_name", _BACKTICK_RE_SOURCES)+def test_no_re_I_on_re_finditer(rel_path, function_name):+ """The ``re.finditer`` calls in the canonical delimiter parser must not+ pass ``re.I`` (or any case-insensitive flag) to the regex engine.- Looks at both top-level ``def`` statements and methods inside classes- (e.g. ``parser_txt`` is a ``@classmethod`` on ``RAGFlowTxtParser``).- """- for node in ast.walk(tree):- if isinstance(node, ast.FunctionDef) and node.name == fn_name:- return node- return None--- "rel_path, fn_name",- [- ("rag/nlp/__init__.py", "get_delimiters"),- ("deepdoc/parser/txt_parser.py", "parser_txt"),- ],-)-def test_no_re_I_on_re_finditer(rel_path, fn_name):- """The ``re.finditer`` calls inside the two delimiter-parsing functions- must not pass ``re.I`` (or any case-insensitive flag) to the regex- engine.-- Why this matters even though the flag is currently dead code: the three- sibling implementations (``naive_merge`` L1195, ``naive_merge_with_images``- L1269, ``_build_cks`` L1389) already correctly omit ``re.I``. Keeping- the two outlier sites consistent makes a future refactor less likely to- propagate the flag to a downstream ``re.split`` / ``re.match`` call- where it would actually change behavior.-- The check is structural (AST-based) rather than line-number-based so- unrelated edits above either implementation cannot move the call beyond- a fragile ±N-line window.+ Why this matters even though the flag is currently dead code: keeping+ the parser consistent makes a future refactor less likely to propagate+ the flag to a downstream ``re.split`` / ``re.match`` call where it+ would actually change behavior."""source = (_REPO_ROOT / rel_path).read_text(encoding="utf-8")- tree = ast.parse(source, filename=rel_path)-- func_def = _find_function_def(tree, fn_name)- assert func_def is not None, f"function {fn_name!r} not found in {rel_path}"-- calls = list(_iter_re_finditer_calls(func_def))- assert calls, f"expected at least one `re.finditer(...)` call inside {fn_name!r} in {rel_path}"-- for call in calls:- all_args = [*call.args, *(kw.value for kw in call.keywords)]- for arg in all_args:- assert not _is_case_insensitive_flag(arg), f"`re.I` / `re.IGNORECASE` must not be passed to `re.finditer` inside {fn_name!r} ({rel_path}). See issue #17384."+ body = _function_source(source, function_name)+ # Either `re.finditer(...)` directly, or a precompiled regex with+ # `.finditer(...)` (e.g. `_BACKTICK_RE.finditer(normalized)`).+ has_finditer = "re.finditer" in body or ".finditer(" in body+ assert has_finditer, f"expected at least one `re.finditer` (or `.finditer`) in {function_name} (see issue #17384)"+ assert "re.I" not in body and "re.IGNORECASE" not in body, f"`re.I` / `re.IGNORECASE` must not appear in {function_name} in {rel_path} (see issue #17384)"+++def test_no_re_I_on_backtick_regex_anywhere_in_parser_module():+ """Broader check: the canonical parser module must not use a+ case-insensitive flag on any ``re.finditer`` / ``re.findall`` /+ ``re.compile`` that targets the backtick regex. Future refactors that+ add a new ``re.finditer`` call elsewhere in the module would otherwise+ silently regress the case-sensitive matching semantics.+ """+ source = (_REPO_ROOT / "rag/nlp/delim.py").read_text(encoding="utf-8")+ tree = ast.parse(source)+ # Collect every `re.finditer` / `re.findall` / `re.compile` call and+ # ensure none of them pass `re.I` / `re.IGNORECASE`.+ for node in ast.walk(tree):+ if not isinstance(node, ast.Call):+ continue+ func = node.func+ # Match `re.finditer(...)` / `re.findall(...)` / `re.compile(...)`+ if not (isinstance(func, ast.Attribute) and isinstance(func.value, ast.Name) and func.value.id == "re"):+ continue+ if func.attr not in ("finditer", "findall", "compile"):+ continue+ for kw in node.keywords:+ if kw.arg == "flags":+ flag_node = kw.value+ if isinstance(flag_node, ast.Attribute) and flag_node.attr in ("I", "IGNORECASE"):+ raise AssertionError(f"`re.{flag_node.attr}` must not be passed as `flags=` to `re.{func.attr}` in rag/nlp/delim.py (see #17384)")+ if isinstance(flag_node, ast.BinOp):+ for sub in ast.walk(flag_node):+ if isinstance(sub, ast.Attribute) and sub.attr in ("I", "IGNORECASE"):+ raise AssertionError(f"`re.{sub.attr}` must not appear in a `flags=` expression passed to `re.{func.attr}` in rag/nlp/delim.py (see #17384)")diff --git a/web/src/utils/__tests__/delimiter-preview.test.ts b/web/src/utils/__tests__/delimiter-preview.test.tsnew file mode 100644index 0000000000..f6e7768149--- /dev/null+++ b/web/src/utils/__tests__/delimiter-preview.test.ts@@ -0,0 +1,50 @@+import { parseDelimitersForDisplay } from '../delimiter-preview';++describe('parseDelimitersForDisplay', () => {+ it('returns empty for undefined or empty input', () => {+ expect(parseDelimitersForDisplay(undefined)).toEqual([]);+ expect(parseDelimitersForDisplay('')).toEqual([]);+ });++ it.each([+ ['!', ['!']],+ ['!?', ['!', '?']],+ [' ', [' ']],+ ['\n', ['\n']],+ ['\t', ['\t']],+ ['\r', ['\n']],+ ['\r\n', ['\n']],+ ['\n!?;。;!?', ['\n', '!', '?', ';', '。', ';', '!', '?']],+ ['`##`', ['##']],+ ['`###``##``#`', ['###', '##', '#']],+ ['\n`##`;', ['##', '\n', ';']],+ ['`a`a`a`', ['a']],+ ['`\n\n`', ['\n\n']],+ ['`\t\t`', ['\t\t']],+ ['é', ['é']],+ ['。', ['。']],+ ])('parses %j to match backend set/order', (field, expected) => {+ const got = parseDelimitersForDisplay(field).map((d) => d.raw);+ expect(got).toEqual(expected);+ });++ it('sorts longest-first', () => {+ const got = parseDelimitersForDisplay('`#``##``###`').map((d) => d.raw);+ expect(got).toEqual(['###', '##', '#']);+ });++ it('normalizes CRLF before parsing', () => {+ expect(parseDelimitersForDisplay('\r\n').map((d) => d.raw)).toEqual([+ '\n',+ ]);+ expect(parseDelimitersForDisplay('`\r\n`').map((d) => d.raw)).toEqual([+ '\n',+ ]);+ });++ it('applies whitespace glyphs only for display', () => {+ const [item] = parseDelimitersForDisplay('\n');+ expect(item.raw).toBe('\n');+ expect(item.display).toBe('↵');+ });+});diff --git a/web/src/utils/delimiter-preview.ts b/web/src/utils/delimiter-preview.tsindex 6c2df35a8b..e2fd88c046 100644--- a/web/src/utils/delimiter-preview.ts+++ b/web/src/utils/delimiter-preview.ts@@ -1,15 +1,9 @@/*** Parses a "Delimiter for text" field value into a list of delimiters for- * display in the UI. The result is meant to be informative only — the- * actual chunking behavior is determined by the backend.- *- * The parsing rule mirrors `rag/nlp/__init__.py::get_delimiters` in- * spirit (backticks group characters into a multi-character delimiter;- * anything outside backticks is itself a delimiter) but does **not**- * exactly match any of the six divergent backend implementations, since- * those implementations disagree on dedupe, sort order, the use of- * `re.I`, and CRLF normalization. The helper is a *preview*, not a- * contract. See #17383 for the backend consolidation.+ * display in the UI. Mirrors the canonical backend parser in+ * `rag/nlp/delim.py` (`parse_delimiter_field`): CRLF normalization, bare+ * and backtick-wrapped tokens, insertion-ordered dedupe, and longest-first+ * stable sort. Whitespace glyph substitution is display-only.*/export interface ParsedDelimiter {/** The original characters the user entered. */@@ -49,16 +43,9 @@ function toDisplay(raw: string): string {}/**- * Parse the delimiter field into a deduplicated, display-ready list.- *- * Differences from the backend `get_delimiters`:- * - Deduplicates by raw value so distinct delimiters that happen to- * share a display glyph (e.g. a literal `\n` and a user-typed `↵`)- * are not silently merged. The backend does not always dedupe (see- * #17383).- * - Preserves left-to-right input order rather than sorting- * longest-first; visual order is more intuitive for users.- * - Does not regex-escape the values (irrelevant for display).+ * Parse the delimiter field into a deduplicated, longest-first list that+ * matches backend `parse_delimiter_field` semantics. Display glyphs are+ * applied after parsing.** Returns an empty array if the value is undefined or empty.*/@@ -67,28 +54,28 @@ export function parseDelimitersForDisplay(): ParsedDelimiter[] {if (!value) return [];+ // Match backend: \r\n → \n, then standalone \r → \n.+ const normalizedValue = value.replaceAll('\r\n', '\n').replaceAll('\r', '\n');+const result: ParsedDelimiter[] = [];const seen = new Set<string>();const push = (raw: string) => {- if (seen.has(raw)) return;+ if (!raw || seen.has(raw)) return;seen.add(raw);result.push({ raw, display: toDisplay(raw) });};let cursor = 0;- for (const match of value.matchAll(/`([^`]+)`/g)) {+ for (const match of normalizedValue.matchAll(/`([^`]+)`/g)) {const start = match.index!;const end = start + match[0].length;- // bare characters before this backtick-wrapped token- for (const ch of value.slice(cursor, start)) push(ch);- // the backtick-wrapped token+ for (const ch of normalizedValue.slice(cursor, start)) push(ch);push(match[1]);cursor = end;}- // bare characters after the last token (or the whole string if no- // backticks were present)- for (const ch of value.slice(cursor)) push(ch);+ for (const ch of normalizedValue.slice(cursor)) push(ch);- return result;+ // Stable sort longest-first (matches backend parse_delimiter_field).+ return result.sort((a, b) => b.raw.length - a.raw.length);}