Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(go): unify children delimiter pattern with backtick-strip + rune order (#17926)

Jack60df86bfa2ac
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 81dc412c84..10fa83510a 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -1134,32 +1134,14 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc
// compileChildrenPattern is the children_delimiters version of
// compileDelimPattern. Returns nil when no delimiters exist.
+// compileChildrenPattern builds the children-split regex from a
+// `children_delimiters` list. Every non-empty entry is active (including bare
+// ones), and backtick-wrapped entries contribute their inner content — see
+// chunk.CompileDelimiterPatternList. Delegating keeps children splitting
+// consistent with the main delimiter list (backtick stripping + rune-descending
+// order) instead of re-implementing a divergent copy.
func compileChildrenPattern(delims []string) *regexp.Regexp {
- if len(delims) == 0 {
- return nil
- }
- escaped := make([]string, 0, len(delims))
- for _, d := range delims {
- if d == "" {
- continue
- }
- escaped = append(escaped, regexp.QuoteMeta(d))
- }
- if len(escaped) == 0 {
- return nil
- }
- sortSlice(escaped)
- return regexp.MustCompile(strings.Join(escaped, "|"))
-}
-
-// sortSlice sorts in place by descending length (longest pattern
-// first, mirroring python's `sorted(set, key=len, reverse=True)`).
-func sortSlice(in []string) {
- for i := 1; i < len(in); i++ {
- for j := i; j > 0 && len(in[j-1]) < len(in[j]); j-- {
- in[j-1], in[j] = in[j], in[j-1]
- }
- }
+ return chunk.CompileDelimiterPatternList(delims, true)
}
// stringFromInputs returns the string value at the first matching key
diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.go
index 8337b1b0e9..b97b89f36f 100644
--- a/internal/ingestion/component/chunker/token_children_delim_test.go
+++ b/internal/ingestion/component/chunker/token_children_delim_test.go
@@ -2,6 +2,7 @@ package chunker
import (
"context"
+ "strings"
"testing"
)
@@ -47,6 +48,60 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {
// children_delimiters split also DROPS the delimiter (applyChildrenDelim /
// applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent
// segment in "mom".
+// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a
+// backtick-wrapped children_delimiter contributes its INNER content as the
+// split pattern (not the literal wrapped token), and the matched delimiter is
+// dropped from each child — consistent with the main delimiter list behavior.
+func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {
+ c, err := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"\n"},
+ "children_delimiters": []string{"`###`"},
+ })
+ if err != nil {
+ t.Fatalf("NewTokenChunker: %v", err)
+ }
+ out, err := c.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": "sec one###sec two###sec three",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ want := []string{"sec one", "sec two", "sec three"}
+ if len(chunks) != len(want) {
+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))
+ }
+ for i, w := range want {
+ got := chunks[i]["text"].(string)
+ if got != w {
+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)
+ }
+ }
+ // The literal backtick token `###` must never match as a whole string.
+ c2, _ := NewTokenChunker(map[string]any{
+ "delimiter_mode": "delimiter",
+ "delimiters": []string{"\n"},
+ "children_delimiters": []string{"`###`"},
+ })
+ out2, err := c2.Invoke(context.Background(), nil, map[string]any{
+ "name": "doc.txt",
+ "output_format": "text",
+ "text": "a `###` b",
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks2, _ := out2["chunks"].([]map[string]any)
+ for _, ck := range chunks2 {
+ if strings.Contains(ck["text"].(string), "###") {
+ t.Errorf("child text kept literal backtick-wrapped token: %q", ck["text"].(string))
+ }
+ }
+}
+
func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {
c, err := NewTokenChunker(map[string]any{
"delimiter_mode": "delimiter",
diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.go
index aebadbd60a..1b897ab5ea 100644
--- a/internal/parser/chunk/delim.go
+++ b/internal/parser/chunk/delim.go
@@ -135,16 +135,28 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {
return regexp.MustCompile(strings.Join(escaped, "|"))
}
-// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
-// list. Entries wrapped in backticks contribute their inner content as a
-// split pattern (Python token_chunker / historical Go compileDelimPattern).
-// Bare list entries are ignored for the active pattern — they are only used
-// by merge paths when no custom pattern exists.
+// CompileDelimiterPatternList builds an alternation regex from a
+// TokenChunker-style []string delimiter list, with a shared policy for how
+// backtick-wrapped and bare entries are treated. Every non-empty entry is
+// regexp.QuoteMeta'd and the result is sorted longest-first (stable by rune
+// count) so a longer delimiter always wins over a shorter prefix inside it.
//
-// Prefer ParseDelimiterField for the single-string parser_config.delimiter
-// field. This helper exists for the dataflow list API.
-func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
- var custom []string
+// - Backtick-wrapped entries (“ `abc` “) contribute their INNER content as
+// the split pattern; the backticks are stripped. This is the Python
+// token_chunker / historical Go compileDelimPattern convention.
+// - Bare entries (e.g. ". ") behave according to keepBare:
+// keepBare=false (the main `delimiters` list): a bare entry is IGNORED for
+// the active pattern — it only influences merge paths when no custom
+// pattern exists.
+// keepBare=true (the `children_delimiters` list): a bare entry is KEPT
+// active, because every non-empty children delimiter splits — including
+// bare ones.
+//
+// Returns nil when no active entry remains. Prefer ParseDelimiterField for the
+// single-string parser_config.delimiter field; this helper exists for the
+// dataflow list API.
+func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp {
+ var out []string
for _, d := range delims {
if d == "" {
continue
@@ -154,22 +166,38 @@ func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
if inner == "" {
continue
}
- custom = append(custom, inner)
+ out = append(out, inner)
+ continue
+ }
+ if !keepBare {
+ continue
}
+ out = append(out, d)
}
- if len(custom) == 0 {
+ if len(out) == 0 {
return nil
}
- sort.SliceStable(custom, func(i, j int) bool {
- return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])
+ sort.SliceStable(out, func(i, j int) bool {
+ return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j])
})
- escaped := make([]string, 0, len(custom))
- for _, d := range custom {
+ escaped := make([]string, 0, len(out))
+ for _, d := range out {
escaped = append(escaped, regexp.QuoteMeta(d))
}
return regexp.MustCompile(strings.Join(escaped, "|"))
}
+// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter
+// list. It is CompileDelimiterPatternList with keepBare=false: backtick entries
+// contribute their inner content and bare entries are ignored for the active
+// pattern (they only steer merge paths when no custom pattern exists).
+//
+// Prefer ParseDelimiterField for the single-string parser_config.delimiter
+// field. This helper exists for the dataflow list API.
+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {
+ return CompileDelimiterPatternList(delims, false)
+}
+
// HasCustomDelimiterList reports whether any entry in a TokenChunker-style
// delimiter list uses backtick syntax.
func HasCustomDelimiterList(delims []string) bool {
diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.go
index 7a1034200b..4c5a65fb43 100644
--- a/internal/parser/chunk/delim_test.go
+++ b/internal/parser/chunk/delim_test.go
@@ -175,6 +175,44 @@ func TestCompileDelimiterListPattern(t *testing.T) {
}
}
+func TestCompileDelimiterPatternListKeepBare(t *testing.T) {
+ // keepBare=true (children_delimiters): bare entries stay active and
+ // backtick entries contribute their inner content, all QuoteMeta'd and
+ // sorted longest-first by rune count.
+ pat := CompileDelimiterPatternList([]string{". ", "`###`", "#"}, true)
+ if pat == nil {
+ t.Fatal("expected a pattern from mixed bare + wrapped entries")
+ }
+ want := `###|\. |#`
+ if got := pat.String(); got != want {
+ t.Errorf("pattern = %q, want %q", got, want)
+ }
+ // Backtick is stripped: splitting on "###" matches the inner content, not
+ // the literal wrapped token.
+ if got := pat.FindString("a###b"); got != "###" {
+ t.Errorf("FindString(a###b) = %q, want %q", got, "###")
+ }
+ // Bare ". " stays active and is matched as the meta-escaped ". ".
+ if got := pat.FindString("alpha. beta"); got != ". " {
+ t.Errorf("FindString(alpha. beta) = %q, want %q", got, ". ")
+ }
+}
+
+func TestCompileDelimiterPatternListKeepBareFalse(t *testing.T) {
+ // keepBare=false must equal CompileDelimiterListPattern: bare entries
+ // ignored, only wrapped inner content participates.
+ if CompileDelimiterPatternList([]string{"\n", "!"}, false) != nil {
+ t.Fatal("bare entries should be ignored when keepBare=false")
+ }
+ pat := CompileDelimiterPatternList([]string{"`##`", "`#`"}, false)
+ if pat == nil {
+ t.Fatal("expected pattern from wrapped entries")
+ }
+ if got := pat.FindString("###"); got != "##" {
+ t.Errorf("got %q want ##", got)
+ }
+}
+
func TestHasCustomDelimiterList(t *testing.T) {
if HasCustomDelimiterList([]string{"\n", "!"}) {
t.Fatal("bare list should be false")