RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(go): unify children delimiter pattern with backtick-strip + rune order (#17926)
Jack60df86bfa2ac
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 81dc412c84..10fa83510a 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -1134,32 +1134,14 @@ func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []sc// compileChildrenPattern is the children_delimiters version of// compileDelimPattern. Returns nil when no delimiters exist.+// compileChildrenPattern builds the children-split regex from a+// `children_delimiters` list. Every non-empty entry is active (including bare+// ones), and backtick-wrapped entries contribute their inner content — see+// chunk.CompileDelimiterPatternList. Delegating keeps children splitting+// consistent with the main delimiter list (backtick stripping + rune-descending+// order) instead of re-implementing a divergent copy.func compileChildrenPattern(delims []string) *regexp.Regexp {- if len(delims) == 0 {- return nil- }- escaped := make([]string, 0, len(delims))- for _, d := range delims {- if d == "" {- continue- }- escaped = append(escaped, regexp.QuoteMeta(d))- }- if len(escaped) == 0 {- return nil- }- sortSlice(escaped)- return regexp.MustCompile(strings.Join(escaped, "|"))-}--// sortSlice sorts in place by descending length (longest pattern-// first, mirroring python's `sorted(set, key=len, reverse=True)`).-func sortSlice(in []string) {- for i := 1; i < len(in); i++ {- for j := i; j > 0 && len(in[j-1]) < len(in[j]); j-- {- in[j-1], in[j] = in[j], in[j-1]- }- }+ return chunk.CompileDelimiterPatternList(delims, true)}// stringFromInputs returns the string value at the first matching keydiff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.goindex 8337b1b0e9..b97b89f36f 100644--- a/internal/ingestion/component/chunker/token_children_delim_test.go+++ b/internal/ingestion/component/chunker/token_children_delim_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "strings""testing")@@ -47,6 +48,60 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {// children_delimiters split also DROPS the delimiter (applyChildrenDelim /// applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent// segment in "mom".+// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a+// backtick-wrapped children_delimiter contributes its INNER content as the+// split pattern (not the literal wrapped token), and the matched delimiter is+// dropped from each child — consistent with the main delimiter list behavior.+func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {+ c, err := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"\n"},+ "children_delimiters": []string{"`###`"},+ })+ if err != nil {+ t.Fatalf("NewTokenChunker: %v", err)+ }+ out, err := c.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": "sec one###sec two###sec three",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ want := []string{"sec one", "sec two", "sec three"}+ if len(chunks) != len(want) {+ t.Fatalf("chunk count: want %d got %d (%v)", len(want), len(chunks), chunkTexts(chunks))+ }+ for i, w := range want {+ got := chunks[i]["text"].(string)+ if got != w {+ t.Errorf("chunk[%d] text: want %q got %q", i, w, got)+ }+ }+ // The literal backtick token `###` must never match as a whole string.+ c2, _ := NewTokenChunker(map[string]any{+ "delimiter_mode": "delimiter",+ "delimiters": []string{"\n"},+ "children_delimiters": []string{"`###`"},+ })+ out2, err := c2.Invoke(context.Background(), nil, map[string]any{+ "name": "doc.txt",+ "output_format": "text",+ "text": "a `###` b",+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks2, _ := out2["chunks"].([]map[string]any)+ for _, ck := range chunks2 {+ if strings.Contains(ck["text"].(string), "###") {+ t.Errorf("child text kept literal backtick-wrapped token: %q", ck["text"].(string))+ }+ }+}+func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {c, err := NewTokenChunker(map[string]any{"delimiter_mode": "delimiter",diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.goindex aebadbd60a..1b897ab5ea 100644--- a/internal/parser/chunk/delim.go+++ b/internal/parser/chunk/delim.go@@ -135,16 +135,28 @@ func CompileDelimiterPattern(delimiters []string) *regexp.Regexp {return regexp.MustCompile(strings.Join(escaped, "|"))}-// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter-// list. Entries wrapped in backticks contribute their inner content as a-// split pattern (Python token_chunker / historical Go compileDelimPattern).-// Bare list entries are ignored for the active pattern — they are only used-// by merge paths when no custom pattern exists.+// CompileDelimiterPatternList builds an alternation regex from a+// TokenChunker-style []string delimiter list, with a shared policy for how+// backtick-wrapped and bare entries are treated. Every non-empty entry is+// regexp.QuoteMeta'd and the result is sorted longest-first (stable by rune+// count) so a longer delimiter always wins over a shorter prefix inside it.//-// Prefer ParseDelimiterField for the single-string parser_config.delimiter-// field. This helper exists for the dataflow list API.-func CompileDelimiterListPattern(delims []string) *regexp.Regexp {- var custom []string+// - Backtick-wrapped entries (“ `abc` “) contribute their INNER content as+// the split pattern; the backticks are stripped. This is the Python+// token_chunker / historical Go compileDelimPattern convention.+// - Bare entries (e.g. ". ") behave according to keepBare:+// keepBare=false (the main `delimiters` list): a bare entry is IGNORED for+// the active pattern — it only influences merge paths when no custom+// pattern exists.+// keepBare=true (the `children_delimiters` list): a bare entry is KEPT+// active, because every non-empty children delimiter splits — including+// bare ones.+//+// Returns nil when no active entry remains. Prefer ParseDelimiterField for the+// single-string parser_config.delimiter field; this helper exists for the+// dataflow list API.+func CompileDelimiterPatternList(delims []string, keepBare bool) *regexp.Regexp {+ var out []stringfor _, d := range delims {if d == "" {continue@@ -154,22 +166,38 @@ func CompileDelimiterListPattern(delims []string) *regexp.Regexp {if inner == "" {continue}- custom = append(custom, inner)+ out = append(out, inner)+ continue+ }+ if !keepBare {+ continue}+ out = append(out, d)}- if len(custom) == 0 {+ if len(out) == 0 {return nil}- sort.SliceStable(custom, func(i, j int) bool {- return utf8.RuneCountInString(custom[i]) > utf8.RuneCountInString(custom[j])+ sort.SliceStable(out, func(i, j int) bool {+ return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j])})- escaped := make([]string, 0, len(custom))- for _, d := range custom {+ escaped := make([]string, 0, len(out))+ for _, d := range out {escaped = append(escaped, regexp.QuoteMeta(d))}return regexp.MustCompile(strings.Join(escaped, "|"))}+// CompileDelimiterListPattern compiles a TokenChunker-style []string delimiter+// list. It is CompileDelimiterPatternList with keepBare=false: backtick entries+// contribute their inner content and bare entries are ignored for the active+// pattern (they only steer merge paths when no custom pattern exists).+//+// Prefer ParseDelimiterField for the single-string parser_config.delimiter+// field. This helper exists for the dataflow list API.+func CompileDelimiterListPattern(delims []string) *regexp.Regexp {+ return CompileDelimiterPatternList(delims, false)+}+// HasCustomDelimiterList reports whether any entry in a TokenChunker-style// delimiter list uses backtick syntax.func HasCustomDelimiterList(delims []string) bool {diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.goindex 7a1034200b..4c5a65fb43 100644--- a/internal/parser/chunk/delim_test.go+++ b/internal/parser/chunk/delim_test.go@@ -175,6 +175,44 @@ func TestCompileDelimiterListPattern(t *testing.T) {}}+func TestCompileDelimiterPatternListKeepBare(t *testing.T) {+ // keepBare=true (children_delimiters): bare entries stay active and+ // backtick entries contribute their inner content, all QuoteMeta'd and+ // sorted longest-first by rune count.+ pat := CompileDelimiterPatternList([]string{". ", "`###`", "#"}, true)+ if pat == nil {+ t.Fatal("expected a pattern from mixed bare + wrapped entries")+ }+ want := `###|\. |#`+ if got := pat.String(); got != want {+ t.Errorf("pattern = %q, want %q", got, want)+ }+ // Backtick is stripped: splitting on "###" matches the inner content, not+ // the literal wrapped token.+ if got := pat.FindString("a###b"); got != "###" {+ t.Errorf("FindString(a###b) = %q, want %q", got, "###")+ }+ // Bare ". " stays active and is matched as the meta-escaped ". ".+ if got := pat.FindString("alpha. beta"); got != ". " {+ t.Errorf("FindString(alpha. beta) = %q, want %q", got, ". ")+ }+}++func TestCompileDelimiterPatternListKeepBareFalse(t *testing.T) {+ // keepBare=false must equal CompileDelimiterListPattern: bare entries+ // ignored, only wrapped inner content participates.+ if CompileDelimiterPatternList([]string{"\n", "!"}, false) != nil {+ t.Fatal("bare entries should be ignored when keepBare=false")+ }+ pat := CompileDelimiterPatternList([]string{"`##`", "`#`"}, false)+ if pat == nil {+ t.Fatal("expected pattern from wrapped entries")+ }+ if got := pat.FindString("###"); got != "##" {+ t.Errorf("got %q want ##", got)+ }+}+func TestHasCustomDelimiterList(t *testing.T) {if HasCustomDelimiterList([]string{"\n", "!"}) {t.Fatal("bare list should be false")