Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(go): clean up TokenChunker delimiter routing and children appliers (#17954)

Jacke587c564608f
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index db1b69dec3..7a07252b97 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -78,9 +78,10 @@ func stringListFromAny(in []any) []string {
// compileDelimPattern compiles a TokenChunker-style []string delimiter list.
// Only backtick-wrapped entries produce an active pattern (Python
-// token_chunker / rag/nlp/delim list helper). Plain entries are ignored here
-// and used by mergeByTokenSize for sentence-level splitting when no active
-// pattern exists.
+// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries
+// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,
+// not this list. Canonical single-string parser_config.delimiter parsing
+// lives in ragflow/internal/parser/chunk (ParseDelimiterField).
func compileDelimPattern(delims []string) *regexp.Regexp {
return chunk.CompileDelimiterListPattern(delims)
}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 865e3c3fdd..93781081a1 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -336,19 +336,18 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text string
if len(cleaned) == 0 {
return emptyOutputs()
}
- docs := applyChildrenDelim(cleaned, childrenPattern)
-
- // Python's naive_merge: custom (backtick) delimiters produce one
- // chunk per segment — no token-size merge (naive_merge:1194-1213).
- if hasCustomDelim(c.param.Delimiters) {
- return chunkOutputs(docs)
- }
-
- // Split-then-merge: split on delimiters, then greedily merge to
- // chunk_token_size with optional overlap.
- perItem := [][]schema.ChunkDoc{docs}
- merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent, c.param.MergeStrategy())
- return chunkOutputs(flatten(merged))
+ textDocs := make([]schema.ChunkDoc, 0, len(cleaned))
+ for _, s := range cleaned {
+ textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})
+ }
+ docs := applyChildrenDelimText(textDocs, childrenPattern)
+
+ // Python's naive_merge: a custom (backtick) delimiter yields one chunk
+ // per segment and no token-size merge (naive_merge:1194-1213). A
+ // non-custom active delimiter cannot reach here — delimPattern is
+ // non-nil only when a backtick delimiter exists, so the split-then-
+ // merge branch was unreachable and has been removed.
+ return chunkOutputs(docs)
}
// sentenceDelimiter is the sentence/clause-boundary regex used to split
@@ -1108,12 +1107,11 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.
// shared text-payload helpers (used by TitleChunker et al.)
// ---------------------------------------------------------------------------
-// hasActiveDelimiter reports whether a regex compiled by
-// compileDelimPattern contains any non-placeholder pattern. The "match
-// nothing" sentinel regexp makes a quick `pattern.MatchString("")`
-// viable as a check without re-walking the source slice.
+// hasActiveDelimiter reports whether a compiled delimiter pattern is
+// present (non-nil). compileDelimPattern returns nil when no active
+// pattern exists, so a nil check is sufficient.
func hasActiveDelimiter(p *regexp.Regexp) bool {
- return p != nil && p.String() != `\A(?!)`
+ return p != nil
}
// hasCustomDelim reports whether any delimiter uses backtick syntax
@@ -1123,34 +1121,6 @@ func hasCustomDelim(delims []string) bool {
return chunk.HasCustomDelimiterList(delims)
}
-// applyChildrenDelim mirrors token_chunker.py:325-334.
-func applyChildrenDelim(segs []string, pattern *regexp.Regexp) []schema.ChunkDoc {
- if pattern == nil {
- out := make([]schema.ChunkDoc, 0, len(segs))
- for _, s := range segs {
- out = append(out, schema.ChunkDoc{
- Text: s,
- DocType: "text",
- CKType: "text",
- })
- }
- return out
- }
- var docs []schema.ChunkDoc
- for _, seg := range segs {
- if strings.TrimSpace(seg) == "" {
- continue
- }
- for _, child := range splitDroppingDelim(seg, pattern) {
- if strings.TrimSpace(child) == "" {
- continue
- }
- docs = append(docs, schema.ChunkDoc{Text: child, Mom: seg})
- }
- }
- return docs
-}
-
func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
if pattern == nil {
return docs
diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.go
index b97b89f36f..bc55046be6 100644
--- a/internal/ingestion/component/chunker/token_children_delim_test.go
+++ b/internal/ingestion/component/chunker/token_children_delim_test.go
@@ -45,9 +45,8 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {
}
// TestTokenChunker_ChildrenDelimiterDroppedText asserts the text/markdown/html
-// children_delimiters split also DROPS the delimiter (applyChildrenDelim /
-// applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent
-// segment in "mom".
+// children_delimiters split also DROPS the delimiter (applyChildrenDelimText
+// mirrors _split_text_by_pattern), keeping the parent segment in "mom".
// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a
// backtick-wrapped children_delimiter contributes its INNER content as the
// split pattern (not the literal wrapped token), and the matched delimiter is