RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(go): clean up TokenChunker delimiter routing and children appliers (#17954)
Jacke587c564608f
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex db1b69dec3..7a07252b97 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -78,9 +78,10 @@ func stringListFromAny(in []any) []string {// compileDelimPattern compiles a TokenChunker-style []string delimiter list.// Only backtick-wrapped entries produce an active pattern (Python-// token_chunker / rag/nlp/delim list helper). Plain entries are ignored here-// and used by mergeByTokenSize for sentence-level splitting when no active-// pattern exists.+// token_chunker / rag/nlp/delim list helper); plain (non-backtick) entries+// are ignored here. mergeByTokenSize uses its own hardcoded sentenceDelimiter,+// not this list. Canonical single-string parser_config.delimiter parsing+// lives in ragflow/internal/parser/chunk (ParseDelimiterField).func compileDelimPattern(delims []string) *regexp.Regexp {return chunk.CompileDelimiterListPattern(delims)}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 865e3c3fdd..93781081a1 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -336,19 +336,18 @@ func (c *TokenChunkerComponent) invokeTextPayload(_ context.Context, text stringif len(cleaned) == 0 {return emptyOutputs()}- docs := applyChildrenDelim(cleaned, childrenPattern)-- // Python's naive_merge: custom (backtick) delimiters produce one- // chunk per segment — no token-size merge (naive_merge:1194-1213).- if hasCustomDelim(c.param.Delimiters) {- return chunkOutputs(docs)- }-- // Split-then-merge: split on delimiters, then greedily merge to- // chunk_token_size with optional overlap.- perItem := [][]schema.ChunkDoc{docs}- merged := mergeByTokenSizeFromJSON(perItem, c.param.ChunkTokenSize, c.param.OverlappedPercent, c.param.MergeStrategy())- return chunkOutputs(flatten(merged))+ textDocs := make([]schema.ChunkDoc, 0, len(cleaned))+ for _, s := range cleaned {+ textDocs = append(textDocs, schema.ChunkDoc{Text: s, DocType: "text", CKType: "text"})+ }+ docs := applyChildrenDelimText(textDocs, childrenPattern)++ // Python's naive_merge: a custom (backtick) delimiter yields one chunk+ // per segment and no token-size merge (naive_merge:1194-1213). A+ // non-custom active delimiter cannot reach here — delimPattern is+ // non-nil only when a backtick delimiter exists, so the split-then-+ // merge branch was unreachable and has been removed.+ return chunkOutputs(docs)}// sentenceDelimiter is the sentence/clause-boundary regex used to split@@ -1108,12 +1107,11 @@ func splitByChildren(chunks []schema.ChunkDoc, pattern *regexp.Regexp) []schema.// shared text-payload helpers (used by TitleChunker et al.)// ----------------------------------------------------------------------------// hasActiveDelimiter reports whether a regex compiled by-// compileDelimPattern contains any non-placeholder pattern. The "match-// nothing" sentinel regexp makes a quick `pattern.MatchString("")`-// viable as a check without re-walking the source slice.+// hasActiveDelimiter reports whether a compiled delimiter pattern is+// present (non-nil). compileDelimPattern returns nil when no active+// pattern exists, so a nil check is sufficient.func hasActiveDelimiter(p *regexp.Regexp) bool {- return p != nil && p.String() != `\A(?!)`+ return p != nil}// hasCustomDelim reports whether any delimiter uses backtick syntax@@ -1123,34 +1121,6 @@ func hasCustomDelim(delims []string) bool {return chunk.HasCustomDelimiterList(delims)}-// applyChildrenDelim mirrors token_chunker.py:325-334.-func applyChildrenDelim(segs []string, pattern *regexp.Regexp) []schema.ChunkDoc {- if pattern == nil {- out := make([]schema.ChunkDoc, 0, len(segs))- for _, s := range segs {- out = append(out, schema.ChunkDoc{- Text: s,- DocType: "text",- CKType: "text",- })- }- return out- }- var docs []schema.ChunkDoc- for _, seg := range segs {- if strings.TrimSpace(seg) == "" {- continue- }- for _, child := range splitDroppingDelim(seg, pattern) {- if strings.TrimSpace(child) == "" {- continue- }- docs = append(docs, schema.ChunkDoc{Text: child, Mom: seg})- }- }- return docs-}-func applyChildrenDelimText(docs []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {if pattern == nil {return docsdiff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.goindex b97b89f36f..bc55046be6 100644--- a/internal/ingestion/component/chunker/token_children_delim_test.go+++ b/internal/ingestion/component/chunker/token_children_delim_test.go@@ -45,9 +45,8 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {}// TestTokenChunker_ChildrenDelimiterDroppedText asserts the text/markdown/html-// children_delimiters split also DROPS the delimiter (applyChildrenDelim /-// applyChildrenDelimText mirror _split_text_by_pattern), keeping the parent-// segment in "mom".+// children_delimiters split also DROPS the delimiter (applyChildrenDelimText+// mirrors _split_text_by_pattern), keeping the parent segment in "mom".// TestTokenChunker_ChildrenDelimiterBacktickStripped asserts that a// backtick-wrapped children_delimiter contributes its INNER content as the// split pattern (not the literal wrapped token), and the matched delimiter is