Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): group DOCX paragraphs into custom-delimiter records (#20713)

Sourav Rajvi1b6c087b3934
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index a67709148c..c48172d20c 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -31,6 +31,7 @@ import (
"regexp"
"sort"
"strings"
+ "unicode"
"unicode/utf8"
"go.uber.org/zap"
@@ -404,17 +405,36 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)
customDelimiter := hasCustomDelim(c.param.Delimiters)
- if !customDelimiter {
+ recordPrefixes := chunk.CustomDelimiterPrefixes(c.param.Delimiters)
+ recordGrouped := false
+ if len(recordPrefixes) > 0 {
+ units, recordGrouped = groupDOCXUnitsByRecordBoundary(units, recordPrefixes)
+ }
+ splitPattern := primaryPattern
+ if recordGrouped {
+ // Grouped records join paragraphs with "\n"; bare "\n" in the default
+ // delimiter list must not re-split them (#20496).
+ splitPattern = chunk.CompileDelimiterListPattern(c.param.Delimiters)
+ }
+ if !customDelimiter && !recordGrouped {
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
units = separateGeneralTextBlocks(units)
}
- units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
- if customDelimiter {
+ if recordGrouped {
+ units = splitGeneralUnitsPreservingRecordPrefix(units, splitPattern, recordPrefixes)
+ } else {
+ units = splitGeneralUnits(units, splitPattern, !customDelimiter)
+ }
+ if customDelimiter || recordGrouped {
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)
} else {
units = mergeDOCXUnits(units, c.param.ChunkTokenSize)
}
- units = applyGeneralOverlap(units, c.param.OverlappedPercent, "")
+ overlapSep := ""
+ if recordGrouped {
+ overlapSep = "\n"
+ }
+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, overlapSep)
units = finalizeGeneralChunks(units, childrenPattern)
if len(units) == 0 {
return emptyOutputs(), nil
@@ -448,6 +468,77 @@ func separateGeneralTextBlocks(units []schema.ChunkDoc) []schema.ChunkDoc {
return units
}
+// groupDOCXUnitsByRecordBoundary merges consecutive text paragraphs into one
+// unit until a paragraph begins with a custom (backtick) delimiter prefix.
+// DOCX upstream emits one paragraph per unit, so a custom delimiter that only
+// appears at the start of each logical record must group paragraphs rather
+// than split within a single paragraph (#20496).
+func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []string) ([]schema.ChunkDoc, bool) {
+ if len(recordPrefixes) == 0 {
+ return units, false
+ }
+ grouped := make([]schema.ChunkDoc, 0, len(units))
+ var current *schema.ChunkDoc
+ sawBoundary := false
+ flush := func() {
+ if current == nil {
+ return
+ }
+ grouped = append(grouped, *current)
+ current = nil
+ }
+ for _, unit := range units {
+ if itemDocType(unit) != "text" {
+ flush()
+ media := cloneChunkDoc(unit)
+ media.DocType = itemDocType(media)
+ media.CKType = media.DocType
+ grouped = append(grouped, media)
+ continue
+ }
+ if strings.TrimSpace(unit.Text) == "" {
+ continue
+ }
+ text := cloneChunkDoc(unit)
+ text.DocType = "text"
+ text.CKType = "text"
+ if docxTextStartsRecordBoundary(text.Text, recordPrefixes) {
+ sawBoundary = true
+ flush()
+ current = &text
+ continue
+ }
+ if !sawBoundary {
+ flush()
+ grouped = append(grouped, text)
+ continue
+ }
+ if current == nil {
+ current = &text
+ continue
+ }
+ mergeGeneralChunk(current, text, "\n")
+ }
+ flush()
+ if !sawBoundary {
+ return units, false
+ }
+ return grouped, true
+}
+
+func docxTextStartsRecordBoundary(text string, recordPrefixes []string) bool {
+ trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)
+ for _, prefix := range recordPrefixes {
+ if prefix == "" {
+ continue
+ }
+ if strings.HasPrefix(text, prefix) || strings.HasPrefix(trimmed, prefix) {
+ return true
+ }
+ }
+ return false
+}
+
// mergeDOCXUnits keeps the previous text merge target across intervening
// table/image units. Paragraph separators and sentence delimiters are already
// carried by the text units.
@@ -1235,6 +1326,62 @@ func joinGeneralOverlapText(prefix, text, separator string) string {
return prefix + separator + text
}
+// splitGeneralUnitsPreservingRecordPrefix is splitGeneralUnits, except a
+// leading custom-delimiter record prefix (for example "问:") is kept on the
+// first fragment. Inline delimiter matches are still dropped in splitByDelim
+// drop mode.
+func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {
+ result := make([]schema.ChunkDoc, 0, len(units))
+ for _, unit := range units {
+ unit = cloneChunkDoc(unit)
+ unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
+ unit.DocType = itemDocType(unit)
+ unit.CKType = unit.DocType
+ prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)
+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {
+ unit.Text = prefix + rest
+ unit.TKNums = intPtr(tokenizeStr(unit.Text))
+ result = append(result, unit)
+ continue
+ }
+ parts := splitByDelim(rest, pattern, false)
+ filtered := make([]string, 0, len(parts))
+ for _, part := range parts {
+ if strings.TrimSpace(part) == "" {
+ continue
+ }
+ filtered = append(filtered, part)
+ }
+ if len(filtered) > 0 {
+ filtered[0] = prefix + filtered[0]
+ }
+ for _, piece := range sliceGeneralSplitPieces(unit, filtered) {
+ result = append(result, piece)
+ }
+ }
+ return result
+}
+
+func peelLeadingRecordPrefix(text string, prefixes []string) (prefix, rest string) {
+ if len(prefixes) == 0 {
+ return "", text
+ }
+ trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)
+ leadingWS := text[:len(text)-len(trimmed)]
+ for _, p := range prefixes {
+ if p == "" {
+ continue
+ }
+ if strings.HasPrefix(text, p) {
+ return p, text[len(p):]
+ }
+ if strings.HasPrefix(trimmed, p) {
+ return leadingWS + p, trimmed[len(p):]
+ }
+ }
+ return "", text
+}
+
func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {
result := make([]schema.ChunkDoc, 0, len(units))
for _, unit := range units {
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index 7a10f0c81a..c337efafb0 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -818,6 +818,55 @@ func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {
}
}
+func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"\n", "`问:`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "问:first question", "doc_type_kwd": "text"},
+ {"text": "名  称:item one", "doc_type_kwd": "text"},
+ {"text": "答:first answer", "doc_type_kwd": "text"},
+ {"text": "问:second question", "doc_type_kwd": "text"},
+ {"text": "答:second answer", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ texts := outputTexts(t, out)
+ if len(texts) != 2 {
+ t.Fatalf("texts = %q, want two record chunks", texts)
+ }
+ if !strings.Contains(texts[0], "问:first question") || !strings.Contains(texts[0], "名  称:item one") || !strings.Contains(texts[0], "答:first answer") {
+ t.Fatalf("first record = %q", texts[0])
+ }
+ if !strings.Contains(texts[1], "问:second question") || !strings.Contains(texts[1], "答:second answer") {
+ t.Fatalf("second record = %q", texts[1])
+ }
+}
+
+func TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "plain paragraph one", DocType: "text", CKType: "text"},
+ {Text: "plain paragraph two", DocType: "text", CKType: "text"},
+ }
+ got, grouped := groupDOCXUnitsByRecordBoundary(units, []string{"问:"})
+ if grouped {
+ t.Fatal("grouping should stay inactive without a record boundary")
+ }
+ if !reflect.DeepEqual(got, units) {
+ t.Fatalf("units = %#v, want unchanged %#v", got, units)
+ }
+}
+
func TestGeneralChunkerDOCXCustomDelimiterDisablesTextMerge(t *testing.T) {
component, err := NewGeneralChunker(map[string]any{
"chunk_token_size": 10,
diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.go
index f4762e85fd..81dad763fe 100644
--- a/internal/parser/chunk/delim.go
+++ b/internal/parser/chunk/delim.go
@@ -178,3 +178,32 @@ func HasCustomDelimiterList(delims []string) bool {
}
return false
}
+
+// CustomDelimiterPrefixes returns the inner text of every backtick-wrapped
+// entry in a TokenChunker-style delimiter list, longest-first and deduplicated.
+// Used when a custom delimiter marks record boundaries across DOCX paragraphs.
+func CustomDelimiterPrefixes(delims []string) []string {
+ var out []string
+ seen := make(map[string]struct{})
+ for _, d := range delims {
+ if !strings.HasPrefix(d, "`") || !strings.HasSuffix(d, "`") || len(d) < 2 {
+ continue
+ }
+ inner := d[1 : len(d)-1]
+ if inner == "" {
+ continue
+ }
+ if _, ok := seen[inner]; ok {
+ continue
+ }
+ seen[inner] = struct{}{}
+ out = append(out, inner)
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ sort.SliceStable(out, func(i, j int) bool {
+ return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j])
+ })
+ return out
+}
diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.go
index b6fea0c68f..c6b0beab36 100644
--- a/internal/parser/chunk/delim_test.go
+++ b/internal/parser/chunk/delim_test.go
@@ -254,6 +254,17 @@ func TestCompileDelimiterPatternListChildrenPrefixOrder(t *testing.T) {
}
}
+func TestCustomDelimiterPrefixes(t *testing.T) {
+ got := CustomDelimiterPrefixes([]string{"\n", "`问:`", "`##`"})
+ want := []string{"问:", "##"}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("CustomDelimiterPrefixes = %#v, want %#v", got, want)
+ }
+ if CustomDelimiterPrefixes([]string{"\n", "!"}) != nil {
+ t.Fatal("expected nil when no backtick entries")
+ }
+}
+
func TestHasCustomDelimiterList(t *testing.T) {
if HasCustomDelimiterList([]string{"\n", "!"}) {
t.Fatal("bare list should be false")