RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): group DOCX paragraphs into custom-delimiter records (#20713)
Sourav Rajvi1b6c087b3934
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex a67709148c..c48172d20c 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -31,6 +31,7 @@ import ("regexp""sort""strings"+ "unicode""unicode/utf8""go.uber.org/zap"@@ -404,17 +405,36 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemaprimaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileDelimPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)- if !customDelimiter {+ recordPrefixes := chunk.CustomDelimiterPrefixes(c.param.Delimiters)+ recordGrouped := false+ if len(recordPrefixes) > 0 {+ units, recordGrouped = groupDOCXUnitsByRecordBoundary(units, recordPrefixes)+ }+ splitPattern := primaryPattern+ if recordGrouped {+ // Grouped records join paragraphs with "\n"; bare "\n" in the default+ // delimiter list must not re-split them (#20496).+ splitPattern = chunk.CompileDelimiterListPattern(c.param.Delimiters)+ }+ if !customDelimiter && !recordGrouped {attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)units = separateGeneralTextBlocks(units)}- units = splitGeneralUnits(units, primaryPattern, !customDelimiter)- if customDelimiter {+ if recordGrouped {+ units = splitGeneralUnitsPreservingRecordPrefix(units, splitPattern, recordPrefixes)+ } else {+ units = splitGeneralUnits(units, splitPattern, !customDelimiter)+ }+ if customDelimiter || recordGrouped {attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)} else {units = mergeDOCXUnits(units, c.param.ChunkTokenSize)}- units = applyGeneralOverlap(units, c.param.OverlappedPercent, "")+ overlapSep := ""+ if recordGrouped {+ overlapSep = "\n"+ }+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, overlapSep)units = finalizeGeneralChunks(units, childrenPattern)if len(units) == 0 {return emptyOutputs(), nil@@ -448,6 +468,77 @@ func separateGeneralTextBlocks(units []schema.ChunkDoc) []schema.ChunkDoc {return units}+// groupDOCXUnitsByRecordBoundary merges consecutive text paragraphs into one+// unit until a paragraph begins with a custom (backtick) delimiter prefix.+// DOCX upstream emits one paragraph per unit, so a custom delimiter that only+// appears at the start of each logical record must group paragraphs rather+// than split within a single paragraph (#20496).+func groupDOCXUnitsByRecordBoundary(units []schema.ChunkDoc, recordPrefixes []string) ([]schema.ChunkDoc, bool) {+ if len(recordPrefixes) == 0 {+ return units, false+ }+ grouped := make([]schema.ChunkDoc, 0, len(units))+ var current *schema.ChunkDoc+ sawBoundary := false+ flush := func() {+ if current == nil {+ return+ }+ grouped = append(grouped, *current)+ current = nil+ }+ for _, unit := range units {+ if itemDocType(unit) != "text" {+ flush()+ media := cloneChunkDoc(unit)+ media.DocType = itemDocType(media)+ media.CKType = media.DocType+ grouped = append(grouped, media)+ continue+ }+ if strings.TrimSpace(unit.Text) == "" {+ continue+ }+ text := cloneChunkDoc(unit)+ text.DocType = "text"+ text.CKType = "text"+ if docxTextStartsRecordBoundary(text.Text, recordPrefixes) {+ sawBoundary = true+ flush()+ current = &text+ continue+ }+ if !sawBoundary {+ flush()+ grouped = append(grouped, text)+ continue+ }+ if current == nil {+ current = &text+ continue+ }+ mergeGeneralChunk(current, text, "\n")+ }+ flush()+ if !sawBoundary {+ return units, false+ }+ return grouped, true+}++func docxTextStartsRecordBoundary(text string, recordPrefixes []string) bool {+ trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)+ for _, prefix := range recordPrefixes {+ if prefix == "" {+ continue+ }+ if strings.HasPrefix(text, prefix) || strings.HasPrefix(trimmed, prefix) {+ return true+ }+ }+ return false+}+// mergeDOCXUnits keeps the previous text merge target across intervening// table/image units. Paragraph separators and sentence delimiters are already// carried by the text units.@@ -1235,6 +1326,62 @@ func joinGeneralOverlapText(prefix, text, separator string) string {return prefix + separator + text}+// splitGeneralUnitsPreservingRecordPrefix is splitGeneralUnits, except a+// leading custom-delimiter record prefix (for example "问:") is kept on the+// first fragment. Inline delimiter matches are still dropped in splitByDelim+// drop mode.+func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {+ result := make([]schema.ChunkDoc, 0, len(units))+ for _, unit := range units {+ unit = cloneChunkDoc(unit)+ unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))+ unit.DocType = itemDocType(unit)+ unit.CKType = unit.DocType+ prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)+ if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {+ unit.Text = prefix + rest+ unit.TKNums = intPtr(tokenizeStr(unit.Text))+ result = append(result, unit)+ continue+ }+ parts := splitByDelim(rest, pattern, false)+ filtered := make([]string, 0, len(parts))+ for _, part := range parts {+ if strings.TrimSpace(part) == "" {+ continue+ }+ filtered = append(filtered, part)+ }+ if len(filtered) > 0 {+ filtered[0] = prefix + filtered[0]+ }+ for _, piece := range sliceGeneralSplitPieces(unit, filtered) {+ result = append(result, piece)+ }+ }+ return result+}++func peelLeadingRecordPrefix(text string, prefixes []string) (prefix, rest string) {+ if len(prefixes) == 0 {+ return "", text+ }+ trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)+ leadingWS := text[:len(text)-len(trimmed)]+ for _, p := range prefixes {+ if p == "" {+ continue+ }+ if strings.HasPrefix(text, p) {+ return p, text[len(p):]+ }+ if strings.HasPrefix(trimmed, p) {+ return leadingWS + p, trimmed[len(p):]+ }+ }+ return "", text+}+func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {result := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex 7a10f0c81a..c337efafb0 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -818,6 +818,55 @@ func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {}}+func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"\n", "`问:`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "问:first question", "doc_type_kwd": "text"},+ {"text": "名 称:item one", "doc_type_kwd": "text"},+ {"text": "答:first answer", "doc_type_kwd": "text"},+ {"text": "问:second question", "doc_type_kwd": "text"},+ {"text": "答:second answer", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ texts := outputTexts(t, out)+ if len(texts) != 2 {+ t.Fatalf("texts = %q, want two record chunks", texts)+ }+ if !strings.Contains(texts[0], "问:first question") || !strings.Contains(texts[0], "名 称:item one") || !strings.Contains(texts[0], "答:first answer") {+ t.Fatalf("first record = %q", texts[0])+ }+ if !strings.Contains(texts[1], "问:second question") || !strings.Contains(texts[1], "答:second answer") {+ t.Fatalf("second record = %q", texts[1])+ }+}++func TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "plain paragraph one", DocType: "text", CKType: "text"},+ {Text: "plain paragraph two", DocType: "text", CKType: "text"},+ }+ got, grouped := groupDOCXUnitsByRecordBoundary(units, []string{"问:"})+ if grouped {+ t.Fatal("grouping should stay inactive without a record boundary")+ }+ if !reflect.DeepEqual(got, units) {+ t.Fatalf("units = %#v, want unchanged %#v", got, units)+ }+}+func TestGeneralChunkerDOCXCustomDelimiterDisablesTextMerge(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10,diff --git a/internal/parser/chunk/delim.go b/internal/parser/chunk/delim.goindex f4762e85fd..81dad763fe 100644--- a/internal/parser/chunk/delim.go+++ b/internal/parser/chunk/delim.go@@ -178,3 +178,32 @@ func HasCustomDelimiterList(delims []string) bool {}return false}++// CustomDelimiterPrefixes returns the inner text of every backtick-wrapped+// entry in a TokenChunker-style delimiter list, longest-first and deduplicated.+// Used when a custom delimiter marks record boundaries across DOCX paragraphs.+func CustomDelimiterPrefixes(delims []string) []string {+ var out []string+ seen := make(map[string]struct{})+ for _, d := range delims {+ if !strings.HasPrefix(d, "`") || !strings.HasSuffix(d, "`") || len(d) < 2 {+ continue+ }+ inner := d[1 : len(d)-1]+ if inner == "" {+ continue+ }+ if _, ok := seen[inner]; ok {+ continue+ }+ seen[inner] = struct{}{}+ out = append(out, inner)+ }+ if len(out) == 0 {+ return nil+ }+ sort.SliceStable(out, func(i, j int) bool {+ return utf8.RuneCountInString(out[i]) > utf8.RuneCountInString(out[j])+ })+ return out+}diff --git a/internal/parser/chunk/delim_test.go b/internal/parser/chunk/delim_test.goindex b6fea0c68f..c6b0beab36 100644--- a/internal/parser/chunk/delim_test.go+++ b/internal/parser/chunk/delim_test.go@@ -254,6 +254,17 @@ func TestCompileDelimiterPatternListChildrenPrefixOrder(t *testing.T) {}}+func TestCustomDelimiterPrefixes(t *testing.T) {+ got := CustomDelimiterPrefixes([]string{"\n", "`问:`", "`##`"})+ want := []string{"问:", "##"}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("CustomDelimiterPrefixes = %#v, want %#v", got, want)+ }+ if CustomDelimiterPrefixes([]string{"\n", "!"}) != nil {+ t.Fatal("expected nil when no backtick entries")+ }+}+func TestHasCustomDelimiterList(t *testing.T) {if HasCustomDelimiterList([]string{"\n", "!"}) {t.Fatal("bare list should be false")