Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): drop DOCX custom-delimiter record prefix (#21760)

jay7772141753ee5a1a3
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index a1ab96d8af..fa47e951aa 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -419,7 +419,7 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
units = separateGeneralTextBlocks(units)
}
if recordGrouped {
- units = splitGeneralUnitsPreservingRecordPrefix(units, splitPattern, recordPrefixes)
+ units = splitGeneralUnitsDroppingRecordPrefix(units, splitPattern, recordPrefixes)
} else {
units = splitGeneralUnits(units, splitPattern, !customDelimiter)
}
@@ -1330,20 +1330,29 @@ func joinGeneralOverlapText(prefix, text, separator string) string {
return prefix + separator + text
}
-// splitGeneralUnitsPreservingRecordPrefix is splitGeneralUnits, except a
-// leading custom-delimiter record prefix (for example "问:") is kept on the
-// first fragment. Inline delimiter matches are still dropped in splitByDelim
-// drop mode.
-func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {
+// splitGeneralUnitsDroppingRecordPrefix splits grouped DOCX records and drops
+// the leading custom-delimiter record prefix (for example "问:"). A custom
+// (backtick) delimiter is a split instruction, not content: Python
+// naive_merge_docx/_build_cks drops it and strips each flushed record. Inline
+// delimiter matches are dropped by splitByDelim drop mode.
+func splitGeneralUnitsDroppingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {
result := make([]schema.ChunkDoc, 0, len(units))
for _, unit := range units {
unit = cloneChunkDoc(unit)
unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))
unit.DocType = itemDocType(unit)
clearStructuralLayout(&unit)
- prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)
- if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {
- unit.Text = prefix + rest
+ if unit.DocType != "text" {
+ unit.TKNums = intPtr(tokenizeStr(unit.Text))
+ result = append(result, unit)
+ continue
+ }
+ rest := dropLeadingRecordPrefix(unit.Text, recordPrefixes)
+ if pattern == nil || !pattern.MatchString(rest) {
+ unit.Text = strings.TrimSpace(rest)
+ if unit.Text == "" && !hasMediaPayload(unit) {
+ continue
+ }
unit.TKNums = intPtr(tokenizeStr(unit.Text))
result = append(result, unit)
continue
@@ -1351,14 +1360,12 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *r
parts := splitByDelim(rest, pattern, false)
filtered := make([]string, 0, len(parts))
for _, part := range parts {
- if strings.TrimSpace(part) == "" {
+ part = strings.TrimSpace(part)
+ if part == "" {
continue
}
filtered = append(filtered, part)
}
- if len(filtered) > 0 {
- filtered[0] = prefix + filtered[0]
- }
for _, piece := range sliceGeneralSplitPieces(unit, filtered) {
result = append(result, piece)
}
@@ -1366,24 +1373,24 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *r
return result
}
-func peelLeadingRecordPrefix(text string, prefixes []string) (prefix, rest string) {
- if len(prefixes) == 0 {
- return "", text
+// dropLeadingRecordPrefix removes the first matching custom-delimiter record
+// prefix from a grouped DOCX record. The original text is matched first so a
+// delimiter that includes leading whitespace wins over a shorter prefix found
+// only after trimming, mirroring the compiled pattern's earliest match. The
+// whitespace-trimmed text is a fallback for delimiters after indentation.
+func dropLeadingRecordPrefix(text string, prefixes []string) string {
+ for _, prefix := range prefixes {
+ if prefix != "" && strings.HasPrefix(text, prefix) {
+ return text[len(prefix):]
+ }
}
trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)
- leadingWS := text[:len(text)-len(trimmed)]
- for _, p := range prefixes {
- if p == "" {
- continue
- }
- if strings.HasPrefix(text, p) {
- return p, text[len(p):]
- }
- if strings.HasPrefix(trimmed, p) {
- return leadingWS + p, trimmed[len(p):]
+ for _, prefix := range prefixes {
+ if prefix != "" && strings.HasPrefix(trimmed, prefix) {
+ return trimmed[len(prefix):]
}
}
- return "", text
+ return text
}
func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index 9a30dc02ec..1e0307e6ed 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -899,14 +899,104 @@ func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testing
if len(texts) != 2 {
t.Fatalf("texts = %q, want two record chunks", texts)
}
- if !strings.Contains(texts[0], "问:first question") || !strings.Contains(texts[0], "名  称:item one") || !strings.Contains(texts[0], "答:first answer") {
+ // A custom delimiter is a split instruction, not content: the leading
+ // record prefix is dropped (Python naive_merge_docx/_build_cks parity).
+ if strings.Contains(texts[0], "问:") || !strings.Contains(texts[0], "first question") || !strings.Contains(texts[0], "名  称:item one") || !strings.Contains(texts[0], "答:first answer") {
t.Fatalf("first record = %q", texts[0])
}
- if !strings.Contains(texts[1], "问:second question") || !strings.Contains(texts[1], "答:second answer") {
+ if strings.Contains(texts[1], "问:") || !strings.Contains(texts[1], "second question") || !strings.Contains(texts[1], "答:second answer") {
t.Fatalf("second record = %q", texts[1])
}
}
+func TestGeneralChunkerDOCXDropsCustomDelimiterRecordPrefix(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"`AAA`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "first section", "doc_type_kwd": "text"},
+ {"text": "AAA", "doc_type_kwd": "text"},
+ {"text": "second section", "doc_type_kwd": "text"},
+ {"text": "AAA", "doc_type_kwd": "text"},
+ {"text": "before AAA after", "doc_type_kwd": "text"},
+ {"text": "AAA", "doc_type_kwd": "text"},
+ {"text": "third section", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ texts := outputTexts(t, out)
+ for _, text := range texts {
+ if strings.Contains(text, "AAA") {
+ t.Fatalf("custom delimiter survived: %q", text)
+ }
+ }
+ if !reflect.DeepEqual(texts, []string{"first section", "second section", "before", "after", "third section"}) {
+ t.Fatalf("texts = %q", texts)
+ }
+}
+
+func TestGeneralChunkerDOCXDropsTrailingDelimiterOnlyRecord(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "overlapped_percent": 50,
+ "delimiters": []string{"`AAA`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "AAAfirst", "doc_type_kwd": "text"},
+ {"text": "AAA", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ // The delimiter-only record must not survive as an empty unit: overlap
+ // would otherwise synthesize a spurious chunk from the previous tail.
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first"}) {
+ t.Fatalf("texts = %q, want [first]", texts)
+ }
+}
+
+func TestGeneralChunkerDOCXPrefersDelimiterMatchBeforeTrimming(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"` A`", "`A123`"},
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx",
+ "file_type": "docx",
+ "output_format": "json",
+ "json": []map[string]any{{"text": " A123hello", "doc_type_kwd": "text"}},
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ // The delimiter that includes the leading whitespace matches the original
+ // text first, so "123" stays in the indexed content.
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"123hello"}) {
+ t.Fatalf("texts = %q, want [123hello]", texts)
+ }
+}
+
func TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {
units := []schema.ChunkDoc{
{Text: "plain paragraph one", DocType: "text"},