RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): drop DOCX custom-delimiter record prefix (#21760)
jay7772141753ee5a1a3
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex a1ab96d8af..fa47e951aa 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -419,7 +419,7 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemaunits = separateGeneralTextBlocks(units)}if recordGrouped {- units = splitGeneralUnitsPreservingRecordPrefix(units, splitPattern, recordPrefixes)+ units = splitGeneralUnitsDroppingRecordPrefix(units, splitPattern, recordPrefixes)} else {units = splitGeneralUnits(units, splitPattern, !customDelimiter)}@@ -1330,20 +1330,29 @@ func joinGeneralOverlapText(prefix, text, separator string) string {return prefix + separator + text}-// splitGeneralUnitsPreservingRecordPrefix is splitGeneralUnits, except a-// leading custom-delimiter record prefix (for example "问:") is kept on the-// first fragment. Inline delimiter matches are still dropped in splitByDelim-// drop mode.-func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {+// splitGeneralUnitsDroppingRecordPrefix splits grouped DOCX records and drops+// the leading custom-delimiter record prefix (for example "问:"). A custom+// (backtick) delimiter is a split instruction, not content: Python+// naive_merge_docx/_build_cks drops it and strips each flushed record. Inline+// delimiter matches are dropped by splitByDelim drop mode.+func splitGeneralUnitsDroppingRecordPrefix(units []schema.ChunkDoc, pattern *regexp.Regexp, recordPrefixes []string) []schema.ChunkDoc {result := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {unit = cloneChunkDoc(unit)unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))unit.DocType = itemDocType(unit)clearStructuralLayout(&unit)- prefix, rest := peelLeadingRecordPrefix(unit.Text, recordPrefixes)- if unit.DocType != "text" || pattern == nil || !pattern.MatchString(rest) {- unit.Text = prefix + rest+ if unit.DocType != "text" {+ unit.TKNums = intPtr(tokenizeStr(unit.Text))+ result = append(result, unit)+ continue+ }+ rest := dropLeadingRecordPrefix(unit.Text, recordPrefixes)+ if pattern == nil || !pattern.MatchString(rest) {+ unit.Text = strings.TrimSpace(rest)+ if unit.Text == "" && !hasMediaPayload(unit) {+ continue+ }unit.TKNums = intPtr(tokenizeStr(unit.Text))result = append(result, unit)continue@@ -1351,14 +1360,12 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *rparts := splitByDelim(rest, pattern, false)filtered := make([]string, 0, len(parts))for _, part := range parts {- if strings.TrimSpace(part) == "" {+ part = strings.TrimSpace(part)+ if part == "" {continue}filtered = append(filtered, part)}- if len(filtered) > 0 {- filtered[0] = prefix + filtered[0]- }for _, piece := range sliceGeneralSplitPieces(unit, filtered) {result = append(result, piece)}@@ -1366,24 +1373,24 @@ func splitGeneralUnitsPreservingRecordPrefix(units []schema.ChunkDoc, pattern *rreturn result}-func peelLeadingRecordPrefix(text string, prefixes []string) (prefix, rest string) {- if len(prefixes) == 0 {- return "", text+// dropLeadingRecordPrefix removes the first matching custom-delimiter record+// prefix from a grouped DOCX record. The original text is matched first so a+// delimiter that includes leading whitespace wins over a shorter prefix found+// only after trimming, mirroring the compiled pattern's earliest match. The+// whitespace-trimmed text is a fallback for delimiters after indentation.+func dropLeadingRecordPrefix(text string, prefixes []string) string {+ for _, prefix := range prefixes {+ if prefix != "" && strings.HasPrefix(text, prefix) {+ return text[len(prefix):]+ }}trimmed := strings.TrimLeftFunc(text, unicode.IsSpace)- leadingWS := text[:len(text)-len(trimmed)]- for _, p := range prefixes {- if p == "" {- continue- }- if strings.HasPrefix(text, p) {- return p, text[len(p):]- }- if strings.HasPrefix(trimmed, p) {- return leadingWS + p, trimmed[len(p):]+ for _, prefix := range prefixes {+ if prefix != "" && strings.HasPrefix(trimmed, prefix) {+ return trimmed[len(prefix):]}}- return "", text+ return text}func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex 9a30dc02ec..1e0307e6ed 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -899,14 +899,104 @@ func TestGeneralChunkerDOCXCustomDelimiterGroupsMultiParagraphRecords(t *testingif len(texts) != 2 {t.Fatalf("texts = %q, want two record chunks", texts)}- if !strings.Contains(texts[0], "问:first question") || !strings.Contains(texts[0], "名 称:item one") || !strings.Contains(texts[0], "答:first answer") {+ // A custom delimiter is a split instruction, not content: the leading+ // record prefix is dropped (Python naive_merge_docx/_build_cks parity).+ if strings.Contains(texts[0], "问:") || !strings.Contains(texts[0], "first question") || !strings.Contains(texts[0], "名 称:item one") || !strings.Contains(texts[0], "答:first answer") {t.Fatalf("first record = %q", texts[0])}- if !strings.Contains(texts[1], "问:second question") || !strings.Contains(texts[1], "答:second answer") {+ if strings.Contains(texts[1], "问:") || !strings.Contains(texts[1], "second question") || !strings.Contains(texts[1], "答:second answer") {t.Fatalf("second record = %q", texts[1])}}+func TestGeneralChunkerDOCXDropsCustomDelimiterRecordPrefix(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"`AAA`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "first section", "doc_type_kwd": "text"},+ {"text": "AAA", "doc_type_kwd": "text"},+ {"text": "second section", "doc_type_kwd": "text"},+ {"text": "AAA", "doc_type_kwd": "text"},+ {"text": "before AAA after", "doc_type_kwd": "text"},+ {"text": "AAA", "doc_type_kwd": "text"},+ {"text": "third section", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ texts := outputTexts(t, out)+ for _, text := range texts {+ if strings.Contains(text, "AAA") {+ t.Fatalf("custom delimiter survived: %q", text)+ }+ }+ if !reflect.DeepEqual(texts, []string{"first section", "second section", "before", "after", "third section"}) {+ t.Fatalf("texts = %q", texts)+ }+}++func TestGeneralChunkerDOCXDropsTrailingDelimiterOnlyRecord(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "overlapped_percent": 50,+ "delimiters": []string{"`AAA`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "AAAfirst", "doc_type_kwd": "text"},+ {"text": "AAA", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ // The delimiter-only record must not survive as an empty unit: overlap+ // would otherwise synthesize a spurious chunk from the previous tail.+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"first"}) {+ t.Fatalf("texts = %q, want [first]", texts)+ }+}++func TestGeneralChunkerDOCXPrefersDelimiterMatchBeforeTrimming(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"` A`", "`A123`"},+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx",+ "file_type": "docx",+ "output_format": "json",+ "json": []map[string]any{{"text": " A123hello", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ // The delimiter that includes the leading whitespace matches the original+ // text first, so "123" stays in the indexed content.+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"123hello"}) {+ t.Fatalf("texts = %q, want [123hello]", texts)+ }+}+func TestGroupDOCXUnitsByRecordBoundaryInactiveWithoutBoundary(t *testing.T) {units := []schema.ChunkDoc{{Text: "plain paragraph one", DocType: "text"},