Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): preserve punctuation in general TXT and DOCX chunks (#20751)

jay77721bb18560be8b8
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.go
index d7205f65e5..50b112f5ac 100644
--- a/internal/ingestion/component/chunker/common.go
+++ b/internal/ingestion/component/chunker/common.go
@@ -109,10 +109,8 @@ func compileDelimPattern(delims []string) *regexp.Regexp {
// break, never a character to delete, so "。" must not vanish (or be swapped
// for "\n") after chunking.
//
-// When keepDelim is false the delimiter is DISCARDED (mirroring Python's
-// _split_text_by_pattern, which keeps only the even-index text parts); that
-// mode is used by the GeneralChunker legacy paths that still match Python's
-// drop behaviour, and by any caller that needs the historical contract.
+// When keepDelim is false the delimiter is discarded. Custom backtick
+// delimiters use this mode because they explicitly mark content to split away.
func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string {
if pattern == nil {
return []string{text}
@@ -156,6 +154,28 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string
return out
}
+// foldWhitespaceSegments attaches blank runs to neighboring text without
+// changing their position in the reconstructed source.
+func foldWhitespaceSegments(parts []string) []string {
+ var kept []string
+ var leading string
+ for _, part := range parts {
+ if strings.TrimSpace(part) == "" {
+ leading += part
+ continue
+ }
+ if leading != "" {
+ part = leading + part
+ leading = ""
+ }
+ kept = append(kept, part)
+ }
+ if leading != "" && len(kept) > 0 {
+ kept[len(kept)-1] += leading
+ }
+ return kept
+}
+
// ---------------------------------------------------------------------------
// chunk-doc helpers
// ---------------------------------------------------------------------------
diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
index 0841533af7..312583b740 100644
--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go
+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go
@@ -97,8 +97,8 @@ func TestGeneralChunkerTextAppliesOverlapThroughInvoke(t *testing.T) {
if len(texts) != 2 {
t.Fatalf("texts = %q, want two overlapped chunks", texts)
}
- if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {
- t.Fatalf("text overlap = %q, want previous tail plus current text", texts[1])
+ if texts[1] != " betagamma delta" {
+ t.Fatalf("text overlap = %q, want previous tail directly followed by current text", texts[1])
}
}
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index b53a204cb0..18b51484cb 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -231,7 +231,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, ups
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
- units = splitGeneralUnits(units, primaryPattern)
+ units = splitGeneralUnits(units, primaryPattern, false)
units = sortPDFUnits(units)
if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {
common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")
@@ -389,10 +389,18 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
- units = splitGeneralUnits(units, primaryPattern)
- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
- units = mergeDOCXUnits(units, c.param.ChunkTokenSize, hasCustomDelim(c.param.Delimiters), "\n")
- units = applyGeneralOverlap(units, c.param.OverlappedPercent, "\n")
+ customDelimiter := hasCustomDelim(c.param.Delimiters)
+ if !customDelimiter {
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+ units = separateDOCXParagraphs(units)
+ }
+ units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
+ if customDelimiter {
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
+ } else {
+ units = mergeDOCXUnits(units, c.param.ChunkTokenSize)
+ }
+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, "")
units = finalizeGeneralChunks(units, childrenPattern)
if len(units) == 0 {
return emptyOutputs(), nil
@@ -400,34 +408,50 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema
return chunkOutputs(units), nil
}
-// mergeDOCXUnits mirrors Python naive_merge_docx: text units keep the
-// previous text merge target across intervening table/image units. This means
-// a later paragraph can extend an earlier text chunk while the media item
-// remains in its original output position.
-func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, joinSep string) []schema.ChunkDoc {
+// separateDOCXParagraphs retains parser paragraph boundaries in the text before
+// delimiter splitting. Sentence fragments can then be concatenated directly.
+func separateDOCXParagraphs(units []schema.ChunkDoc) []schema.ChunkDoc {
+ units = append([]schema.ChunkDoc(nil), units...)
+ previousText := -1
+ for i := range units {
+ if itemDocType(units[i]) != "text" {
+ continue
+ }
+ if strings.TrimSpace(units[i].Text) == "" {
+ continue
+ }
+ if previousText >= 0 {
+ previous := units[previousText].Text
+ current := units[i].Text
+ // Splitting normalizes both CR and LF boundaries to LF.
+ if !strings.HasSuffix(previous, "\n") && !strings.HasSuffix(previous, "\r") &&
+ !strings.HasPrefix(current, "\n") && !strings.HasPrefix(current, "\r") {
+ units[previousText].Text += "\n"
+ }
+ }
+ previousText = i
+ }
+ return units
+}
+
+// mergeDOCXUnits keeps the previous text merge target across intervening
+// table/image units. Paragraph separators and sentence delimiters are already
+// carried by the text units.
+func mergeDOCXUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {
merged := make([]schema.ChunkDoc, 0, len(units))
previousText := -1
for _, unit := range units {
if itemDocType(unit) != "text" {
- media := cloneChunkDoc(unit)
- media.DocType = itemDocType(media)
- media.CKType = media.DocType
- merged = append(merged, media)
+ merged = append(merged, cloneChunkDoc(unit))
continue
}
- text := cloneChunkDoc(unit)
- text.DocType = "text"
- text.CKType = "text"
- text.TKNums = intPtr(generalUnitTokens(text))
- if previousText < 0 || customDelimiter || intValue(merged[previousText].TKNums) >= target {
- merged = append(merged, text)
+ if previousText < 0 || intValue(merged[previousText].TKNums) >= target {
+ merged = append(merged, cloneChunkDoc(unit))
previousText = len(merged) - 1
continue
}
- mergeGeneralChunk(&merged[previousText], text, joinSep)
- merged[previousText].DocType = "text"
- merged[previousText].CKType = "text"
+ mergeGeneralChunk(&merged[previousText], unit, "")
}
return merged
}
@@ -438,11 +462,12 @@ func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, j
// complete sentence units instead.
func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {
for i := range units {
- if units[i].CKType != "table" && units[i].CKType != "image" {
+ docType := itemDocType(units[i])
+ if docType != "table" && docType != "image" {
continue
}
budget := imageTokens
- if units[i].CKType == "table" {
+ if docType == "table" {
budget = tableTokens
}
if budget <= 0 {
@@ -468,6 +493,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abov
if !ok {
continue
}
+ text = normalizeGeneralNewlines(text)
tokens := tokenizeStr(text)
if tokens >= remaining {
piece := takeContextSentences(text, remaining, above)
@@ -994,18 +1020,11 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
}
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
- units = splitGeneralUnits(units, primaryPattern)
+ customDelimiter := hasCustomDelim(c.param.Delimiters)
+ units = splitGeneralUnits(units, primaryPattern, !customDelimiter)
attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
- if !hasCustomDelim(c.param.Delimiters) {
- // Python naive_merge prefixes each delimiter atom with a newline before
- // counting it. The prefix is a budgeting detail, not emitted content;
- // without it BPE counts can shift General's soft-cap boundaries.
- for i := range units {
- if itemDocType(units[i]) == "text" {
- units[i].TKNums = intPtr(tokenizeStr("\n" + units[i].Text))
- }
- }
- units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")
+ if !customDelimiter {
+ units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "")
}
units = finalizeGeneralChunks(units, childrenPattern)
if len(units) == 0 {
@@ -1170,7 +1189,7 @@ func joinGeneralOverlapText(prefix, text, separator string) string {
return prefix + separator + text
}
-func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {
+func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {
result := make([]schema.ChunkDoc, 0, len(units))
for _, unit := range units {
unit = cloneChunkDoc(unit)
@@ -1182,8 +1201,12 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema
result = append(result, unit)
continue
}
- for _, part := range splitByDelim(unit.Text, pattern, false) {
- if strings.TrimSpace(part) == "" {
+ parts := splitByDelim(unit.Text, pattern, keepDelimiter)
+ if keepDelimiter {
+ parts = foldWhitespaceSegments(parts)
+ }
+ for _, part := range parts {
+ if !keepDelimiter && strings.TrimSpace(part) == "" {
continue
}
piece := cloneChunkDoc(unit)
diff --git a/internal/ingestion/component/chunker/general_delimiter_test.go b/internal/ingestion/component/chunker/general_delimiter_test.go
new file mode 100644
index 0000000000..ac2ebcd1b4
--- /dev/null
+++ b/internal/ingestion/component/chunker/general_delimiter_test.go
@@ -0,0 +1,163 @@
+package chunker
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+
+ "ragflow/internal/parser/parser"
+)
+
+func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {
+ for _, fileType := range []string{"txt", "docx"} {
+ for _, source := range []string{
+ "第一句。第二句。",
+ "First!Second?Third;第四句;第五句!第六句?",
+ "\n第一句。\n\n第二句。\n",
+ "第一句。 第二句。\t\n",
+ } {
+ t.Run(fileType+"/"+source, func(t *testing.T) {
+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "document.txt", []byte(source))
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"\n", "!", "?", ";", "。", ";", "!", "?"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document." + fileType, "file_type": fileType,
+ "output_format": "json", "json": parsed.JSON,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := outputTexts(t, out); !reflect.DeepEqual(got, []string{source}) {
+ t.Fatalf("chunks = %q, want original text %q", got, source)
+ }
+ })
+ }
+ }
+}
+
+func TestGeneralChunkerPreservesDelimitersAcrossChunks(t *testing.T) {
+ const source = "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine."
+ for _, fileType := range []string{"txt", "docx"} {
+ t.Run(fileType, func(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 5,
+ "delimiters": []string{"\n", "!", "?", ";", "。", ";", "!", "?"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ input := generalTextInput(source)
+ input["name"] = "document." + fileType
+ input["file_type"] = fileType
+ out, err := component.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ texts := outputTexts(t, out)
+ if len(texts) < 2 {
+ t.Fatalf("chunks = %q, want multiple chunks", texts)
+ }
+ if got := strings.Join(texts, ""); got != source {
+ t.Fatalf("reconstructed text = %q, want %q", got, source)
+ }
+ })
+ }
+}
+
+func TestGeneralChunkerDOCXPreservesParagraphBoundaries(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ first string
+ second string
+ }{
+ {"implicit", "第一段。还有一句。", "第二段。"},
+ {"trailing CRLF", "第一段。还有一句。\r\n", "第二段。"},
+ {"trailing CR", "第一段。还有一句。\r", "第二段。"},
+ {"leading CRLF", "第一段。还有一句。", "\r\n第二段。"},
+ {"leading CR", "第一段。还有一句。", "\r第二段。"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 512,
+ "delimiters": []string{"\n", "。"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.docx", "file_type": "docx", "output_format": "json",
+ "json": []map[string]any{
+ {"text": tc.first, "doc_type_kwd": "text"},
+ {"text": tc.second, "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, want := outputTexts(t, out), []string{"第一段。还有一句。\n第二段。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("chunks = %q, want %q", got, want)
+ }
+ })
+ }
+}
+
+func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {
+ for _, fileType := range []string{"txt", "docx"} {
+ t.Run(fileType, func(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 1, "overlapped_percent": 50,
+ "delimiters": []string{"。"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ input := generalTextInput("甲乙。丙丁。")
+ input["name"] = "document." + fileType
+ input["file_type"] = fileType
+ out, err := component.Invoke(t.Context(), nil, input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got, want := outputTexts(t, out), []string{"甲乙。", "乙。丙丁。"}; !reflect.DeepEqual(got, want) {
+ t.Fatalf("chunks = %q, want %q", got, want)
+ }
+ })
+ }
+}
+
+func TestGeneralChunkerDelimiterSplitPreservesMetadata(t *testing.T) {
+ for _, fileType := range []string{"txt", "docx"} {
+ t.Run(fileType, func(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 0, "delimiters": []string{";"},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document." + fileType, "file_type": fileType, "output_format": "json",
+ "json": []map[string]any{{
+ "text": "alpha;beta", "doc_type_kwd": "text",
+ "summary": "source summary", "keywords": "source keywords", "source_order": 3,
+ }},
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ chunks := outputChunks(t, out)
+ if len(chunks) != 2 {
+ t.Fatalf("chunks = %#v, want two chunks", chunks)
+ }
+ for _, chunk := range chunks {
+ if chunk["summary"] != "source summary" || chunk["keywords"] != "source keywords" || chunk["source_order"] != float64(3) {
+ t.Fatalf("split lost source metadata: %#v", chunk)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index 678aca5260..887169b215 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -446,7 +446,7 @@ func TestGeneralChunkerBareDelimiterCreatesMergeAtoms(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha", "beta"}) {
+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha|", "beta"}) {
t.Fatalf("texts = %q", texts)
}
}
diff --git a/internal/ingestion/component/chunker/general_python_parity_test.go b/internal/ingestion/component/chunker/general_python_parity_test.go
deleted file mode 100644
index 4e467f7706..0000000000
--- a/internal/ingestion/component/chunker/general_python_parity_test.go
+++ /dev/null
@@ -1,65 +0,0 @@
-package chunker
-
-import (
- "context"
- "encoding/json"
- "os"
- "reflect"
- "strings"
- "testing"
-
- "ragflow/internal/parser/parser"
-)
-
-type generalTextPythonGolden struct {
- Input string `json:"input"`
- Params map[string]any `json:"params"`
- Chunks []struct {
- Text string `json:"text"`
- } `json:"chunks"`
-}
-
-// TestGeneralChunkerTextPythonParity verifies the post-refactor boundary:
-// Python's built-in general path parses and chunks text in one operation,
-// whereas Go sends the full normalized TextParser result to GeneralChunker.
-// The golden is captured from rag.app.naive.chunk and locks the resulting
-// chunk boundaries after removing Python's representation-only leading newline.
-func TestGeneralChunkerTextPythonParity(t *testing.T) {
- raw, err := os.ReadFile("testdata/general/text_default.python.golden.json")
- if err != nil {
- t.Fatalf("read Python golden: %v", err)
- }
- var golden generalTextPythonGolden
- if err := json.Unmarshal(raw, &golden); err != nil {
- t.Fatalf("decode Python golden: %v", err)
- }
-
- parsed := parser.NewTextParser().ParseWithResult(
- context.Background(), "general.txt", []byte(golden.Input),
- )
- if parsed.Err != nil {
- t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)
- }
- component, err := NewGeneralChunker(golden.Params)
- if err != nil {
- t.Fatalf("NewGeneralChunker: %v", err)
- }
- out, err := component.Invoke(t.Context(), nil, map[string]any{
- "name": "general.txt",
- "file_type": "txt",
- "output_format": "json",
- "json": parsed.JSON,
- })
- if err != nil {
- t.Fatalf("GeneralChunker.Invoke: %v", err)
- }
-
- got := outputTexts(t, out)
- want := make([]string, 0, len(golden.Chunks))
- for _, chunk := range golden.Chunks {
- want = append(want, strings.TrimPrefix(chunk.Text, "\n"))
- }
- if !reflect.DeepEqual(got, want) {
- t.Fatalf("GeneralChunker text chunks = %#v, want Python %v", got, want)
- }
-}
diff --git a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json
deleted file mode 100644
index 6b2c4aeb87..0000000000
--- a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
- "input": "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine.",
- "params": {
- "chunk_token_size": 5,
- "delimiters": [
- "\n",
- "!",
- "?",
- ";",
- "。",
- ";",
- "!",
- "?"
- ],
- "overlapped_percent": 0
- },
- "chunks": [
- {
- "text": "\nAlpha one.\nBeta two"
- },
- {
- "text": "\n Gamma three\nDelta four"
- },
- {
- "text": "\n Epsilon five\nZeta six"
- },
- {
- "text": "\nEta seven\nTheta eight"
- },
- {
- "text": "\nIota nine."
- }
- ]
-}
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index bee33050a1..00ac4cf52d 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -739,32 +739,7 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim bo
if !delimPattern.MatchString(txt) {
return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}
}
- // Fold whitespace-only pieces into an adjacent real piece instead of
- // dropping them, so a bare retained delimiter that produces a blank run
- // (the blank line between two "\n", or a leading "\n") survives in the
- // emitted text. This mirrors the text/children paths and keeps the chunk
- // reconstruction lossless for the common case.
- var kept []string
- var leading string
- for _, p := range parts {
- if strings.TrimSpace(p) == "" {
- leading += p
- continue
- }
- if leading != "" {
- p = leading + p
- leading = ""
- }
- kept = append(kept, p)
- }
- // A pending blank run belongs to the source and must be preserved. A run
- // that preceded the first real piece is already prepended during the loop;
- // any run left over after the last real piece (a trailing delimiter) is
- // appended to the last kept piece so the emitted text reconstructs the
- // source exactly instead of being shifted to the front.
- if leading != "" && len(kept) > 0 {
- kept[len(kept)-1] += leading
- }
+ kept := foldWhitespaceSegments(parts)
if len(kept) == 0 {
return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}
}