RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): preserve punctuation in general TXT and DOCX chunks (#20751)
jay77721bb18560be8b8
diff --git a/internal/ingestion/component/chunker/common.go b/internal/ingestion/component/chunker/common.goindex d7205f65e5..50b112f5ac 100644--- a/internal/ingestion/component/chunker/common.go+++ b/internal/ingestion/component/chunker/common.go@@ -109,10 +109,8 @@ func compileDelimPattern(delims []string) *regexp.Regexp {// break, never a character to delete, so "。" must not vanish (or be swapped// for "\n") after chunking.//-// When keepDelim is false the delimiter is DISCARDED (mirroring Python's-// _split_text_by_pattern, which keeps only the even-index text parts); that-// mode is used by the GeneralChunker legacy paths that still match Python's-// drop behaviour, and by any caller that needs the historical contract.+// When keepDelim is false the delimiter is discarded. Custom backtick+// delimiters use this mode because they explicitly mark content to split away.func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []string {if pattern == nil {return []string{text}@@ -156,6 +154,28 @@ func splitByDelim(text string, pattern *regexp.Regexp, keepDelim bool) []stringreturn out}+// foldWhitespaceSegments attaches blank runs to neighboring text without+// changing their position in the reconstructed source.+func foldWhitespaceSegments(parts []string) []string {+ var kept []string+ var leading string+ for _, part := range parts {+ if strings.TrimSpace(part) == "" {+ leading += part+ continue+ }+ if leading != "" {+ part = leading + part+ leading = ""+ }+ kept = append(kept, part)+ }+ if leading != "" && len(kept) > 0 {+ kept[len(kept)-1] += leading+ }+ return kept+}+// ---------------------------------------------------------------------------// chunk-doc helpers// ---------------------------------------------------------------------------diff --git a/internal/ingestion/component/chunker/coverage_blind_spots_test.go b/internal/ingestion/component/chunker/coverage_blind_spots_test.goindex 0841533af7..312583b740 100644--- a/internal/ingestion/component/chunker/coverage_blind_spots_test.go+++ b/internal/ingestion/component/chunker/coverage_blind_spots_test.go@@ -97,8 +97,8 @@ func TestGeneralChunkerTextAppliesOverlapThroughInvoke(t *testing.T) {if len(texts) != 2 {t.Fatalf("texts = %q, want two overlapped chunks", texts)}- if texts[1] == "gamma delta" || !strings.HasSuffix(texts[1], "\ngamma delta") {- t.Fatalf("text overlap = %q, want previous tail plus current text", texts[1])+ if texts[1] != " betagamma delta" {+ t.Fatalf("text overlap = %q, want previous tail directly followed by current text", texts[1])}}diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex b53a204cb0..18b51484cb 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -231,7 +231,7 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsprimaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)- units = splitGeneralUnits(units, primaryPattern)+ units = splitGeneralUnits(units, primaryPattern, false)units = sortPDFUnits(units)if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")@@ -389,10 +389,18 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schema}primaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)- units = splitGeneralUnits(units, primaryPattern)- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)- units = mergeDOCXUnits(units, c.param.ChunkTokenSize, hasCustomDelim(c.param.Delimiters), "\n")- units = applyGeneralOverlap(units, c.param.OverlappedPercent, "\n")+ customDelimiter := hasCustomDelim(c.param.Delimiters)+ if !customDelimiter {+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ units = separateDOCXParagraphs(units)+ }+ units = splitGeneralUnits(units, primaryPattern, !customDelimiter)+ if customDelimiter {+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ } else {+ units = mergeDOCXUnits(units, c.param.ChunkTokenSize)+ }+ units = applyGeneralOverlap(units, c.param.OverlappedPercent, "")units = finalizeGeneralChunks(units, childrenPattern)if len(units) == 0 {return emptyOutputs(), nil@@ -400,34 +408,50 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemareturn chunkOutputs(units), nil}-// mergeDOCXUnits mirrors Python naive_merge_docx: text units keep the-// previous text merge target across intervening table/image units. This means-// a later paragraph can extend an earlier text chunk while the media item-// remains in its original output position.-func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, joinSep string) []schema.ChunkDoc {+// separateDOCXParagraphs retains parser paragraph boundaries in the text before+// delimiter splitting. Sentence fragments can then be concatenated directly.+func separateDOCXParagraphs(units []schema.ChunkDoc) []schema.ChunkDoc {+ units = append([]schema.ChunkDoc(nil), units...)+ previousText := -1+ for i := range units {+ if itemDocType(units[i]) != "text" {+ continue+ }+ if strings.TrimSpace(units[i].Text) == "" {+ continue+ }+ if previousText >= 0 {+ previous := units[previousText].Text+ current := units[i].Text+ // Splitting normalizes both CR and LF boundaries to LF.+ if !strings.HasSuffix(previous, "\n") && !strings.HasSuffix(previous, "\r") &&+ !strings.HasPrefix(current, "\n") && !strings.HasPrefix(current, "\r") {+ units[previousText].Text += "\n"+ }+ }+ previousText = i+ }+ return units+}++// mergeDOCXUnits keeps the previous text merge target across intervening+// table/image units. Paragraph separators and sentence delimiters are already+// carried by the text units.+func mergeDOCXUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {merged := make([]schema.ChunkDoc, 0, len(units))previousText := -1for _, unit := range units {if itemDocType(unit) != "text" {- media := cloneChunkDoc(unit)- media.DocType = itemDocType(media)- media.CKType = media.DocType- merged = append(merged, media)+ merged = append(merged, cloneChunkDoc(unit))continue}- text := cloneChunkDoc(unit)- text.DocType = "text"- text.CKType = "text"- text.TKNums = intPtr(generalUnitTokens(text))- if previousText < 0 || customDelimiter || intValue(merged[previousText].TKNums) >= target {- merged = append(merged, text)+ if previousText < 0 || intValue(merged[previousText].TKNums) >= target {+ merged = append(merged, cloneChunkDoc(unit))previousText = len(merged) - 1continue}- mergeGeneralChunk(&merged[previousText], text, joinSep)- merged[previousText].DocType = "text"- merged[previousText].CKType = "text"+ mergeGeneralChunk(&merged[previousText], unit, "")}return merged}@@ -438,11 +462,12 @@ func mergeDOCXUnits(units []schema.ChunkDoc, target int, customDelimiter bool, j// complete sentence units instead.func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {for i := range units {- if units[i].CKType != "table" && units[i].CKType != "image" {+ docType := itemDocType(units[i])+ if docType != "table" && docType != "image" {continue}budget := imageTokens- if units[i].CKType == "table" {+ if docType == "table" {budget = tableTokens}if budget <= 0 {@@ -468,6 +493,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abovif !ok {continue}+ text = normalizeGeneralNewlines(text)tokens := tokenizeStr(text)if tokens >= remaining {piece := takeContextSentences(text, remaining, above)@@ -994,18 +1020,11 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch}primaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)- units = splitGeneralUnits(units, primaryPattern)+ customDelimiter := hasCustomDelim(c.param.Delimiters)+ units = splitGeneralUnits(units, primaryPattern, !customDelimiter)attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)- if !hasCustomDelim(c.param.Delimiters) {- // Python naive_merge prefixes each delimiter atom with a newline before- // counting it. The prefix is a budgeting detail, not emitted content;- // without it BPE counts can shift General's soft-cap boundaries.- for i := range units {- if itemDocType(units[i]) == "text" {- units[i].TKNums = intPtr(tokenizeStr("\n" + units[i].Text))- }- }- units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")+ if !customDelimiter {+ units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "")}units = finalizeGeneralChunks(units, childrenPattern)if len(units) == 0 {@@ -1170,7 +1189,7 @@ func joinGeneralOverlapText(prefix, text, separator string) string {return prefix + separator + text}-func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {+func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) []schema.ChunkDoc {result := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {unit = cloneChunkDoc(unit)@@ -1182,8 +1201,12 @@ func splitGeneralUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schemaresult = append(result, unit)continue}- for _, part := range splitByDelim(unit.Text, pattern, false) {- if strings.TrimSpace(part) == "" {+ parts := splitByDelim(unit.Text, pattern, keepDelimiter)+ if keepDelimiter {+ parts = foldWhitespaceSegments(parts)+ }+ for _, part := range parts {+ if !keepDelimiter && strings.TrimSpace(part) == "" {continue}piece := cloneChunkDoc(unit)diff --git a/internal/ingestion/component/chunker/general_delimiter_test.go b/internal/ingestion/component/chunker/general_delimiter_test.gonew file mode 100644index 0000000000..ac2ebcd1b4--- /dev/null+++ b/internal/ingestion/component/chunker/general_delimiter_test.go@@ -0,0 +1,163 @@+package chunker++import (+ "reflect"+ "strings"+ "testing"++ "ragflow/internal/parser/parser"+)++func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {+ for _, fileType := range []string{"txt", "docx"} {+ for _, source := range []string{+ "第一句。第二句。",+ "First!Second?Third;第四句;第五句!第六句?",+ "\n第一句。\n\n第二句。\n",+ "第一句。 第二句。\t\n",+ } {+ t.Run(fileType+"/"+source, func(t *testing.T) {+ parsed := parser.NewTextParser().ParseWithResult(t.Context(), "document.txt", []byte(source))+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"\n", "!", "?", ";", "。", ";", "!", "?"},+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document." + fileType, "file_type": fileType,+ "output_format": "json", "json": parsed.JSON,+ })+ if err != nil {+ t.Fatal(err)+ }+ if got := outputTexts(t, out); !reflect.DeepEqual(got, []string{source}) {+ t.Fatalf("chunks = %q, want original text %q", got, source)+ }+ })+ }+ }+}++func TestGeneralChunkerPreservesDelimitersAcrossChunks(t *testing.T) {+ const source = "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine."+ for _, fileType := range []string{"txt", "docx"} {+ t.Run(fileType, func(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 5,+ "delimiters": []string{"\n", "!", "?", ";", "。", ";", "!", "?"},+ })+ if err != nil {+ t.Fatal(err)+ }+ input := generalTextInput(source)+ input["name"] = "document." + fileType+ input["file_type"] = fileType+ out, err := component.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatal(err)+ }+ texts := outputTexts(t, out)+ if len(texts) < 2 {+ t.Fatalf("chunks = %q, want multiple chunks", texts)+ }+ if got := strings.Join(texts, ""); got != source {+ t.Fatalf("reconstructed text = %q, want %q", got, source)+ }+ })+ }+}++func TestGeneralChunkerDOCXPreservesParagraphBoundaries(t *testing.T) {+ for _, tc := range []struct {+ name string+ first string+ second string+ }{+ {"implicit", "第一段。还有一句。", "第二段。"},+ {"trailing CRLF", "第一段。还有一句。\r\n", "第二段。"},+ {"trailing CR", "第一段。还有一句。\r", "第二段。"},+ {"leading CRLF", "第一段。还有一句。", "\r\n第二段。"},+ {"leading CR", "第一段。还有一句。", "\r第二段。"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512,+ "delimiters": []string{"\n", "。"},+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.docx", "file_type": "docx", "output_format": "json",+ "json": []map[string]any{+ {"text": tc.first, "doc_type_kwd": "text"},+ {"text": tc.second, "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ if got, want := outputTexts(t, out), []string{"第一段。还有一句。\n第二段。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("chunks = %q, want %q", got, want)+ }+ })+ }+}++func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {+ for _, fileType := range []string{"txt", "docx"} {+ t.Run(fileType, func(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 1, "overlapped_percent": 50,+ "delimiters": []string{"。"},+ })+ if err != nil {+ t.Fatal(err)+ }+ input := generalTextInput("甲乙。丙丁。")+ input["name"] = "document." + fileType+ input["file_type"] = fileType+ out, err := component.Invoke(t.Context(), nil, input)+ if err != nil {+ t.Fatal(err)+ }+ if got, want := outputTexts(t, out), []string{"甲乙。", "乙。丙丁。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("chunks = %q, want %q", got, want)+ }+ })+ }+}++func TestGeneralChunkerDelimiterSplitPreservesMetadata(t *testing.T) {+ for _, fileType := range []string{"txt", "docx"} {+ t.Run(fileType, func(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 0, "delimiters": []string{";"},+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document." + fileType, "file_type": fileType, "output_format": "json",+ "json": []map[string]any{{+ "text": "alpha;beta", "doc_type_kwd": "text",+ "summary": "source summary", "keywords": "source keywords", "source_order": 3,+ }},+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v, want two chunks", chunks)+ }+ for _, chunk := range chunks {+ if chunk["summary"] != "source summary" || chunk["keywords"] != "source keywords" || chunk["source_order"] != float64(3) {+ t.Fatalf("split lost source metadata: %#v", chunk)+ }+ }+ })+ }+}diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex 678aca5260..887169b215 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -446,7 +446,7 @@ func TestGeneralChunkerBareDelimiterCreatesMergeAtoms(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha", "beta"}) {+ if texts := outputTexts(t, out); !reflect.DeepEqual(texts, []string{"alpha|", "beta"}) {t.Fatalf("texts = %q", texts)}}diff --git a/internal/ingestion/component/chunker/general_python_parity_test.go b/internal/ingestion/component/chunker/general_python_parity_test.godeleted file mode 100644index 4e467f7706..0000000000--- a/internal/ingestion/component/chunker/general_python_parity_test.go+++ /dev/null@@ -1,65 +0,0 @@-package chunker--import (- "context"- "encoding/json"- "os"- "reflect"- "strings"- "testing"-- "ragflow/internal/parser/parser"-)--type generalTextPythonGolden struct {- Input string `json:"input"`- Params map[string]any `json:"params"`- Chunks []struct {- Text string `json:"text"`- } `json:"chunks"`-}--// TestGeneralChunkerTextPythonParity verifies the post-refactor boundary:-// Python's built-in general path parses and chunks text in one operation,-// whereas Go sends the full normalized TextParser result to GeneralChunker.-// The golden is captured from rag.app.naive.chunk and locks the resulting-// chunk boundaries after removing Python's representation-only leading newline.-func TestGeneralChunkerTextPythonParity(t *testing.T) {- raw, err := os.ReadFile("testdata/general/text_default.python.golden.json")- if err != nil {- t.Fatalf("read Python golden: %v", err)- }- var golden generalTextPythonGolden- if err := json.Unmarshal(raw, &golden); err != nil {- t.Fatalf("decode Python golden: %v", err)- }-- parsed := parser.NewTextParser().ParseWithResult(- context.Background(), "general.txt", []byte(golden.Input),- )- if parsed.Err != nil {- t.Fatalf("TextParser.ParseWithResult: %v", parsed.Err)- }- component, err := NewGeneralChunker(golden.Params)- if err != nil {- t.Fatalf("NewGeneralChunker: %v", err)- }- out, err := component.Invoke(t.Context(), nil, map[string]any{- "name": "general.txt",- "file_type": "txt",- "output_format": "json",- "json": parsed.JSON,- })- if err != nil {- t.Fatalf("GeneralChunker.Invoke: %v", err)- }-- got := outputTexts(t, out)- want := make([]string, 0, len(golden.Chunks))- for _, chunk := range golden.Chunks {- want = append(want, strings.TrimPrefix(chunk.Text, "\n"))- }- if !reflect.DeepEqual(got, want) {- t.Fatalf("GeneralChunker text chunks = %#v, want Python %v", got, want)- }-}diff --git a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json b/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.jsondeleted file mode 100644index 6b2c4aeb87..0000000000--- a/internal/ingestion/component/chunker/testdata/general/text_default.python.golden.json+++ /dev/null@@ -1,34 +0,0 @@-{- "input": "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine.",- "params": {- "chunk_token_size": 5,- "delimiters": [- "\n",- "!",- "?",- ";",- "。",- ";",- "!",- "?"- ],- "overlapped_percent": 0- },- "chunks": [- {- "text": "\nAlpha one.\nBeta two"- },- {- "text": "\n Gamma three\nDelta four"- },- {- "text": "\n Epsilon five\nZeta six"- },- {- "text": "\nEta seven\nTheta eight"- },- {- "text": "\nIota nine."- }- ]-}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex bee33050a1..00ac4cf52d 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -739,32 +739,7 @@ func chunkFromItem(it schema.ChunkDoc, delimPattern *regexp.Regexp, keepDelim boif !delimPattern.MatchString(txt) {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}- // Fold whitespace-only pieces into an adjacent real piece instead of- // dropping them, so a bare retained delimiter that produces a blank run- // (the blank line between two "\n", or a leading "\n") survives in the- // emitted text. This mirrors the text/children paths and keeps the chunk- // reconstruction lossless for the common case.- var kept []string- var leading string- for _, p := range parts {- if strings.TrimSpace(p) == "" {- leading += p- continue- }- if leading != "" {- p = leading + p- leading = ""- }- kept = append(kept, p)- }- // A pending blank run belongs to the source and must be preserved. A run- // that preceded the first real piece is already prepended during the loop;- // any run left over after the last real piece (a trailing delimiter) is- // appended to the last kept piece so the emitted text reconstructs the- // source exactly instead of being shifted to the front.- if leading != "" && len(kept) > 0 {- kept[len(kept)-1] += leading- }+ kept := foldWhitespaceSegments(parts)if len(kept) == 0 {return []schema.ChunkDoc{buildChunkDoc(it, "text", txt, "", "")}}