RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): preserve PDF and Markdown sentence delimiters (#20812)
jay77721845e94d701a7
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 18b51484cb..e921f4bad1 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -231,12 +231,15 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsprimaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)- units = splitGeneralUnits(units, primaryPattern, false)+ customDelimiter := hasCustomDelim(c.param.Delimiters)+ if customDelimiter {+ units = splitGeneralUnits(units, primaryPattern, false)+ }units = sortPDFUnits(units)if hasPDFPositions(units) && hasUnpositionedPDFMedia(units) {common.Warn("GeneralChunker: PDF media is missing position metadata; using degraded context/order fallback")}- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)chunks := make([]schema.ChunkDoc, 0, len(units))body := make([]schema.ChunkDoc, 0, len(units))@@ -244,7 +247,13 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsif len(body) == 0 {return}- chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")...)+ joinSep := "\n"+ if !customDelimiter {+ body = separateGeneralTextBlocks(body)+ body = splitGeneralUnits(body, primaryPattern, true)+ joinSep = ""+ }+ chunks = append(chunks, mergeGeneralUnits(body, c.param.ChunkTokenSize, c.param.OverlappedPercent, joinSep)...)body = body[:0]}for _, unit := range units {@@ -252,7 +261,11 @@ func (c *GeneralChunkerComponent) chunkPDF(ctx context.Context, db *gorm.DB, upsbody = append(body, unit)} else {flushBody()- chunks = append(chunks, cloneChunkDoc(unit))+ media := cloneChunkDoc(unit)+ media.Text = normalizeGeneralNewlines(media.Text)+ media.DocType = itemDocType(media)+ media.CKType = media.DocType+ chunks = append(chunks, media)}}flushBody()@@ -391,12 +404,12 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemachildrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)if !customDelimiter {- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)- units = separateDOCXParagraphs(units)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)+ units = separateGeneralTextBlocks(units)}units = splitGeneralUnits(units, primaryPattern, !customDelimiter)if customDelimiter {- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)} else {units = mergeDOCXUnits(units, c.param.ChunkTokenSize)}@@ -408,9 +421,9 @@ func (c *GeneralChunkerComponent) chunkDOCX(ctx context.Context, upstream schemareturn chunkOutputs(units), nil}-// separateDOCXParagraphs retains parser paragraph boundaries in the text before+// separateGeneralTextBlocks retains parser block boundaries in the text before// delimiter splitting. Sentence fragments can then be concatenated directly.-func separateDOCXParagraphs(units []schema.ChunkDoc) []schema.ChunkDoc {+func separateGeneralTextBlocks(units []schema.ChunkDoc) []schema.ChunkDoc {units = append([]schema.ChunkDoc(nil), units...)previousText := -1for i := range units {@@ -456,11 +469,16 @@ func mergeDOCXUnits(units []schema.ChunkDoc, target int) []schema.ChunkDoc {return merged}-// attachGeneralMediaContext follows Python's sentence-aware media context-// extraction. The shared TokenChunker helper intentionally returns the-// smallest token-fitting rune suffix/prefix; DOCX/PDF general strategies use-// complete sentence units instead.-func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int) {+// attachGeneralMediaContext selects complete configured delimiter units or+// sentences from the neighboring source blocks.+func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokens int, primaryPattern *regexp.Regexp) {+ if tableTokens <= 0 && imageTokens <= 0 {+ return+ }+ pattern := contextSentencePattern+ if primaryPattern != nil {+ pattern = regexp.MustCompile("(?:" + primaryPattern.String() + ")|(?:" + pattern.String() + ")")+ }for i := range units {docType := itemDocType(units[i])if docType != "table" && docType != "image" {@@ -473,12 +491,12 @@ func attachGeneralMediaContext(units []schema.ChunkDoc, tableTokens, imageTokensif budget <= 0 {continue}- units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true)- units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false)+ units[i].ContextAbove = collectGeneralMediaContext(units, i, budget, true, pattern)+ units[i].ContextBelow = collectGeneralMediaContext(units, i, budget, false, pattern)}}-func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool) string {+func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, above bool, pattern *regexp.Regexp) string {var parts []stringremaining := budgetstep := -1@@ -496,7 +514,7 @@ func collectGeneralMediaContext(units []schema.ChunkDoc, index, budget int, abovtext = normalizeGeneralNewlines(text)tokens := tokenizeStr(text)if tokens >= remaining {- piece := takeContextSentences(text, remaining, above)+ piece := takeContextSentences(text, remaining, above, pattern)if above {parts = append([]string{piece}, parts...)} else {@@ -568,13 +586,10 @@ func hasSpreadsheetIdentity(doc schema.ChunkDoc) bool {// combined token count reaches the budget, taking the last sentence even when// it overshoots: a sentence is the smallest unit a context window may cut at,// so a text without a sentence boundary comes back whole. Shared by the media-// context collectors of both chunkers (general.go, token.go).-func takeContextSentences(text string, budget int, fromEnd bool) string {- sentences := splitContextSentences(text)- if len(sentences) == 0 {- return text- }- // The sentence pieces are contiguous slices of text. Track byte offsets+// context collectors of both chunkers use the supplied boundary pattern.+func takeContextSentences(text string, budget int, fromEnd bool, pattern *regexp.Regexp) string {+ sentences := splitByDelim(text, pattern, true)+ // The sentence pieces cover contiguous source segments. Track byte offsets// instead of repeatedly prepending slices and joining the selected text;// each candidate is still tokenized independently because BPE token counts// are not additive across sentence boundaries.@@ -599,26 +614,6 @@ func takeContextSentences(text string, budget int, fromEnd bool) string {return text[:end]}-func splitContextSentences(text string) []string {- indices := contextSentencePattern.FindAllStringIndex(text, -1)- if len(indices) == 0 {- if text == "" {- return nil- }- return []string{text}- }- parts := make([]string, 0, len(indices)+1)- start := 0- for _, index := range indices {- parts = append(parts, text[start:index[1]])- start = index[1]- }- if start < len(text) {- parts = append(parts, text[start:])- }- return parts-}-func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream schema.ChunkerFromUpstream) (map[string]any, error) {if err := ctx.Err(); err != nil {return nil, fmt.Errorf("GeneralChunker: %w", err)@@ -632,9 +627,15 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream sc}primaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)- units = splitMarkdownUnits(units, primaryPattern)- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)- units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")+ customDelimiter := hasCustomDelim(c.param.Delimiters)+ if !customDelimiter {+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)+ }+ parts := splitMarkdownUnits(units, primaryPattern, !customDelimiter)+ if customDelimiter {+ attachGeneralMediaContext(parts.chunks, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)+ }+ units = mergeMarkdownUnits(parts, c.param.ChunkTokenSize, c.param.OverlappedPercent)units = finalizeGeneralChunks(units, childrenPattern)if len(units) == 0 {return emptyOutputs(), nil@@ -642,44 +643,74 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream screturn chunkOutputs(units), nil}+// markdownSplit keeps each fragment's parser block boundary beside its chunk.+type markdownSplit struct {+ chunks []schema.ChunkDoc+ blockStarts []bool+ shortHeadings []bool // Classified from the complete source heading.+}+// splitMarkdownUnits expands only text units while retaining the semantic// parser type on every piece. Markdown heading detection depends on ck_type,// so the generic splitter's type normalization cannot be used here.-func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schema.ChunkDoc {- result := make([]schema.ChunkDoc, 0, len(units))+func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp, keepDelimiter bool) markdownSplit {+ result := markdownSplit{+ chunks: make([]schema.ChunkDoc, 0, len(units)),+ blockStarts: make([]bool, 0, len(units)),+ shortHeadings: make([]bool, 0, len(units)),+ }for _, unit := range units {unit = cloneChunkDoc(unit)- unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))+ unit.Text = normalizeGeneralNewlines(itemTextOrFallback(unit))+ if !keepDelimiter || unit.CKType == "heading" {+ unit.Text = strings.TrimSpace(unit.Text)+ }unit.DocType = itemDocType(unit)if unit.DocType == "table" || unit.DocType == "image" {unit.CKType = unit.DocType}+ shortHeading := isShortMarkdownHeading(unit)if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {unit.TKNums = intPtr(tokenizeStr(unit.Text))- result = append(result, unit)+ result.chunks = append(result.chunks, unit)+ result.blockStarts = append(result.blockStarts, true)+ result.shortHeadings = append(result.shortHeadings, shortHeading)continue}- for _, part := range splitByDelim(unit.Text, pattern, false) {- part = strings.TrimSpace(part)- if part == "" {- continue+ parts := splitByDelim(unit.Text, pattern, keepDelimiter)+ if keepDelimiter {+ parts = foldWhitespaceSegments(parts)+ }+ startsBlock := true+ for _, part := range parts {+ if !keepDelimiter {+ part = strings.TrimSpace(part)+ if part == "" {+ continue+ }}piece := cloneChunkDoc(unit)piece.Text = partpiece.TKNums = intPtr(tokenizeStr(part))- result = append(result, piece)+ result.chunks = append(result.chunks, piece)+ result.blockStarts = append(result.blockStarts, startsBlock)+ result.shortHeadings = append(result.shortHeadings, shortHeading)+ startsBlock = !keepDelimiter}}return result}// stripTextLinesFromMediaContextAbove removes folded unit text from collected-// media context so materializeMediaContext does not emit it twice.+// media context, including a suffix selected by a small context budget.func stripTextLinesFromMediaContextAbove(contextAbove, folded string) string {folded = strings.TrimSpace(folded)if folded == "" || contextAbove == "" {return contextAbove}+ if strings.HasSuffix(folded, strings.TrimSpace(contextAbove)) {+ return ""+ }parts := strings.Split(contextAbove, "\n")filtered := make([]string, 0, len(parts))for _, part := range parts {@@ -698,20 +729,25 @@ func markdownImageUnitHasMediaContext(unit schema.ChunkDoc) bool {return unit.ContextAbove != "" || unit.ContextBelow != ""}-// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:-// ordinary units use a projected token cap, while a short heading is always+// mergeMarkdownUnits uses a projected token cap, while a short heading is always// kept with the following unit. Markdown images are block attachments rather// than standalone media chunks, so image-bearing units participate in the// text merge and retain their image payload. Images with configured media// context are emitted as standalone chunks instead.-func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {+func mergeMarkdownUnits(parts markdownSplit, target int, overlapPct float64) []schema.ChunkDoc {overlapPct = max(0, min(100, overlapPct))- merged := make([]schema.ChunkDoc, 0, len(units))+ merged := make([]schema.ChunkDoc, 0, len(parts.chunks))current := -1currentTokens := 0- for _, unit := range units {+ for index, unit := range parts.chunks {+ startsBlock := parts.blockStarts[index]+ previousShortHeading := index > 0 && parts.shortHeadings[index-1]+ joinSep := ""+ if startsBlock && current >= 0 && !strings.HasSuffix(merged[current].Text, "\n") && !strings.HasPrefix(unit.Text, "\n") {+ joinSep = "\n"+ }if itemDocType(unit) == "table" {- if current >= 0 && isShortMarkdownHeading(merged[current]) {+ if current >= 0 && merged[current].CKType == "heading" && previousShortHeading {mergedImage, imagesOK := mergeMarkdownImagesChecked(merged[current].Image, unit.Image)if !imagesOK {merged = append(merged, cloneChunkDoc(unit))@@ -775,13 +811,13 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,previous := &merged[current]unitTokens := generalUnitTokens(unit)- if unit.CKType == "heading" && previous.CKType != "heading" {+ if startsBlock && unit.CKType == "heading" && previous.CKType != "heading" {merged = append(merged, unit)current = len(merged) - 1currentTokens = unitTokenscontinue}- forceMerge := isShortMarkdownHeading(*previous)+ forceMerge := previous.CKType == "heading" && previousShortHeadingmergedImage, imagesOK := mergeMarkdownImagesChecked(previous.Image, unit.Image)if !imagesOK {merged = append(merged, unit)@@ -810,10 +846,12 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,current = len(merged) - 1continue}- mergeMarkdownChunk(previous, unit, joinSep)+ mergeGeneralChunk(previous, unit, joinSep)previous.Image = mergedImageprevious.DocType = "text"- previous.CKType = "text"+ if startsBlock || unit.CKType != "heading" {+ previous.CKType = "text"+ }currentTokens = projectedprevious.TKNums = intPtr(currentTokens)}@@ -842,10 +880,6 @@ func markdownHeadingContent(text string) string {return strings.TrimSpace(text[level:])}-func mergeMarkdownChunk(dst *schema.ChunkDoc, src schema.ChunkDoc, joinSep string) {- mergeGeneralChunk(dst, src, joinSep)-}-const (maxMarkdownImageDimension = 8192maxMarkdownImagePixels = 32 * 1024 * 1024@@ -925,7 +959,7 @@ func (c *GeneralChunkerComponent) chunkSpreadsheet(ctx context.Context, upstreamif len(units) == 0 {return emptyOutputs(), nil}- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, nil)chunks := make([]schema.ChunkDoc, 0, len(units))for _, unit := range units {if itemDocType(unit) == "table" && tableDataOf(unit) != nil {@@ -1022,7 +1056,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schchildrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)customDelimiter := hasCustomDelim(c.param.Delimiters)units = splitGeneralUnits(units, primaryPattern, !customDelimiter)- attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize, primaryPattern)if !customDelimiter {units = mergeGeneralUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "")}diff --git a/internal/ingestion/component/chunker/general_block_delimiter_test.go b/internal/ingestion/component/chunker/general_block_delimiter_test.gonew file mode 100644index 0000000000..2cb87e718e--- /dev/null+++ b/internal/ingestion/component/chunker/general_block_delimiter_test.go@@ -0,0 +1,184 @@+package chunker++import (+ "reflect"+ "testing"++ "ragflow/internal/parser/parser"+)++func TestGeneralChunkerPDFAndMarkdownPreserveBlockBoundaries(t *testing.T) {+ for _, fileType := range []string{"pdf", "md"} {+ for _, tc := range []struct {+ name, first, second string+ }{+ {"implicit", "第一段。还有一句。", "第二段。"},+ {"trailing CRLF", "第一段。还有一句。\r\n", "第二段。"},+ {"leading CR", "第一段。还有一句。", "\r第二段。"},+ } {+ t.Run(fileType+"/"+tc.name, func(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"\n", "。"}})+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "document." + fileType, "file_type": fileType, "output_format": "json",+ "json": []map[string]any{{"text": tc.first, "doc_type_kwd": "text"}, {"text": tc.second, "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatal(err)+ }+ if got, want := outputTexts(t, out), []string{"第一段。还有一句。\n第二段。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("got %q, want %q", got, want)+ }+ })+ }+ }+}++func TestGeneralChunkerPDFPunctuationPreservesMediaOrderAndContext(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512, "delimiters": []string{"\n", "。"}, "table_context_size": 512,+ })+ if err != nil {+ t.Fatal(err)+ }+ position := func(top int) [][]int { return [][]int{{1, 0, 10, top, top + 5}} }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "document.pdf", "file_type": "pdf", "output_format": "json",+ "json": []map[string]any{+ {"text": "后文。还有一句。", "doc_type_kwd": "text", "positions": position(30)},+ {"text": "<table>A</table>", "doc_type_kwd": "table", "positions": position(20)},+ {"text": "前文。还有一句。", "doc_type_kwd": "text", "positions": position(10)},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 || chunks[0]["text"] != "前文。还有一句。" || chunks[2]["text"] != "后文。还有一句。" {+ t.Fatalf("chunks = %#v", chunks)+ }+ assertMaterializedMediaContext(t, chunks[1], "前文。还有一句。<table>A</table>后文。还有一句。")+ if got, want := chunks[0]["positions"], [][]float64{{1, 0, 10, 10, 15}}; !reflect.DeepEqual(got, want) {+ t.Fatalf("positions = %#v, want %#v", got, want)+ }+}++func TestGeneralChunkerMarkdownParserPreservesPunctuationAndHeading(t *testing.T) {+ p, err := parser.NewMarkdownParser(parser.GoMarkdown)+ if err != nil {+ t.Fatal(err)+ }+ parsed := p.ParseWithResult(t.Context(), "document.md", []byte("# 标题。还有标题!\n\n第一句。第二句。\n\n下一段。"))+ if parsed.Err != nil {+ t.Fatal(parsed.Err)+ }+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"\n", "。", "!"}})+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{"name": "document.md", "file_type": "md", "output_format": "json", "json": parsed.JSON})+ if err != nil {+ t.Fatal(err)+ }+ if got, want := outputTexts(t, out), []string{"# 标题。还有标题!\n第一句。第二句。\n下一段。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("got %q, want %q", got, want)+ }+}++func TestGeneralChunkerPDFAndMarkdownCustomDelimiterStillDropsMarker(t *testing.T) {+ for _, fileType := range []string{"pdf", "md"} {+ t.Run(fileType, func(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512, "delimiters": []string{"`SEP`"}})+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "document." + fileType, "file_type": fileType, "output_format": "json",+ "json": []map[string]any{{"text": "第一句。SEP第二句。", "doc_type_kwd": "text"}},+ })+ if err != nil {+ t.Fatal(err)+ }+ if got, want := outputTexts(t, out), []string{"第一句。\n第二句。"}; !reflect.DeepEqual(got, want) {+ t.Fatalf("got %q, want %q", got, want)+ }+ })+ }+}++func TestGeneralChunkerPDFAndMarkdownCustomMediaContextDropsMarker(t *testing.T) {+ for _, fileType := range []string{"pdf", "md"} {+ t.Run(fileType, func(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512, "delimiters": []string{"`SEP`"}, "table_context_size": 512,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "document." + fileType, "file_type": fileType, "output_format": "json",+ "json": []map[string]any{+ {"text": "前文。SEP还有一句。", "doc_type_kwd": "text"},+ {"text": "<table>A</table>", "doc_type_kwd": "table"},+ {"text": "后文。", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 3 {+ t.Fatalf("chunks = %#v", chunks)+ }+ assertMaterializedMediaContext(t, chunks[1], "前文。\n还有一句。<table>A</table>后文。")+ })+ }+}++func TestGeneralChunkerMarkdownPunctuationPreservesMediaBoundaries(t *testing.T) {+ for _, tc := range []struct {+ name, docType, ckType, first, media string+ contextSize int+ want []string+ }{+ {"heading and table", "table", "heading", "标题。还有标题!", "<table>A</table>", 512,+ []string{"标题。还有标题!\n<table>A</table>后文。还有一句。", "后文。还有一句。"}},+ {"image attachment", "image", "text", "前文。还有一句。", "图注。", 0,+ []string{"前文。还有一句。\n图注。\n后文。还有一句。"}},+ {"standalone image", "image", "text", "前文。还有一句。", "图注。", 512,+ []string{"前文。还有一句。", "前文。还有一句。图注。后文。还有一句。", "后文。还有一句。"}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512, "delimiters": []string{"\n", "。", "!"},+ "table_context_size": tc.contextSize, "image_context_size": tc.contextSize,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md", "file_type": "md", "output_format": "json",+ "json": []map[string]any{+ {"text": tc.first, "doc_type_kwd": "text", "ck_type": tc.ckType},+ {"text": tc.media, "doc_type_kwd": tc.docType, "img_id": "media-id"},+ {"text": "后文。还有一句。", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ if got := outputTexts(t, out); !reflect.DeepEqual(got, tc.want) {+ t.Fatalf("got %q, want %q", got, tc.want)+ }+ foundMedia := false+ for _, chunk := range outputChunks(t, out) {+ foundMedia = foundMedia || chunk["img_id"] == "media-id"+ }+ if !foundMedia {+ t.Fatal("media identity was lost")+ }+ })+ }+}diff --git a/internal/ingestion/component/chunker/general_context_regression_test.go b/internal/ingestion/component/chunker/general_context_regression_test.gonew file mode 100644index 0000000000..e33dc23f0f--- /dev/null+++ b/internal/ingestion/component/chunker/general_context_regression_test.go@@ -0,0 +1,119 @@+package chunker++import (+ "strings"+ "testing"+)++func TestGeneralChunkerMediaContextUsesConfiguredDelimiters(t *testing.T) {+ for _, fileType := range []string{"pdf", "md"} {+ for _, tc := range []struct {+ name, delimiter, text, above, below string+ }{+ {"semicolon", ";", "first;middle;last;", "last;<table>A</table>", "<table>A</table>first;"},+ {"literal pipe", "|", "first|middle|last|", "last|<table>A</table>", "<table>A</table>first|"},+ {"multi-character", "。END", "first。ENDmiddle。ENDlast。END", "last。END<table>A</table>", "<table>A</table>first。END"},+ } {+ for _, direction := range []struct {+ name string+ above bool+ }{{"above", true}, {"below", false}} {+ t.Run(fileType+"/"+tc.name+"/"+direction.name, func(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512, "delimiters": []string{"\n", tc.delimiter}, "table_context_size": 1,+ })+ if err != nil {+ t.Fatal(err)+ }+ text := map[string]any{"text": tc.text, "doc_type_kwd": "text"}+ table := map[string]any{"text": "<table>A</table>", "doc_type_kwd": "table"}+ items := []map[string]any{table, text}+ tableIndex, want := 0, tc.below+ if direction.above {+ items = []map[string]any{text, table}+ tableIndex, want = 1, tc.above+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "test." + fileType, "file_type": fileType, "output_format": "json", "json": items,+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 2 {+ t.Fatalf("chunks = %#v", chunks)+ }+ assertMaterializedMediaContext(t, chunks[tableIndex], want)+ })+ }+ }+ }+}++func TestGeneralChunkerMarkdownFoldedHeadingDropsPartialContext(t *testing.T) {+ c, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 512, "delimiters": []string{"\n", "。", "!"}, "table_context_size": 1,+ })+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "test.md", "file_type": "md", "output_format": "json",+ "json": []map[string]any{+ {"text": "# 标题。还有标题!", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "<table>A</table>", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ chunks := outputChunks(t, out)+ if len(chunks) != 1 {+ t.Fatalf("chunks = %#v", chunks)+ }+ assertMaterializedMediaContext(t, chunks[0], "# 标题。还有标题!\n<table>A</table>")+}++func TestGeneralChunkerMarkdownLongHeadingUsesSourceClassification(t *testing.T) {+ for _, tc := range []struct {+ name, delimiter, marker string+ minChunks, maxWords int+ }{+ {"bare", "!", "! ", 20, 3},+ {"custom", "`SEP`", "SEP", 12, 5},+ } {+ t.Run(tc.name, func(t *testing.T) {+ heading := "# " + strings.Repeat("heading"+tc.marker, 60)+ c, err := NewGeneralChunker(map[string]any{"chunk_token_size": 5, "delimiters": []string{tc.delimiter}})+ if err != nil {+ t.Fatal(err)+ }+ out, err := c.Invoke(t.Context(), nil, map[string]any{+ "name": "test.md", "file_type": "md", "output_format": "json",+ "json": []map[string]any{+ {"text": heading, "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "body", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatal(err)+ }+ texts := outputTexts(t, out)+ if len(texts) < tc.minChunks {+ t.Fatalf("long heading produced only %d chunks: %q", len(texts), texts)+ }+ wordCount := 0+ for _, text := range texts {+ wordCount += strings.Count(text, "heading")+ // Retained punctuation costs tokens; removed custom markers+ // leave one-token atoms, so more occurrences fit the budget.+ if strings.Count(text, "heading") > tc.maxWords {+ t.Fatalf("long heading received the short-heading merge exception: %q", text)+ }+ }+ if wordCount != 60 || !strings.HasSuffix(texts[len(texts)-1], "body") {+ t.Fatalf("heading or body content was lost: %q", texts)+ }+ })+ }+}diff --git a/internal/ingestion/component/chunker/general_delimiter_test.go b/internal/ingestion/component/chunker/general_delimiter_test.goindex ac2ebcd1b4..5094b615bb 100644--- a/internal/ingestion/component/chunker/general_delimiter_test.go+++ b/internal/ingestion/component/chunker/general_delimiter_test.go@@ -9,7 +9,7 @@ import ()func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {- for _, fileType := range []string{"txt", "docx"} {+ for _, fileType := range []string{"txt", "docx", "pdf", "md"} {for _, source := range []string{"第一句。第二句。","First!Second?Third;第四句;第五句!第六句?",@@ -42,7 +42,7 @@ func TestGeneralChunkerPreservesBareDelimiters(t *testing.T) {func TestGeneralChunkerPreservesDelimitersAcrossChunks(t *testing.T) {const source = "Alpha one.\nBeta two; Gamma three!\nDelta four? Epsilon five。Zeta six;Eta seven!Theta eight?Iota nine."- for _, fileType := range []string{"txt", "docx"} {+ for _, fileType := range []string{"txt", "docx", "pdf", "md"} {t.Run(fileType, func(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 5,@@ -107,7 +107,7 @@ func TestGeneralChunkerDOCXPreservesParagraphBoundaries(t *testing.T) {}func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {- for _, fileType := range []string{"txt", "docx"} {+ for _, fileType := range []string{"txt", "docx", "pdf", "md"} {t.Run(fileType, func(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 1, "overlapped_percent": 50,@@ -131,7 +131,7 @@ func TestGeneralChunkerOverlapPreservesSentenceDelimiters(t *testing.T) {}func TestGeneralChunkerDelimiterSplitPreservesMetadata(t *testing.T) {- for _, fileType := range []string{"txt", "docx"} {+ for _, fileType := range []string{"txt", "docx", "pdf", "md"} {t.Run(fileType, func(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 0, "delimiters": []string{";"},diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex 887169b215..6717a209fb 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -135,7 +135,7 @@ func TestMergeMarkdownUnitsCarriesCharacterOverlapIntoNextBudget(t *testing.T) {}target := intValue(units[0].TKNums) + intValue(units[1].TKNums)- got := mergeMarkdownUnits(units, target, 50, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true, true}, shortHeadings: []bool{false, false, false, false}}, target, 50)want := []string{"alpha beta\ngamma delta","gamma delta\nepsilon zeta",@@ -174,7 +174,7 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {}target := intValue(units[0].TKNums)- got := mergeMarkdownUnits(units, target, 50, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true}, shortHeadings: []bool{false, false}}, target, 50)if len(got) != 2 {t.Fatalf("chunks = %#v, want previous and overlapped current chunks", got)}@@ -212,7 +212,7 @@ func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.TTKNums: intPtr(1),},}- got := mergeMarkdownUnits(units, 10, 0, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true}, shortHeadings: []bool{true, false}}, 10, 0)if len(got) != 1 {t.Fatalf("chunks = %#v, want one table chunk", got)}@@ -236,7 +236,7 @@ func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testiTKNums: intPtr(1),},}- got := mergeMarkdownUnits(units, 10, 0, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 10, 0)if len(got) != 2 {t.Fatalf("chunks = %#v, want intro text and table", got)}@@ -259,7 +259,7 @@ func TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {},{Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},}- got := mergeMarkdownUnits(units, 10, 0, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, false, false}}, 10, 0)if len(got) != 3 {t.Fatalf("chunks = %d, want text, image, text", len(got))}@@ -275,7 +275,7 @@ func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {{Text: "The architecture is split into parser and chunker.", DocType: "text", CKType: "text", TKNums: intPtr(1)},}- got := mergeMarkdownUnits(units, 3, 0, "\n")+ got := mergeMarkdownUnits(markdownSplit{chunks: units, blockStarts: []bool{true, true, true}, shortHeadings: []bool{false, true, false}}, 3, 0)if texts := generalChunkTexts(got); !reflect.DeepEqual(texts, []string{"Background details","## Architecture\nThe architecture is split into parser and chunker.",@@ -327,10 +327,10 @@ func TestMergeGeneralUnitsOverlapDoesNotCopyPreviousMediaMetadata(t *testing.T)func TestTakeContextSentencesPreservesBoundarySelection(t *testing.T) {text := "one!two!three!"- if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true), "two!three!"; got != want {+ if got, want := takeContextSentences(text, tokenizeStr("two!three!"), true, contextSentencePattern), "two!three!"; got != want {t.Fatalf("suffix context = %q, want %q", got, want)}- if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false), "one!two!"; got != want {+ if got, want := takeContextSentences(text, tokenizeStr("one!two!"), false, contextSentencePattern), "one!two!"; got != want {t.Fatalf("prefix context = %q, want %q", got, want)}}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex daeea3f92a..cbd9152d61 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -802,10 +802,10 @@ func TestGeneralMediaContextSeparatesAdjacentSourceUnits(t *testing.T) {{Text: "after two", DocType: "text", CKType: "text", TKNums: intPtr(1)},}- if got, want := collectGeneralMediaContext(units, 2, 10, true), "before one\nbefore two"; got != want {+ if got, want := collectGeneralMediaContext(units, 2, 10, true, contextSentencePattern), "before one\nbefore two"; got != want {t.Fatalf("above context = %q, want %q", got, want)}- if got, want := collectGeneralMediaContext(units, 2, 10, false), "after one\nafter two"; got != want {+ if got, want := collectGeneralMediaContext(units, 2, 10, false, contextSentencePattern), "after one\nafter two"; got != want {t.Fatalf("below context = %q, want %q", got, want)}}diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 00ac4cf52d..89d68eaf67 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -902,7 +902,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stritk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {- parts = append([]string{takeContextSentences(txt, remain, true)}, parts...)+ parts = append([]string{takeContextSentences(txt, remain, true, contextSentencePattern)}, parts...)remain = 0break}@@ -918,7 +918,7 @@ func collectContext(chunks []schema.ChunkDoc, i, ctxTokens int, above bool) stritk := intValue(chunks[pos].TKNums)txt := chunks[pos].Textif tk >= remain {- parts = append(parts, takeContextSentences(txt, remain, false))+ parts = append(parts, takeContextSentences(txt, remain, false, contextSentencePattern))remain = 0break}diff --git a/internal/ingestion/component/chunker/token_batch1_test.go b/internal/ingestion/component/chunker/token_batch1_test.goindex 45f14b7e33..32e3946853 100644--- a/internal/ingestion/component/chunker/token_batch1_test.go+++ b/internal/ingestion/component/chunker/token_batch1_test.go@@ -311,10 +311,10 @@ func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {const text = "alpha beta. gamma delta. epsilon zeta."budget := tokenizeStr("gamma delta. epsilon zeta.")- if got, want := takeContextSentences(text, budget, true), "gamma delta. epsilon zeta."; got != want {+ if got, want := takeContextSentences(text, budget, true, contextSentencePattern), "gamma delta. epsilon zeta."; got != want {t.Errorf("tail context = %q, want %q", got, want)}- if got, want := takeContextSentences(text, budget, false), "alpha beta. gamma delta. "; got != want {+ if got, want := takeContextSentences(text, budget, false, contextSentencePattern), "alpha beta. gamma delta. "; got != want {t.Errorf("prefix context = %q, want %q", got, want)}// One sentence less falls short of the budget, so the walk stopped at the@@ -330,10 +330,10 @@ func TestTakeContextSentencesCoversBudgetOnSentenceBoundaries(t *testing.T) {// — comes back whole.func TestTakeContextSentencesKeepsBoundarylessText(t *testing.T) {s := strings.Repeat("中", 60)- if got := takeContextSentences(s, 20, true); got != s {+ if got := takeContextSentences(s, 20, true, contextSentencePattern); got != s {t.Fatalf("context = %q, want the whole boundary-less text", got)}- if got := takeContextSentences(s, 20, false); got != s {+ if got := takeContextSentences(s, 20, false, contextSentencePattern); got != s {t.Fatalf("context = %q, want the whole boundary-less text", got)}}