RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): attach media context on General markdown and HTML paths (#20141) (#20265)
Sourav Rajvi65f4b4805f08
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 9700c159b2..f533038218 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -602,6 +602,7 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream scprimaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)units = splitMarkdownUnits(units, primaryPattern)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")units = finalizeGeneralChunks(units, childrenPattern)if len(units) == 0 {@@ -619,6 +620,9 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schemunit = cloneChunkDoc(unit)unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))unit.DocType = itemDocType(unit)+ if unit.DocType == "table" || unit.DocType == "image" {+ unit.CKType = unit.DocType+ }if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {unit.TKNums = intPtr(tokenizeStr(unit.Text))result = append(result, unit)@@ -638,11 +642,37 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schemreturn result}+// stripTextLinesFromMediaContextAbove removes folded unit text from collected+// media context so materializeMediaContext does not emit it twice.+func stripTextLinesFromMediaContextAbove(contextAbove, folded string) string {+ folded = strings.TrimSpace(folded)+ if folded == "" || contextAbove == "" {+ return contextAbove+ }+ parts := strings.Split(contextAbove, "\n")+ filtered := make([]string, 0, len(parts))+ for _, part := range parts {+ if strings.TrimSpace(part) == folded {+ continue+ }+ filtered = append(filtered, part)+ }+ return strings.Join(filtered, "\n")+}++// markdownImageUnitHasMediaContext reports whether attachGeneralMediaContext+// populated context on a markdown image unit. Such units stay out of the text+// merge so ContextAbove and ContextBelow survive until materialization.+func markdownImageUnitHasMediaContext(unit schema.ChunkDoc) bool {+ return unit.ContextAbove != "" || unit.ContextBelow != ""+}+// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:// ordinary units use a projected token cap, while a short heading is always// kept with the following unit. Markdown images are block attachments rather// than standalone media chunks, so image-bearing units participate in the-// text merge and retain their image payload.+// text merge and retain their image payload. Images with configured media+// context are emitted as standalone chunks instead.func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {overlapPct = max(0, min(100, overlapPct))merged := make([]schema.ChunkDoc, 0, len(units))@@ -672,6 +702,7 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,table.Image = mergedImagetable.DocType = "table"table.CKType = "table"+ table.ContextAbove = stripTextLinesFromMediaContextAbove(table.ContextAbove, heading.Text)merged[current] = tablecurrent = -1currentTokens = 0@@ -683,6 +714,21 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,continue}+ if itemDocType(unit) == "image" && markdownImageUnitHasMediaContext(unit) {+ if current >= 0 {+ current = -1+ currentTokens = 0+ }+ standalone := cloneChunkDoc(unit)+ standalone.DocType = "image"+ if standalone.CKType == "" {+ standalone.CKType = "image"+ }+ standalone.TKNums = intPtr(generalUnitTokens(standalone))+ merged = append(merged, standalone)+ continue+ }+unit = cloneChunkDoc(unit)unit.DocType = "text"if unit.CKType == "" {@@ -937,6 +983,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream schprimaryPattern := compileDelimPattern(c.param.Delimiters)childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)units = splitGeneralUnits(units, primaryPattern)+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)if !hasCustomDelim(c.param.Delimiters) {// Python naive_merge prefixes each delimiter atom with a newline before// counting it. The prefix is a budgeting detail, not emitted content;diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex 53d8e21d1d..678aca5260 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -201,6 +201,73 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {}}+func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},+ {+ Text: "<table></table>",+ DocType: "table",+ CKType: "table",+ ContextAbove: "## Section",+ TKNums: intPtr(1),+ },+ }+ got := mergeMarkdownUnits(units, 10, 0, "\n")+ if len(got) != 1 {+ t.Fatalf("chunks = %#v, want one table chunk", got)+ }+ if got[0].ContextAbove != "" {+ t.Fatalf("ContextAbove = %q, want empty after heading fold", got[0].ContextAbove)+ }+ if got[0].Text != "## Section\n<table></table>" {+ t.Fatalf("text = %q, want heading folded into body", got[0].Text)+ }+}++func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "intro", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},+ {+ Text: "<table></table>",+ DocType: "table",+ CKType: "table",+ ContextAbove: "intro\n## Section",+ TKNums: intPtr(1),+ },+ }+ got := mergeMarkdownUnits(units, 10, 0, "\n")+ if len(got) != 2 {+ t.Fatalf("chunks = %#v, want intro text and table", got)+ }+ if got[1].ContextAbove != "intro" {+ t.Fatalf("ContextAbove = %q, want intro only", got[1].ContextAbove)+ }+}++func TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {+ units := []schema.ChunkDoc{+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ {+ Text: "figure",+ DocType: "image",+ CKType: "image",+ ContextAbove: "before",+ ContextBelow: "after",+ Image: "data:image/png;base64,AAAA",+ TKNums: intPtr(1),+ },+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},+ }+ got := mergeMarkdownUnits(units, 10, 0, "\n")+ if len(got) != 3 {+ t.Fatalf("chunks = %d, want text, image, text", len(got))+ }+ if got[1].CKType != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {+ t.Fatalf("image chunk = %#v, want preserved media context", got[1])+ }+}+func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {units := []schema.ChunkDoc{{Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex 455fa6f1cf..c384ab8d59 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -580,6 +580,185 @@ func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantText}}+func TestGeneralChunkerMarkdownAttachesMediaContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var table map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "table" {+ table = ck+ break+ }+ }+ if table == nil {+ t.Fatalf("table chunk missing: %#v", chunks)+ }+ assertMaterializedMediaContext(t, table, "before<table><tr><td>A</td></tr></table>after")+}++func TestGeneralChunkerMarkdownTableDoesNotDuplicateShortHeadingInContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var table map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "table" {+ table = ck+ break+ }+ }+ if table == nil {+ t.Fatalf("table chunk missing: %#v", chunks)+ }+ want := "## Section\n<table><tr><td>A</td></tr></table>"+ assertMaterializedMediaContext(t, table, want)+ if strings.Count(table["text"].(string), "## Section") != 1 {+ t.Fatalf("heading repeated in table chunk: %q", table["text"])+ }+}++func TestGeneralChunkerMarkdownTableKeepsEarlierContextWhenHeadingFolds(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 3,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "intro", "doc_type_kwd": "text"},+ {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var table map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "table" {+ table = ck+ break+ }+ }+ if table == nil {+ t.Fatalf("table chunk missing: %#v", chunks)+ }+ want := "intro## Section\n<table><tr><td>A</td></tr></table>"+ assertMaterializedMediaContext(t, table, want)+}++func TestGeneralChunkerMarkdownImageKeepsMediaContextWhenConfigured(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "image_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "document.md",+ "file_type": "md",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "figure", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var image map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "image" {+ image = ck+ break+ }+ }+ if image == nil {+ t.Fatalf("image chunk missing: %#v", chunks)+ }+ assertMaterializedMediaContext(t, image, "beforefigureafter")+}++func TestGeneralChunkerHTMLAttachesMediaContext(t *testing.T) {+ component, err := NewGeneralChunker(map[string]any{+ "chunk_token_size": 10,+ "table_context_size": 2,+ })+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ out, err := component.Invoke(t.Context(), nil, map[string]any{+ "name": "page.html",+ "file_type": "html",+ "output_format": "json",+ "json": []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},+ {"text": "after", "doc_type_kwd": "text"},+ },+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := outputChunks(t, out)+ var table map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "table" {+ table = ck+ break+ }+ }+ if table == nil {+ t.Fatalf("table chunk missing: %#v", chunks)+ }+ assertMaterializedMediaContext(t, table, "before<table><tr><td>A</td></tr></table>after")+}+func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {component, err := NewGeneralChunker(map[string]any{"chunk_token_size": 10,