Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): attach media context on General markdown and HTML paths (#20141) (#20265)

Sourav Rajvi65f4b4805f08
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 9700c159b2..f533038218 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -602,6 +602,7 @@ func (c *GeneralChunkerComponent) chunkMarkdown(ctx context.Context, upstream sc
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
units = splitMarkdownUnits(units, primaryPattern)
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
units = mergeMarkdownUnits(units, c.param.ChunkTokenSize, c.param.OverlappedPercent, "\n")
units = finalizeGeneralChunks(units, childrenPattern)
if len(units) == 0 {
@@ -619,6 +620,9 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schem
unit = cloneChunkDoc(unit)
unit.Text = strings.TrimSpace(normalizeGeneralNewlines(itemTextOrFallback(unit)))
unit.DocType = itemDocType(unit)
+ if unit.DocType == "table" || unit.DocType == "image" {
+ unit.CKType = unit.DocType
+ }
if unit.DocType != "text" || pattern == nil || !pattern.MatchString(unit.Text) {
unit.TKNums = intPtr(tokenizeStr(unit.Text))
result = append(result, unit)
@@ -638,11 +642,37 @@ func splitMarkdownUnits(units []schema.ChunkDoc, pattern *regexp.Regexp) []schem
return result
}
+// stripTextLinesFromMediaContextAbove removes folded unit text from collected
+// media context so materializeMediaContext does not emit it twice.
+func stripTextLinesFromMediaContextAbove(contextAbove, folded string) string {
+ folded = strings.TrimSpace(folded)
+ if folded == "" || contextAbove == "" {
+ return contextAbove
+ }
+ parts := strings.Split(contextAbove, "\n")
+ filtered := make([]string, 0, len(parts))
+ for _, part := range parts {
+ if strings.TrimSpace(part) == folded {
+ continue
+ }
+ filtered = append(filtered, part)
+ }
+ return strings.Join(filtered, "\n")
+}
+
+// markdownImageUnitHasMediaContext reports whether attachGeneralMediaContext
+// populated context on a markdown image unit. Such units stay out of the text
+// merge so ContextAbove and ContextBelow survive until materialization.
+func markdownImageUnitHasMediaContext(unit schema.ChunkDoc) bool {
+ return unit.ContextAbove != "" || unit.ContextBelow != ""
+}
+
// mergeMarkdownUnits mirrors the Markdown branch of Python naive.chunk:
// ordinary units use a projected token cap, while a short heading is always
// kept with the following unit. Markdown images are block attachments rather
// than standalone media chunks, so image-bearing units participate in the
-// text merge and retain their image payload.
+// text merge and retain their image payload. Images with configured media
+// context are emitted as standalone chunks instead.
func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64, joinSep string) []schema.ChunkDoc {
overlapPct = max(0, min(100, overlapPct))
merged := make([]schema.ChunkDoc, 0, len(units))
@@ -672,6 +702,7 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,
table.Image = mergedImage
table.DocType = "table"
table.CKType = "table"
+ table.ContextAbove = stripTextLinesFromMediaContextAbove(table.ContextAbove, heading.Text)
merged[current] = table
current = -1
currentTokens = 0
@@ -683,6 +714,21 @@ func mergeMarkdownUnits(units []schema.ChunkDoc, target int, overlapPct float64,
continue
}
+ if itemDocType(unit) == "image" && markdownImageUnitHasMediaContext(unit) {
+ if current >= 0 {
+ current = -1
+ currentTokens = 0
+ }
+ standalone := cloneChunkDoc(unit)
+ standalone.DocType = "image"
+ if standalone.CKType == "" {
+ standalone.CKType = "image"
+ }
+ standalone.TKNums = intPtr(generalUnitTokens(standalone))
+ merged = append(merged, standalone)
+ continue
+ }
+
unit = cloneChunkDoc(unit)
unit.DocType = "text"
if unit.CKType == "" {
@@ -937,6 +983,7 @@ func (c *GeneralChunkerComponent) chunkGeneral(ctx context.Context, upstream sch
primaryPattern := compileDelimPattern(c.param.Delimiters)
childrenPattern := compileChildrenPattern(c.param.ChildrenDelimiters)
units = splitGeneralUnits(units, primaryPattern)
+ attachGeneralMediaContext(units, c.param.TableContextSize, c.param.ImageContextSize)
if !hasCustomDelim(c.param.Delimiters) {
// Python naive_merge prefixes each delimiter atom with a newline before
// counting it. The prefix is a budgeting detail, not emitted content;
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index 53d8e21d1d..678aca5260 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -201,6 +201,73 @@ func TestMergeMarkdownUnitsOverlapKeepsCurrentChunkMetadata(t *testing.T) {
}
}
+func TestMergeMarkdownUnitsStripsFoldedHeadingFromTableContextAbove(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
+ {
+ Text: "<table></table>",
+ DocType: "table",
+ CKType: "table",
+ ContextAbove: "## Section",
+ TKNums: intPtr(1),
+ },
+ }
+ got := mergeMarkdownUnits(units, 10, 0, "\n")
+ if len(got) != 1 {
+ t.Fatalf("chunks = %#v, want one table chunk", got)
+ }
+ if got[0].ContextAbove != "" {
+ t.Fatalf("ContextAbove = %q, want empty after heading fold", got[0].ContextAbove)
+ }
+ if got[0].Text != "## Section\n<table></table>" {
+ t.Fatalf("text = %q, want heading folded into body", got[0].Text)
+ }
+}
+
+func TestMergeMarkdownUnitsStripsOnlyFoldedHeadingFromTableContextAbove(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "intro", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {Text: "## Section", DocType: "text", CKType: "heading", TKNums: intPtr(1)},
+ {
+ Text: "<table></table>",
+ DocType: "table",
+ CKType: "table",
+ ContextAbove: "intro\n## Section",
+ TKNums: intPtr(1),
+ },
+ }
+ got := mergeMarkdownUnits(units, 10, 0, "\n")
+ if len(got) != 2 {
+ t.Fatalf("chunks = %#v, want intro text and table", got)
+ }
+ if got[1].ContextAbove != "intro" {
+ t.Fatalf("ContextAbove = %q, want intro only", got[1].ContextAbove)
+ }
+}
+
+func TestMergeMarkdownUnitsKeepsImageWithMediaContextStandalone(t *testing.T) {
+ units := []schema.ChunkDoc{
+ {Text: "before", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ {
+ Text: "figure",
+ DocType: "image",
+ CKType: "image",
+ ContextAbove: "before",
+ ContextBelow: "after",
+ Image: "data:image/png;base64,AAAA",
+ TKNums: intPtr(1),
+ },
+ {Text: "after", DocType: "text", CKType: "text", TKNums: intPtr(1)},
+ }
+ got := mergeMarkdownUnits(units, 10, 0, "\n")
+ if len(got) != 3 {
+ t.Fatalf("chunks = %d, want text, image, text", len(got))
+ }
+ if got[1].CKType != "image" || got[1].ContextAbove != "before" || got[1].ContextBelow != "after" {
+ t.Fatalf("image chunk = %#v, want preserved media context", got[1])
+ }
+}
+
func TestMergeMarkdownUnitsStartsNewChunkForIncomingHeading(t *testing.T) {
units := []schema.ChunkDoc{
{Text: "Background details", DocType: "text", CKType: "text", TKNums: intPtr(1)},
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index 455fa6f1cf..c384ab8d59 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -580,6 +580,185 @@ func assertMaterializedMediaContext(t *testing.T, chunk map[string]any, wantText
}
}
+func TestGeneralChunkerMarkdownAttachesMediaContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var table map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "table" {
+ table = ck
+ break
+ }
+ }
+ if table == nil {
+ t.Fatalf("table chunk missing: %#v", chunks)
+ }
+ assertMaterializedMediaContext(t, table, "before<table><tr><td>A</td></tr></table>after")
+}
+
+func TestGeneralChunkerMarkdownTableDoesNotDuplicateShortHeadingInContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var table map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "table" {
+ table = ck
+ break
+ }
+ }
+ if table == nil {
+ t.Fatalf("table chunk missing: %#v", chunks)
+ }
+ want := "## Section\n<table><tr><td>A</td></tr></table>"
+ assertMaterializedMediaContext(t, table, want)
+ if strings.Count(table["text"].(string), "## Section") != 1 {
+ t.Fatalf("heading repeated in table chunk: %q", table["text"])
+ }
+}
+
+func TestGeneralChunkerMarkdownTableKeepsEarlierContextWhenHeadingFolds(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 3,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "intro", "doc_type_kwd": "text"},
+ {"text": "## Section", "doc_type_kwd": "text", "ck_type": "heading"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var table map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "table" {
+ table = ck
+ break
+ }
+ }
+ if table == nil {
+ t.Fatalf("table chunk missing: %#v", chunks)
+ }
+ want := "intro## Section\n<table><tr><td>A</td></tr></table>"
+ assertMaterializedMediaContext(t, table, want)
+}
+
+func TestGeneralChunkerMarkdownImageKeepsMediaContextWhenConfigured(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "image_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "document.md",
+ "file_type": "md",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "figure", "doc_type_kwd": "image", "image": "data:image/png;base64,AAAA"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var image map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "image" {
+ image = ck
+ break
+ }
+ }
+ if image == nil {
+ t.Fatalf("image chunk missing: %#v", chunks)
+ }
+ assertMaterializedMediaContext(t, image, "beforefigureafter")
+}
+
+func TestGeneralChunkerHTMLAttachesMediaContext(t *testing.T) {
+ component, err := NewGeneralChunker(map[string]any{
+ "chunk_token_size": 10,
+ "table_context_size": 2,
+ })
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ out, err := component.Invoke(t.Context(), nil, map[string]any{
+ "name": "page.html",
+ "file_type": "html",
+ "output_format": "json",
+ "json": []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "<table><tr><td>A</td></tr></table>", "doc_type_kwd": "table"},
+ {"text": "after", "doc_type_kwd": "text"},
+ },
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := outputChunks(t, out)
+ var table map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "table" {
+ table = ck
+ break
+ }
+ }
+ if table == nil {
+ t.Fatalf("table chunk missing: %#v", chunks)
+ }
+ assertMaterializedMediaContext(t, table, "before<table><tr><td>A</td></tr></table>after")
+}
+
func TestGeneralChunkerDOCXAttachesMediaContextBeforeTextMerge(t *testing.T) {
component, err := NewGeneralChunker(map[string]any{
"chunk_token_size": 10,