Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): label PDF table items "table" only when they carry <table> markup (#20092)

Yash Raj Pandey516b9cf0fc9f
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.go
index dd191f2431..dc9c324466 100644
--- a/internal/parser/parser/pdf_parser_opendataloader.go
+++ b/internal/parser/parser/pdf_parser_opendataloader.go
@@ -8,6 +8,7 @@ import (
"mime/multipart"
"net/http"
"ragflow/internal/common"
+ "ragflow/internal/deepdoc/parser/pdf/table"
"strings"
)
@@ -137,16 +138,18 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {
if html == "" {
html = strings.TrimSpace(stringValue(el["html_content"]))
}
+ if html == "" {
+ if rows := openDataLoaderCellRows(el["cells"]); rowsHaveText(rows) {
+ html = table.SimpleRowsToHTML(rows)
+ }
+ }
if html != "" {
text = html
}
- if text == "" {
- text = openDataLoaderCellsText(el["cells"])
- }
if text == "" {
return nil
}
- return map[string]any{"text": text, "doc_type_kwd": "table", "layout": "table"}
+ return map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}
case "image", "picture", "figure":
if text == "" {
text = "[Image]"
@@ -169,10 +172,10 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {
}
}
-func openDataLoaderCellsText(raw any) string {
+func openDataLoaderCellRows(raw any) [][]string {
cells, ok := raw.([]any)
if !ok {
- return ""
+ return nil
}
rows := make(map[int][]string)
maxRow := -1
@@ -191,15 +194,15 @@ func openDataLoaderCellsText(raw any) string {
}
}
if len(rows) == 0 || maxRow < 0 {
- return ""
+ return nil
}
- parts := make([]string, 0, len(rows))
+ ordered := make([][]string, 0, len(rows))
for i := 0; i <= maxRow; i++ {
if cols, ok := rows[i]; ok {
- parts = append(parts, strings.Join(cols, " | "))
+ ordered = append(ordered, cols)
}
}
- return strings.Join(parts, "\n")
+ return ordered
}
func openDataLoaderPageCount(root any) int {
diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.go
index 9d39683232..ebf5f55707 100644
--- a/internal/parser/parser/pdf_parser_opendataloader_test.go
+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go
@@ -134,8 +134,8 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {
if len(items) != 1 {
t.Fatalf("items len = %d, want 1", len(items))
}
- if got, _ := json.Marshal(items[0]); !strings.Contains(string(got), "a | b") {
- t.Fatalf("item = %s, want row text", string(got))
+ if got, want := items[0]["text"], "<table><tr><th >a</th><th >b</th></tr></table>"; got != want {
+ t.Fatalf("text = %q, want %q", got, want)
}
}
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index dfcb6c6c0a..acde5ace67 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -107,3 +107,26 @@ func collectPDFPageNumbers(raw any) map[int]struct{} {
walk(raw)
return pages
}
+
+// pdfTableDocType labels a table region "table" only when its text is <table>
+// markup, because consumers of that label read the text as HTML rows. Free
+// text recognized inside a table region is labelled "text".
+func pdfTableDocType(text string) string {
+ if isTableHTML(text) {
+ return "table"
+ }
+ return "text"
+}
+
+// rowsHaveText reports whether any cell holds non-whitespace text. Blank rows
+// must not replace the content of a table region with an empty <table>.
+func rowsHaveText(rows [][]string) bool {
+ for _, row := range rows {
+ for _, cell := range row {
+ if strings.TrimSpace(cell) != "" {
+ return true
+ }
+ }
+ }
+ return false
+}
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.go
index d9b0c88931..75dc22782b 100644
--- a/internal/parser/parser/pdf_parser_somark.go
+++ b/internal/parser/parser/pdf_parser_somark.go
@@ -196,7 +196,7 @@ func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]a
if content == "" {
return nil
}
- return map[string]any{"text": content, "doc_type_kwd": "table", "layout": "table"}
+ return map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}
case "equation":
if content == "" {
return nil
diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.go
new file mode 100644
index 0000000000..adf662418b
--- /dev/null
+++ b/internal/parser/parser/pdf_parser_table_contract_test.go
@@ -0,0 +1,84 @@
+package parser
+
+import (
+ "strings"
+ "testing"
+)
+
+// Consumers of doc_type_kwd "table" parse the item text as <table> markup
+// (the QA chunker reads its <tr> rows), so a table item that carries other
+// text loses its rows downstream.
+func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {
+ cells := []any{
+ map[string]any{"row": 0, "content": "What is RAGFlow?"},
+ map[string]any{"row": 0, "content": "A RAG engine."},
+ map[string]any{"row": 1, "content": "Is it open source?"},
+ map[string]any{"row": 1, "content": "Yes."},
+ }
+ rows := map[string]any{"rows": []any{
+ []any{"What is RAGFlow?", "A RAG engine."},
+ []any{"Is it open source?", "Yes."},
+ }}
+ const tableHTML = "<table><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr></table>"
+ const pipeText = "What is RAGFlow? | A RAG engine."
+ blankCells := []any{
+ map[string]any{"row": 0, "content": ""},
+ map[string]any{"row": 0, "content": " "},
+ }
+ blankRows := map[string]any{"rows": []any{[]any{"", " "}}}
+ const markdownTable = "| What is RAGFlow? | A RAG engine. |\n| --- | --- |\n| Is it open source? | Yes. |"
+
+ cases := []struct {
+ name string
+ produce func() []map[string]any
+ wantDocType string
+ }{
+ {"opendataloader html", func() []map[string]any {
+ return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})
+ }, "table"},
+ {"opendataloader content only", func() []map[string]any {
+ return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText})
+ }, "text"},
+ {"opendataloader cells only", func() []map[string]any {
+ return openDataLoaderItems(map[string]any{"type": "table", "cells": cells})
+ }, "table"},
+ {"opendataloader blank cells keep content", func() []map[string]any {
+ return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText, "cells": blankCells})
+ }, "text"},
+ {"tcadp html content", func() []map[string]any {
+ return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})
+ }, "table"},
+ {"tcadp content only", func() []map[string]any {
+ return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText})
+ }, "text"},
+ {"tcadp table_data rows only", func() []map[string]any {
+ return tcadpAnyToItems(map[string]any{"type": "table", "table_data": rows})
+ }, "table"},
+ {"tcadp blank rows keep content", func() []map[string]any {
+ return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText, "table_data": blankRows})
+ }, "text"},
+ {"somark html", func() []map[string]any {
+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}
+ }, "table"},
+ {"somark markdown", func() []map[string]any {
+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": markdownTable}, false)}
+ }, "text"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ items := tc.produce()
+ if len(items) != 1 || items[0] == nil {
+ t.Fatalf("items = %v, want one item", items)
+ }
+ item := items[0]
+ docType, _ := item["doc_type_kwd"].(string)
+ text, _ := item["text"].(string)
+ if docType == "table" && !strings.HasPrefix(strings.TrimSpace(text), "<table") {
+ t.Fatalf("doc_type_kwd table carries non-<table> text %q", text)
+ }
+ if docType != tc.wantDocType {
+ t.Fatalf("doc_type_kwd = %q, want %q (text %q)", docType, tc.wantDocType, text)
+ }
+ })
+ }
+}
diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.go
index 5cca9141a8..4aa517bac9 100644
--- a/internal/parser/parser/pdf_parser_tcadp.go
+++ b/internal/parser/parser/pdf_parser_tcadp.go
@@ -7,6 +7,7 @@ import (
"encoding/json"
"fmt"
"io"
+ "ragflow/internal/deepdoc/parser/pdf/table"
"strings"
)
@@ -93,13 +94,13 @@ func tcadpAnyToItems(raw any) []map[string]any {
}
switch contentType {
case "table":
- if text == "" {
- text = tcadpTableRowsText(v["table_data"])
+ if rows := tcadpTableRows(v["table_data"]); rowsHaveText(rows) {
+ text = table.SimpleRowsToHTML(rows)
}
if text == "" {
return nil
}
- return emit(text, "table", "table")
+ return emit(text, pdfTableDocType(text), "table")
case "image":
caption := strings.TrimSpace(stringValue(v["caption"]))
if caption == "" {
@@ -135,17 +136,17 @@ func extractTCADPPage(v map[string]any) int {
return 0
}
-func tcadpTableRowsText(raw any) string {
- table, ok := raw.(map[string]any)
+func tcadpTableRows(raw any) [][]string {
+ data, ok := raw.(map[string]any)
if !ok {
- return ""
+ return nil
}
- rows, ok := table["rows"].([]any)
+ rawRows, ok := data["rows"].([]any)
if !ok {
- return ""
+ return nil
}
- lines := make([]string, 0, len(rows))
- for _, rowRaw := range rows {
+ rows := make([][]string, 0, len(rawRows))
+ for _, rowRaw := range rawRows {
row, ok := rowRaw.([]any)
if !ok {
continue
@@ -154,9 +155,9 @@ func tcadpTableRowsText(raw any) string {
for _, col := range row {
cols = append(cols, stringValue(col))
}
- lines = append(lines, strings.Join(cols, " | "))
+ rows = append(rows, cols)
}
- return strings.Join(lines, "\n")
+ return rows
}
func bearer(apiKey string) string {