RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): label PDF table items "table" only when they carry <table> markup (#20092)
Yash Raj Pandey516b9cf0fc9f
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex dd191f2431..dc9c324466 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -8,6 +8,7 @@ import ("mime/multipart""net/http""ragflow/internal/common"+ "ragflow/internal/deepdoc/parser/pdf/table""strings")@@ -137,16 +138,18 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {if html == "" {html = strings.TrimSpace(stringValue(el["html_content"]))}+ if html == "" {+ if rows := openDataLoaderCellRows(el["cells"]); rowsHaveText(rows) {+ html = table.SimpleRowsToHTML(rows)+ }+ }if html != "" {text = html}- if text == "" {- text = openDataLoaderCellsText(el["cells"])- }if text == "" {return nil}- return map[string]any{"text": text, "doc_type_kwd": "table", "layout": "table"}+ return map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}case "image", "picture", "figure":if text == "" {text = "[Image]"@@ -169,10 +172,10 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {}}-func openDataLoaderCellsText(raw any) string {+func openDataLoaderCellRows(raw any) [][]string {cells, ok := raw.([]any)if !ok {- return ""+ return nil}rows := make(map[int][]string)maxRow := -1@@ -191,15 +194,15 @@ func openDataLoaderCellsText(raw any) string {}}if len(rows) == 0 || maxRow < 0 {- return ""+ return nil}- parts := make([]string, 0, len(rows))+ ordered := make([][]string, 0, len(rows))for i := 0; i <= maxRow; i++ {if cols, ok := rows[i]; ok {- parts = append(parts, strings.Join(cols, " | "))+ ordered = append(ordered, cols)}}- return strings.Join(parts, "\n")+ return ordered}func openDataLoaderPageCount(root any) int {diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex 9d39683232..ebf5f55707 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -134,8 +134,8 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {if len(items) != 1 {t.Fatalf("items len = %d, want 1", len(items))}- if got, _ := json.Marshal(items[0]); !strings.Contains(string(got), "a | b") {- t.Fatalf("item = %s, want row text", string(got))+ if got, want := items[0]["text"], "<table><tr><th >a</th><th >b</th></tr></table>"; got != want {+ t.Fatalf("text = %q, want %q", got, want)}}diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex dfcb6c6c0a..acde5ace67 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -107,3 +107,26 @@ func collectPDFPageNumbers(raw any) map[int]struct{} {walk(raw)return pages}++// pdfTableDocType labels a table region "table" only when its text is <table>+// markup, because consumers of that label read the text as HTML rows. Free+// text recognized inside a table region is labelled "text".+func pdfTableDocType(text string) string {+ if isTableHTML(text) {+ return "table"+ }+ return "text"+}++// rowsHaveText reports whether any cell holds non-whitespace text. Blank rows+// must not replace the content of a table region with an empty <table>.+func rowsHaveText(rows [][]string) bool {+ for _, row := range rows {+ for _, cell := range row {+ if strings.TrimSpace(cell) != "" {+ return true+ }+ }+ }+ return false+}diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex d9b0c88931..75dc22782b 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -196,7 +196,7 @@ func soMarkBlockToItem(block map[string]any, keepHeaderFooter bool) map[string]aif content == "" {return nil}- return map[string]any{"text": content, "doc_type_kwd": "table", "layout": "table"}+ return map[string]any{"text": content, "doc_type_kwd": pdfTableDocType(content), "layout": "table"}case "equation":if content == "" {return nildiff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.gonew file mode 100644index 0000000000..adf662418b--- /dev/null+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -0,0 +1,84 @@+package parser++import (+ "strings"+ "testing"+)++// Consumers of doc_type_kwd "table" parse the item text as <table> markup+// (the QA chunker reads its <tr> rows), so a table item that carries other+// text loses its rows downstream.+func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {+ cells := []any{+ map[string]any{"row": 0, "content": "What is RAGFlow?"},+ map[string]any{"row": 0, "content": "A RAG engine."},+ map[string]any{"row": 1, "content": "Is it open source?"},+ map[string]any{"row": 1, "content": "Yes."},+ }+ rows := map[string]any{"rows": []any{+ []any{"What is RAGFlow?", "A RAG engine."},+ []any{"Is it open source?", "Yes."},+ }}+ const tableHTML = "<table><tr><td>What is RAGFlow?</td><td>A RAG engine.</td></tr></table>"+ const pipeText = "What is RAGFlow? | A RAG engine."+ blankCells := []any{+ map[string]any{"row": 0, "content": ""},+ map[string]any{"row": 0, "content": " "},+ }+ blankRows := map[string]any{"rows": []any{[]any{"", " "}}}+ const markdownTable = "| What is RAGFlow? | A RAG engine. |\n| --- | --- |\n| Is it open source? | Yes. |"++ cases := []struct {+ name string+ produce func() []map[string]any+ wantDocType string+ }{+ {"opendataloader html", func() []map[string]any {+ return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})+ }, "table"},+ {"opendataloader content only", func() []map[string]any {+ return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText})+ }, "text"},+ {"opendataloader cells only", func() []map[string]any {+ return openDataLoaderItems(map[string]any{"type": "table", "cells": cells})+ }, "table"},+ {"opendataloader blank cells keep content", func() []map[string]any {+ return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText, "cells": blankCells})+ }, "text"},+ {"tcadp html content", func() []map[string]any {+ return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})+ }, "table"},+ {"tcadp content only", func() []map[string]any {+ return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText})+ }, "text"},+ {"tcadp table_data rows only", func() []map[string]any {+ return tcadpAnyToItems(map[string]any{"type": "table", "table_data": rows})+ }, "table"},+ {"tcadp blank rows keep content", func() []map[string]any {+ return tcadpAnyToItems(map[string]any{"type": "table", "content": pipeText, "table_data": blankRows})+ }, "text"},+ {"somark html", func() []map[string]any {+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": tableHTML}, false)}+ }, "table"},+ {"somark markdown", func() []map[string]any {+ return []map[string]any{soMarkBlockToItem(map[string]any{"type": "table", "content": markdownTable}, false)}+ }, "text"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ items := tc.produce()+ if len(items) != 1 || items[0] == nil {+ t.Fatalf("items = %v, want one item", items)+ }+ item := items[0]+ docType, _ := item["doc_type_kwd"].(string)+ text, _ := item["text"].(string)+ if docType == "table" && !strings.HasPrefix(strings.TrimSpace(text), "<table") {+ t.Fatalf("doc_type_kwd table carries non-<table> text %q", text)+ }+ if docType != tc.wantDocType {+ t.Fatalf("doc_type_kwd = %q, want %q (text %q)", docType, tc.wantDocType, text)+ }+ })+ }+}diff --git a/internal/parser/parser/pdf_parser_tcadp.go b/internal/parser/parser/pdf_parser_tcadp.goindex 5cca9141a8..4aa517bac9 100644--- a/internal/parser/parser/pdf_parser_tcadp.go+++ b/internal/parser/parser/pdf_parser_tcadp.go@@ -7,6 +7,7 @@ import ("encoding/json""fmt""io"+ "ragflow/internal/deepdoc/parser/pdf/table""strings")@@ -93,13 +94,13 @@ func tcadpAnyToItems(raw any) []map[string]any {}switch contentType {case "table":- if text == "" {- text = tcadpTableRowsText(v["table_data"])+ if rows := tcadpTableRows(v["table_data"]); rowsHaveText(rows) {+ text = table.SimpleRowsToHTML(rows)}if text == "" {return nil}- return emit(text, "table", "table")+ return emit(text, pdfTableDocType(text), "table")case "image":caption := strings.TrimSpace(stringValue(v["caption"]))if caption == "" {@@ -135,17 +136,17 @@ func extractTCADPPage(v map[string]any) int {return 0}-func tcadpTableRowsText(raw any) string {- table, ok := raw.(map[string]any)+func tcadpTableRows(raw any) [][]string {+ data, ok := raw.(map[string]any)if !ok {- return ""+ return nil}- rows, ok := table["rows"].([]any)+ rawRows, ok := data["rows"].([]any)if !ok {- return ""+ return nil}- lines := make([]string, 0, len(rows))- for _, rowRaw := range rows {+ rows := make([][]string, 0, len(rawRows))+ for _, rowRaw := range rawRows {row, ok := rowRaw.([]any)if !ok {continue@@ -154,9 +155,9 @@ func tcadpTableRowsText(raw any) string {for _, col := range row {cols = append(cols, stringValue(col))}- lines = append(lines, strings.Join(cols, " | "))+ rows = append(rows, cols)}- return strings.Join(lines, "\n")+ return rows}func bearer(apiKey string) string {