Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): stop dropping QA pairs from multiline table cells (#19536)

jay77721868451c6391a
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index a9b70eaae2..1d49834b6f 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -32,12 +32,12 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
- "html"
"regexp"
"strings"
"github.com/gomarkdown/markdown"
"github.com/gomarkdown/markdown/parser"
+ "golang.org/x/net/html"
"gorm.io/gorm"
"ragflow/internal/agent/runtime"
@@ -207,28 +207,127 @@ func isCSV(name string) bool {
// HTML / spreadsheet QA extraction
// ---------------------------------------------------------------------------
-var htmlTR = regexp.MustCompile(`(?i)<tr[^>]*>(.*?)</tr>`)
-var htmlTD = regexp.MustCompile(`(?i)<t[dh][^>]*>(.*?)</t[dh]>`)
-var htmlTag = regexp.MustCompile(`<[^>]+>`)
+// tableRows walks the parsed HTML and returns the <td>/<th> text of every
+// <tr>, in document order.
+//
+// The markup is parsed into a tree rather than matched with a regex because
+// this input is not guaranteed to be well formed: seven parsers render table
+// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates
+// from user-supplied documents. A tree also settles the cases a tag-level
+// scan gets wrong: a nested <table> no longer terminates its enclosing row
+// early — that row keeps its own cells, with the nested table's text folded
+// into the cell holding it — and a row or cell missing its closing tag is
+// recovered rather than dropped.
+func tableRows(htmlStr string) [][]string {
+ // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion
+ // mode, so a bare row fragment would yield nothing. Give the parser the
+ // table context it needs instead of dropping the rows silently.
+ lower := strings.ToLower(htmlStr)
+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {
+ htmlStr = "<table>" + htmlStr + "</table>"
+ }
+ doc, err := html.Parse(strings.NewReader(htmlStr))
+ if err != nil {
+ return nil
+ }
+ var rows [][]string
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ // An inert subtree is parsed but never rendered. <template> puts its
+ // content straight into the ordinary child list (the parser has no
+ // separate template-contents field), so without this its rows would
+ // be read as rows of the enclosing table.
+ if n.Type == html.ElementNode && isInertElement(n.Data) {
+ return
+ }
+ if isHTMLElement(n, "tr") {
+ var cells []string
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {
+ cells = append(cells, cellText(c))
+ }
+ }
+ // Return without descending: the cells above already collected
+ // the nested table's text, so its rows must not be reported a
+ // second time as rows of the enclosing table.
+ rows = append(rows, cells)
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(doc)
+ return rows
+}
+
+// cellText returns the visible text of a table cell. The parser hands text
+// nodes over already unescaped, nested markup contributes its text without
+// its tags (a nested table's cells are concatenated, not separated), and a
+// <br> becomes a newline instead of silently gluing the two halves of the
+// cell together.
+func cellText(cell *html.Node) string {
+ var sb strings.Builder
+ var walk func(*html.Node)
+ walk = func(n *html.Node) {
+ switch {
+ case n.Type == html.TextNode:
+ sb.WriteString(n.Data)
+ case isHTMLElement(n, "br"):
+ sb.WriteByte('\n')
+ case n.Type == html.ElementNode && isInertElement(n.Data):
+ // Stop here rather than descending: the content is parsed but
+ // never rendered, so it is not text a reader of the cell sees.
+ return
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ walk(c)
+ }
+ }
+ walk(cell)
+ return strings.TrimSpace(sb.String())
+}
+
+// isHTMLElement reports whether n is an element with the given tag name in
+// the HTML namespace. Foreign content reuses HTML tag names for unrelated
+// elements — an <svg><tr> is not a table row — so matching on the tag name
+// alone would read markup that carries no table semantics.
+func isHTMLElement(n *html.Node, tag string) bool {
+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag
+}
+
+// isInertElement reports whether an element's content is inert — parsed, but
+// never rendered as visible text. An inline <script> or <style> inside a
+// table cell of a user-supplied document would otherwise be embedded into the
+// Q&A pair as if it were part of the sentence, and a <template>'s placeholder
+// rows would be read as real ones.
+func isInertElement(tag string) bool {
+ switch tag {
+ case "script", "style", "noscript", "template":
+ return true
+ }
+ return false
+}
+// extractQATable turns table markup into Q&A pairs: the first two non-empty
+// cells of a row become the question and the answer. strictPairs is the CSV
+// contract (Python qa.py:365) and requires a row to have exactly two cells
+// instead of taking the first two.
func extractQATable(htmlStr string, strictPairs bool) []qaPair {
if htmlStr == "" {
return nil
}
- rows := htmlTR.FindAllStringSubmatch(htmlStr, -1)
+ rows := tableRows(htmlStr)
pairs := make([]qaPair, 0, len(rows))
- for _, row := range rows {
- cells := htmlTD.FindAllStringSubmatch(row[1], -1)
+ for _, cells := range rows {
// Python qa.py:365 requires exactly two fields for CSV pairs.
if strictPairs && len(cells) != 2 {
continue
}
var texts []string
for _, cell := range cells {
- t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))
- t = strings.TrimSpace(t)
- if t != "" {
- texts = append(texts, t)
+ if cell != "" {
+ texts = append(texts, cell)
}
}
if len(texts) >= 2 {
diff --git a/internal/ingestion/component/chunker/qa_table_test.go b/internal/ingestion/component/chunker/qa_table_test.go
new file mode 100644
index 0000000000..4a551b8bbf
--- /dev/null
+++ b/internal/ingestion/component/chunker/qa_table_test.go
@@ -0,0 +1,213 @@
+package chunker
+
+import (
+ "fmt"
+ "strings"
+ "testing"
+)
+
+// Extraction must follow the table's structure rather than a tag pattern.
+// The markup is rendered by several parsers and some of it comes from
+// user-supplied documents, so it is not guaranteed to be well formed —
+// a tag-level scan mishandles most cases below.
+func TestExtractQATableFollowsStructure(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ html string
+ strict bool
+ want [][2]string
+ }{
+ {
+ name: "newline inside a cell",
+ html: "<table><tr><th>问题\n跨行</th><th>答案</th></tr></table>",
+ want: [][2]string{{"问题\n跨行", "答案"}},
+ },
+ {
+ name: "colspan",
+ html: `<table><tr><td colspan="2">q</td><td>a</td></tr></table>`,
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "missing </td>",
+ html: "<table><tr><td>q<td>a</tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "missing </tr>",
+ html: "<table><tr><td>q</td><td>a</td><tr><td>q2</td><td>a2</td></tr></table>",
+ want: [][2]string{{"q", "a"}, {"q2", "a2"}},
+ },
+ {
+ name: "attribute containing '>'",
+ html: `<table><tr><td data-x="a>b">q</td><td>a</td></tr></table>`,
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "<br> inside a cell",
+ html: "<table><tr><td>q<br>L2</td><td>a</td></tr></table>",
+ want: [][2]string{{"q\nL2", "a"}},
+ },
+ {
+ // Content that is parsed but never rendered must not join the
+ // sentence: an inline <script>/<style> is not text a reader sees.
+ name: "inert elements contribute no text",
+ html: "<table><tr><td>q<script>var x=1;</script></td>" +
+ "<td>a<style>.c{color:red}</style></td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "noscript and template contribute no text",
+ html: "<table><tr><td>q<noscript>enable js</noscript></td>" +
+ "<td>a<template>tpl</template></td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ // The counterpart of the two cases above: elements whose content
+ // *is* rendered keep their text, so the skip list stays narrow.
+ name: "rendered descendants keep their text",
+ html: "<table><tr><td>q<textarea>notes</textarea></td>" +
+ "<td>a<span>bold</span></td></tr></table>",
+ want: [][2]string{{"qnotes", "abold"}},
+ },
+ {
+ // A template's content goes into the ordinary child list, so its
+ // placeholder rows would otherwise be read as rows of the table.
+ name: "template rows are not table rows",
+ html: "<table><tr><td>q</td><td>a</td></tr>" +
+ "<template><tr><td>TQ</td><td>TA</td></tr></template></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ // Foreign content reuses HTML tag names: an <svg><tr> carries no
+ // table semantics.
+ name: "svg rows are not table rows",
+ html: "<table><tr><td>q</td><td>a</td></tr>" +
+ "<svg><tr><td>VQ</td><td>VA</td></tr></svg></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ // A template's rows are markup, not rows: the script's content is
+ // raw text to the parser, so only the real row becomes a pair.
+ // Found on a real page, where the unrendered template would
+ // otherwise contribute placeholder pairs.
+ name: "markup inside a script is not a row",
+ html: "<table><script type=\"text/tpl\"><tr><td>tpl</td><td>tpl2</td></tr></script>" +
+ "<tr><td>q</td><td>a</td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "commented-out markup is not a pair",
+ html: "<table><!-- <tr><td>old</td><td>value</td></tr> --><tr><td>q</td><td>a</td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "commented-out cell is not a cell",
+ html: "<table><tr><!-- <td>dead</td> --><td>q</td><td>a</td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ // The nested table's text is folded into the cell holding it, and
+ // its own rows are not reported as rows of the enclosing table.
+ name: "nested table",
+ html: "<table><tr><td><table><tr><td>in</td><td>x</td></tr></table></td><td>a</td></tr></table>",
+ want: [][2]string{{"inx", "a"}},
+ },
+ {
+ // A bare row fragment: the HTML5 "in body" mode would discard the
+ // <tr>, so it has to be parsed in a table context.
+ name: "row without a table wrapper",
+ html: "<tr><td>q</td><td>a</td></tr>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "rows without a table wrapper",
+ html: "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>",
+ want: [][2]string{{"q", "a"}, {"q2", "a2"}},
+ },
+ {
+ name: "rows wrapped in tbody only",
+ html: "<tbody><tr><td>q</td><td>a</td></tr></tbody>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ // Deliberately narrow: cells without a row are not a pair, matching
+ // the previous behaviour instead of inventing a row for them.
+ name: "cells without a row yield nothing",
+ html: "<td>q</td><td>a</td>",
+ want: nil,
+ },
+ {
+ name: "empty cell is skipped when picking the pair",
+ html: "<table><tr><td>q</td><td></td><td>a</td></tr></table>",
+ want: [][2]string{{"q", "a"}},
+ },
+ {
+ name: "unbalanced row yields nothing",
+ html: "<table><tr><td>q only</td></tr></table>",
+ want: nil,
+ },
+ {
+ // The CSV contract (Python qa.py:365) is exactly two fields.
+ name: "three columns are rejected in strict mode",
+ html: "<table><tr><td>q</td><td>a</td><td>extra</td></tr></table>",
+ strict: true,
+ want: nil,
+ },
+ {
+ name: "three columns keep the first two in lax mode",
+ html: "<table><tr><td>q</td><td>a</td><td>extra</td></tr></table>",
+ strict: false,
+ want: [][2]string{{"q", "a"}},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ pairs := extractQATable(tc.html, tc.strict)
+ got := make([][2]string, 0, len(pairs))
+ for _, p := range pairs {
+ got = append(got, [2]string{p.Question, p.Answer})
+ }
+ if len(got) != len(tc.want) {
+ t.Fatalf("pairs = %v, want %v", got, tc.want)
+ }
+ for i := range got {
+ if got[i] != tc.want[i] {
+ t.Errorf("pair %d = %q, want %q", i, got[i], tc.want[i])
+ }
+ }
+ })
+ }
+}
+
+// BenchmarkExtractQATable makes the cost of the extraction reproducible:
+//
+// go test -run '^$' -bench BenchmarkExtractQATable -benchmem ./internal/ingestion/component/chunker/
+//
+// The cells carry the two shapes that the previous tag-level scan handled
+// worst — an embedded newline and an entity — so the numbers reflect the
+// markup table items actually carry.
+func BenchmarkExtractQATable(b *testing.B) {
+ for _, rows := range []int{10, 256, 1000} {
+ markup := benchmarkTable(rows)
+ b.Run(fmt.Sprintf("%d_rows", rows), func(b *testing.B) {
+ b.SetBytes(int64(len(markup)))
+ for i := 0; i < b.N; i++ {
+ extractQATable(markup, false)
+ }
+ })
+ }
+}
+
+// benchmarkTable renders a two-column table of the given size, one row per
+// Q&A pair, with newlines and entities inside the cells.
+func benchmarkTable(rows int) string {
+ var sb strings.Builder
+ sb.WriteString("<table><caption>Sheet1</caption>" +
+ "<tr><th>question</th><th>answer</th></tr>")
+ for i := 0; i < rows; i++ {
+ fmt.Fprintf(&sb,
+ "<tr><td>question %d, which spans\na second line</td>"+
+ "<td>answer %d, with an entity &amp; and more text</td></tr>", i, i)
+ }
+ sb.WriteString("</table>")
+ return sb.String()
+}
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 342170853c..4700710f2c 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -3,35 +3,44 @@ package chunker
import (
"bytes"
"context"
+ "strings"
"testing"
"github.com/xuri/excelize/v2"
+
"ragflow/internal/parser/parser"
)
-func TestXLSXQARegression(t *testing.T) {
+// xlsxWorkbook renders rows into an in-memory workbook so the tests run
+// against the real spreadsheet parser instead of hand-written markup.
+func xlsxWorkbook(t *testing.T, rows [][]string) []byte {
+ t.Helper()
f := excelize.NewFile()
sh := f.GetSheetName(0)
- for i, r := range [][]string{
- {"question", "answer"},
- {"What is RAGFlow?", "A RAG engine."},
- {"Where are the docs?", "On the website."},
- } {
+ for i, r := range rows {
for j, c := range r {
cell, _ := excelize.CoordinatesToCellName(j+1, i+1)
- _ = f.SetCellValue(sh, cell, c)
+ if err := f.SetCellValue(sh, cell, c); err != nil {
+ t.Fatal(err)
+ }
}
}
var buf bytes.Buffer
if err := f.Write(&buf); err != nil {
t.Fatal(err)
}
+ return buf.Bytes()
+}
+// qaChunksFromXLSX drives a workbook through the real XLSX parser and then
+// the QA chunker, returning the chunks the pipeline would emit.
+func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
+ t.Helper()
p, err := parser.NewXLSXParser("")
if err != nil {
t.Fatal(err)
}
- res := p.ParseWithResult(context.Background(), "qa.xlsx", buf.Bytes())
+ res := p.ParseWithResult(context.Background(), "qa.xlsx", data)
if res.Err != nil {
t.Fatal(res.Err)
}
@@ -53,8 +62,46 @@ func TestXLSXQARegression(t *testing.T) {
t.Fatal(err)
}
chunks, _ := out["chunks"].([]map[string]any)
- t.Logf("format=%q QA chunks=%d", res.OutputFormat, len(chunks))
+ return chunks
+}
+
+// TestXLSXQARegression is the end-to-end smoke test: every row of the sheet
+// becomes one chunk. The chunker has no header concept, so the first row is
+// a Q&A pair like any other.
+func TestXLSXQARegression(t *testing.T) {
+ chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{
+ {"question", "answer"},
+ {"What is RAGFlow?", "A RAG engine."},
+ {"Where are the docs?", "On the website."},
+ }))
+ t.Logf("QA chunks=%d", len(chunks))
+ if len(chunks) != 3 {
+ t.Fatalf("expected 3 chunks, got %d", len(chunks))
+ }
+}
+
+// A spreadsheet cell keeps the newline its author typed (Alt+Enter) and the
+// parser renders it into the <tr>/<td> HTML verbatim, so a QA pair whose
+// question or answer spans lines must survive extraction intact. These rows
+// used to disappear from the chunk list without a trace.
+func TestXLSXQAMultilineCells(t *testing.T) {
+ const multilineQ = "请问全国碳排放权交易市场纳入配额管理的重点排放单\n位名录,是否会公布?"
+ const multilineA = "需要公布。根据《碳排放权交易管理办法(试行)》。"
+ const multilineAnswer = "跨行的答案\n第二行\n第三行"
+
+ chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{
+ {"question", "answer"},
+ {multilineQ, multilineA},
+ {"跨行的问句\n第二行", multilineAnswer},
+ }))
+ t.Logf("QA chunks=%d", len(chunks))
if len(chunks) != 3 {
t.Fatalf("expected 3 chunks, got %d", len(chunks))
}
+ texts := strings.Join(chunkTexts(chunks), "\n")
+ for _, want := range []string{multilineQ, multilineA, multilineAnswer} {
+ if !strings.Contains(texts, want) {
+ t.Errorf("chunk text lost the newline-bearing cell %q", want)
+ }
+ }
}