RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): stop dropping QA pairs from multiline table cells (#19536)
jay77721868451c6391a
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex a9b70eaae2..1d49834b6f 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -32,12 +32,12 @@ import ("encoding/csv""encoding/json""fmt"- "html""regexp""strings""github.com/gomarkdown/markdown""github.com/gomarkdown/markdown/parser"+ "golang.org/x/net/html""gorm.io/gorm""ragflow/internal/agent/runtime"@@ -207,28 +207,127 @@ func isCSV(name string) bool {// HTML / spreadsheet QA extraction// ----------------------------------------------------------------------------var htmlTR = regexp.MustCompile(`(?i)<tr[^>]*>(.*?)</tr>`)-var htmlTD = regexp.MustCompile(`(?i)<t[dh][^>]*>(.*?)</t[dh]>`)-var htmlTag = regexp.MustCompile(`<[^>]+>`)+// tableRows walks the parsed HTML and returns the <td>/<th> text of every+// <tr>, in document order.+//+// The markup is parsed into a tree rather than matched with a regex because+// this input is not guaranteed to be well formed: seven parsers render table+// items (xlsx, csv, docx, html, pdf, …) and some of that markup originates+// from user-supplied documents. A tree also settles the cases a tag-level+// scan gets wrong: a nested <table> no longer terminates its enclosing row+// early — that row keeps its own cells, with the nested table's text folded+// into the cell holding it — and a row or cell missing its closing tag is+// recovered rather than dropped.+func tableRows(htmlStr string) [][]string {+ // A <tr> outside a <table> is discarded by the HTML5 "in body" insertion+ // mode, so a bare row fragment would yield nothing. Give the parser the+ // table context it needs instead of dropping the rows silently.+ lower := strings.ToLower(htmlStr)+ if strings.Contains(lower, "<tr") && !strings.Contains(lower, "<table") {+ htmlStr = "<table>" + htmlStr + "</table>"+ }+ doc, err := html.Parse(strings.NewReader(htmlStr))+ if err != nil {+ return nil+ }+ var rows [][]string+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ // An inert subtree is parsed but never rendered. <template> puts its+ // content straight into the ordinary child list (the parser has no+ // separate template-contents field), so without this its rows would+ // be read as rows of the enclosing table.+ if n.Type == html.ElementNode && isInertElement(n.Data) {+ return+ }+ if isHTMLElement(n, "tr") {+ var cells []string+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ if isHTMLElement(c, "td") || isHTMLElement(c, "th") {+ cells = append(cells, cellText(c))+ }+ }+ // Return without descending: the cells above already collected+ // the nested table's text, so its rows must not be reported a+ // second time as rows of the enclosing table.+ rows = append(rows, cells)+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(doc)+ return rows+}++// cellText returns the visible text of a table cell. The parser hands text+// nodes over already unescaped, nested markup contributes its text without+// its tags (a nested table's cells are concatenated, not separated), and a+// <br> becomes a newline instead of silently gluing the two halves of the+// cell together.+func cellText(cell *html.Node) string {+ var sb strings.Builder+ var walk func(*html.Node)+ walk = func(n *html.Node) {+ switch {+ case n.Type == html.TextNode:+ sb.WriteString(n.Data)+ case isHTMLElement(n, "br"):+ sb.WriteByte('\n')+ case n.Type == html.ElementNode && isInertElement(n.Data):+ // Stop here rather than descending: the content is parsed but+ // never rendered, so it is not text a reader of the cell sees.+ return+ }+ for c := n.FirstChild; c != nil; c = c.NextSibling {+ walk(c)+ }+ }+ walk(cell)+ return strings.TrimSpace(sb.String())+}++// isHTMLElement reports whether n is an element with the given tag name in+// the HTML namespace. Foreign content reuses HTML tag names for unrelated+// elements — an <svg><tr> is not a table row — so matching on the tag name+// alone would read markup that carries no table semantics.+func isHTMLElement(n *html.Node, tag string) bool {+ return n.Type == html.ElementNode && n.Namespace == "" && n.Data == tag+}++// isInertElement reports whether an element's content is inert — parsed, but+// never rendered as visible text. An inline <script> or <style> inside a+// table cell of a user-supplied document would otherwise be embedded into the+// Q&A pair as if it were part of the sentence, and a <template>'s placeholder+// rows would be read as real ones.+func isInertElement(tag string) bool {+ switch tag {+ case "script", "style", "noscript", "template":+ return true+ }+ return false+}+// extractQATable turns table markup into Q&A pairs: the first two non-empty+// cells of a row become the question and the answer. strictPairs is the CSV+// contract (Python qa.py:365) and requires a row to have exactly two cells+// instead of taking the first two.func extractQATable(htmlStr string, strictPairs bool) []qaPair {if htmlStr == "" {return nil}- rows := htmlTR.FindAllStringSubmatch(htmlStr, -1)+ rows := tableRows(htmlStr)pairs := make([]qaPair, 0, len(rows))- for _, row := range rows {- cells := htmlTD.FindAllStringSubmatch(row[1], -1)+ for _, cells := range rows {// Python qa.py:365 requires exactly two fields for CSV pairs.if strictPairs && len(cells) != 2 {continue}var texts []stringfor _, cell := range cells {- t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))- t = strings.TrimSpace(t)- if t != "" {- texts = append(texts, t)+ if cell != "" {+ texts = append(texts, cell)}}if len(texts) >= 2 {diff --git a/internal/ingestion/component/chunker/qa_table_test.go b/internal/ingestion/component/chunker/qa_table_test.gonew file mode 100644index 0000000000..4a551b8bbf--- /dev/null+++ b/internal/ingestion/component/chunker/qa_table_test.go@@ -0,0 +1,213 @@+package chunker++import (+ "fmt"+ "strings"+ "testing"+)++// Extraction must follow the table's structure rather than a tag pattern.+// The markup is rendered by several parsers and some of it comes from+// user-supplied documents, so it is not guaranteed to be well formed —+// a tag-level scan mishandles most cases below.+func TestExtractQATableFollowsStructure(t *testing.T) {+ for _, tc := range []struct {+ name string+ html string+ strict bool+ want [][2]string+ }{+ {+ name: "newline inside a cell",+ html: "<table><tr><th>问题\n跨行</th><th>答案</th></tr></table>",+ want: [][2]string{{"问题\n跨行", "答案"}},+ },+ {+ name: "colspan",+ html: `<table><tr><td colspan="2">q</td><td>a</td></tr></table>`,+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "missing </td>",+ html: "<table><tr><td>q<td>a</tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "missing </tr>",+ html: "<table><tr><td>q</td><td>a</td><tr><td>q2</td><td>a2</td></tr></table>",+ want: [][2]string{{"q", "a"}, {"q2", "a2"}},+ },+ {+ name: "attribute containing '>'",+ html: `<table><tr><td data-x="a>b">q</td><td>a</td></tr></table>`,+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "<br> inside a cell",+ html: "<table><tr><td>q<br>L2</td><td>a</td></tr></table>",+ want: [][2]string{{"q\nL2", "a"}},+ },+ {+ // Content that is parsed but never rendered must not join the+ // sentence: an inline <script>/<style> is not text a reader sees.+ name: "inert elements contribute no text",+ html: "<table><tr><td>q<script>var x=1;</script></td>" ++ "<td>a<style>.c{color:red}</style></td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "noscript and template contribute no text",+ html: "<table><tr><td>q<noscript>enable js</noscript></td>" ++ "<td>a<template>tpl</template></td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ // The counterpart of the two cases above: elements whose content+ // *is* rendered keep their text, so the skip list stays narrow.+ name: "rendered descendants keep their text",+ html: "<table><tr><td>q<textarea>notes</textarea></td>" ++ "<td>a<span>bold</span></td></tr></table>",+ want: [][2]string{{"qnotes", "abold"}},+ },+ {+ // A template's content goes into the ordinary child list, so its+ // placeholder rows would otherwise be read as rows of the table.+ name: "template rows are not table rows",+ html: "<table><tr><td>q</td><td>a</td></tr>" ++ "<template><tr><td>TQ</td><td>TA</td></tr></template></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ // Foreign content reuses HTML tag names: an <svg><tr> carries no+ // table semantics.+ name: "svg rows are not table rows",+ html: "<table><tr><td>q</td><td>a</td></tr>" ++ "<svg><tr><td>VQ</td><td>VA</td></tr></svg></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ // A template's rows are markup, not rows: the script's content is+ // raw text to the parser, so only the real row becomes a pair.+ // Found on a real page, where the unrendered template would+ // otherwise contribute placeholder pairs.+ name: "markup inside a script is not a row",+ html: "<table><script type=\"text/tpl\"><tr><td>tpl</td><td>tpl2</td></tr></script>" ++ "<tr><td>q</td><td>a</td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "commented-out markup is not a pair",+ html: "<table><!-- <tr><td>old</td><td>value</td></tr> --><tr><td>q</td><td>a</td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "commented-out cell is not a cell",+ html: "<table><tr><!-- <td>dead</td> --><td>q</td><td>a</td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ // The nested table's text is folded into the cell holding it, and+ // its own rows are not reported as rows of the enclosing table.+ name: "nested table",+ html: "<table><tr><td><table><tr><td>in</td><td>x</td></tr></table></td><td>a</td></tr></table>",+ want: [][2]string{{"inx", "a"}},+ },+ {+ // A bare row fragment: the HTML5 "in body" mode would discard the+ // <tr>, so it has to be parsed in a table context.+ name: "row without a table wrapper",+ html: "<tr><td>q</td><td>a</td></tr>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "rows without a table wrapper",+ html: "<tr><td>q</td><td>a</td></tr><tr><td>q2</td><td>a2</td></tr>",+ want: [][2]string{{"q", "a"}, {"q2", "a2"}},+ },+ {+ name: "rows wrapped in tbody only",+ html: "<tbody><tr><td>q</td><td>a</td></tr></tbody>",+ want: [][2]string{{"q", "a"}},+ },+ {+ // Deliberately narrow: cells without a row are not a pair, matching+ // the previous behaviour instead of inventing a row for them.+ name: "cells without a row yield nothing",+ html: "<td>q</td><td>a</td>",+ want: nil,+ },+ {+ name: "empty cell is skipped when picking the pair",+ html: "<table><tr><td>q</td><td></td><td>a</td></tr></table>",+ want: [][2]string{{"q", "a"}},+ },+ {+ name: "unbalanced row yields nothing",+ html: "<table><tr><td>q only</td></tr></table>",+ want: nil,+ },+ {+ // The CSV contract (Python qa.py:365) is exactly two fields.+ name: "three columns are rejected in strict mode",+ html: "<table><tr><td>q</td><td>a</td><td>extra</td></tr></table>",+ strict: true,+ want: nil,+ },+ {+ name: "three columns keep the first two in lax mode",+ html: "<table><tr><td>q</td><td>a</td><td>extra</td></tr></table>",+ strict: false,+ want: [][2]string{{"q", "a"}},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ pairs := extractQATable(tc.html, tc.strict)+ got := make([][2]string, 0, len(pairs))+ for _, p := range pairs {+ got = append(got, [2]string{p.Question, p.Answer})+ }+ if len(got) != len(tc.want) {+ t.Fatalf("pairs = %v, want %v", got, tc.want)+ }+ for i := range got {+ if got[i] != tc.want[i] {+ t.Errorf("pair %d = %q, want %q", i, got[i], tc.want[i])+ }+ }+ })+ }+}++// BenchmarkExtractQATable makes the cost of the extraction reproducible:+//+// go test -run '^$' -bench BenchmarkExtractQATable -benchmem ./internal/ingestion/component/chunker/+//+// The cells carry the two shapes that the previous tag-level scan handled+// worst — an embedded newline and an entity — so the numbers reflect the+// markup table items actually carry.+func BenchmarkExtractQATable(b *testing.B) {+ for _, rows := range []int{10, 256, 1000} {+ markup := benchmarkTable(rows)+ b.Run(fmt.Sprintf("%d_rows", rows), func(b *testing.B) {+ b.SetBytes(int64(len(markup)))+ for i := 0; i < b.N; i++ {+ extractQATable(markup, false)+ }+ })+ }+}++// benchmarkTable renders a two-column table of the given size, one row per+// Q&A pair, with newlines and entities inside the cells.+func benchmarkTable(rows int) string {+ var sb strings.Builder+ sb.WriteString("<table><caption>Sheet1</caption>" ++ "<tr><th>question</th><th>answer</th></tr>")+ for i := 0; i < rows; i++ {+ fmt.Fprintf(&sb,+ "<tr><td>question %d, which spans\na second line</td>"++ "<td>answer %d, with an entity & and more text</td></tr>", i, i)+ }+ sb.WriteString("</table>")+ return sb.String()+}diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.goindex 342170853c..4700710f2c 100644--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go@@ -3,35 +3,44 @@ package chunkerimport ("bytes""context"+ "strings""testing""github.com/xuri/excelize/v2"+"ragflow/internal/parser/parser")-func TestXLSXQARegression(t *testing.T) {+// xlsxWorkbook renders rows into an in-memory workbook so the tests run+// against the real spreadsheet parser instead of hand-written markup.+func xlsxWorkbook(t *testing.T, rows [][]string) []byte {+ t.Helper()f := excelize.NewFile()sh := f.GetSheetName(0)- for i, r := range [][]string{- {"question", "answer"},- {"What is RAGFlow?", "A RAG engine."},- {"Where are the docs?", "On the website."},- } {+ for i, r := range rows {for j, c := range r {cell, _ := excelize.CoordinatesToCellName(j+1, i+1)- _ = f.SetCellValue(sh, cell, c)+ if err := f.SetCellValue(sh, cell, c); err != nil {+ t.Fatal(err)+ }}}var buf bytes.Bufferif err := f.Write(&buf); err != nil {t.Fatal(err)}+ return buf.Bytes()+}+// qaChunksFromXLSX drives a workbook through the real XLSX parser and then+// the QA chunker, returning the chunks the pipeline would emit.+func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {+ t.Helper()p, err := parser.NewXLSXParser("")if err != nil {t.Fatal(err)}- res := p.ParseWithResult(context.Background(), "qa.xlsx", buf.Bytes())+ res := p.ParseWithResult(context.Background(), "qa.xlsx", data)if res.Err != nil {t.Fatal(res.Err)}@@ -53,8 +62,46 @@ func TestXLSXQARegression(t *testing.T) {t.Fatal(err)}chunks, _ := out["chunks"].([]map[string]any)- t.Logf("format=%q QA chunks=%d", res.OutputFormat, len(chunks))+ return chunks+}++// TestXLSXQARegression is the end-to-end smoke test: every row of the sheet+// becomes one chunk. The chunker has no header concept, so the first row is+// a Q&A pair like any other.+func TestXLSXQARegression(t *testing.T) {+ chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{+ {"question", "answer"},+ {"What is RAGFlow?", "A RAG engine."},+ {"Where are the docs?", "On the website."},+ }))+ t.Logf("QA chunks=%d", len(chunks))+ if len(chunks) != 3 {+ t.Fatalf("expected 3 chunks, got %d", len(chunks))+ }+}++// A spreadsheet cell keeps the newline its author typed (Alt+Enter) and the+// parser renders it into the <tr>/<td> HTML verbatim, so a QA pair whose+// question or answer spans lines must survive extraction intact. These rows+// used to disappear from the chunk list without a trace.+func TestXLSXQAMultilineCells(t *testing.T) {+ const multilineQ = "请问全国碳排放权交易市场纳入配额管理的重点排放单\n位名录,是否会公布?"+ const multilineA = "需要公布。根据《碳排放权交易管理办法(试行)》。"+ const multilineAnswer = "跨行的答案\n第二行\n第三行"++ chunks := qaChunksFromXLSX(t, xlsxWorkbook(t, [][]string{+ {"question", "answer"},+ {multilineQ, multilineA},+ {"跨行的问句\n第二行", multilineAnswer},+ }))+ t.Logf("QA chunks=%d", len(chunks))if len(chunks) != 3 {t.Fatalf("expected 3 chunks, got %d", len(chunks))}+ texts := strings.Join(chunkTexts(chunks), "\n")+ for _, want := range []string{multilineQ, multilineA, multilineAnswer} {+ if !strings.Contains(texts, want) {+ t.Errorf("chunk text lost the newline-bearing cell %q", want)+ }+ }}