Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Require exactly two cells for a CSV QA pair (#19008)

Yash Raj Pandeyb502ddf6ed58
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 9ec591ab38..8f6d02961a 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -111,7 +111,7 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m
var isMarkdown bool
switch upstream.OutputFormat {
case schema.PayloadFormatHTML:
- qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult))
+ qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult), isCSV(upstream.Name))
case schema.PayloadFormatMarkdown:
qaPairs = extractQAMarkdown(stringPtrVal(upstream.MarkdownResult))
isMarkdown = true
@@ -193,6 +193,10 @@ func stringPtrVal(s *string) string {
return *s
}
+func isCSV(name string) bool {
+ return strings.HasSuffix(strings.ToLower(name), ".csv")
+}
+
// ---------------------------------------------------------------------------
// HTML / spreadsheet QA extraction
// ---------------------------------------------------------------------------
@@ -201,7 +205,7 @@ var htmlTR = regexp.MustCompile(`(?i)<tr[^>]*>(.*?)</tr>`)
var htmlTD = regexp.MustCompile(`(?i)<t[dh][^>]*>(.*?)</t[dh]>`)
var htmlTag = regexp.MustCompile(`<[^>]+>`)
-func extractQATable(htmlStr string) []qaPair {
+func extractQATable(htmlStr string, strictPairs bool) []qaPair {
if htmlStr == "" {
return nil
}
@@ -209,6 +213,10 @@ func extractQATable(htmlStr string) []qaPair {
pairs := make([]qaPair, 0, len(rows))
for _, row := range rows {
cells := htmlTD.FindAllStringSubmatch(row[1], -1)
+ // Python qa.py:365 requires exactly two fields for CSV pairs.
+ if strictPairs && len(cells) != 2 {
+ continue
+ }
var texts []string
for _, cell := range cells {
t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index b1a3ba854d..b2a7f58114 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -127,6 +127,77 @@ func TestQAChunker_HTMLTable(t *testing.T) {
}
}
+func TestQAChunker_CSVStrictPairRejectsThreeCells(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ inputs := map[string]any{
+ "name": "test.CSV",
+ "output_format": "html",
+ "html": "<table><tr><td>question</td><td></td><td>extra</td></tr></table>",
+ }
+ out, err := comp.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke failed: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 0 {
+ t.Fatalf("expected 0 chunks, got %d", len(chunks))
+ }
+}
+
+func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ inputs := map[string]any{
+ "name": "test.csv",
+ "output_format": "html",
+ "html": "<table><tr><td>question</td><td>answer</td></tr></table>",
+ }
+ out, err := comp.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke failed: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk, got %d", len(chunks))
+ }
+ cww, _ := chunks[0]["content_with_weight"].(string)
+ if cww != "Question: question\tAnswer: answer" {
+ t.Fatalf("unexpected content: %q", cww)
+ }
+}
+
+// Non-CSV names keep the old "first two non-empty cells" rule on the HTML
+// table path. Since #18800 an .xlsx file reaches the chunker as "json", not
+// "html", so this covers the shared HTML branch and not the XLSX pipeline.
+func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {
+ comp, err := NewQAChunker(map[string]any{"lang": "english"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ inputs := map[string]any{
+ "name": "test.xls",
+ "output_format": "html",
+ "html": "<table><tr><td>question</td><td></td><td>extra</td></tr></table>",
+ }
+ out, err := comp.Invoke(t.Context(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke failed: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ if len(chunks) != 1 {
+ t.Fatalf("expected 1 chunk, got %d", len(chunks))
+ }
+ cww, _ := chunks[0]["content_with_weight"].(string)
+ if cww != "Question: question\tAnswer: extra" {
+ t.Fatalf("unexpected content: %q", cww)
+ }
+}
+
func TestQAChunker_RmQAPrefix(t *testing.T) {
comp, err := NewQAChunker(map[string]any{"lang": "english"})
if err != nil {