RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Require exactly two cells for a CSV QA pair (#19008)
Yash Raj Pandeyb502ddf6ed58
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 9ec591ab38..8f6d02961a 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -111,7 +111,7 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (mvar isMarkdown boolswitch upstream.OutputFormat {case schema.PayloadFormatHTML:- qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult))+ qaPairs = extractQATable(stringPtrVal(upstream.HTMLResult), isCSV(upstream.Name))case schema.PayloadFormatMarkdown:qaPairs = extractQAMarkdown(stringPtrVal(upstream.MarkdownResult))isMarkdown = true@@ -193,6 +193,10 @@ func stringPtrVal(s *string) string {return *s}+func isCSV(name string) bool {+ return strings.HasSuffix(strings.ToLower(name), ".csv")+}+// ---------------------------------------------------------------------------// HTML / spreadsheet QA extraction// ---------------------------------------------------------------------------@@ -201,7 +205,7 @@ var htmlTR = regexp.MustCompile(`(?i)<tr[^>]*>(.*?)</tr>`)var htmlTD = regexp.MustCompile(`(?i)<t[dh][^>]*>(.*?)</t[dh]>`)var htmlTag = regexp.MustCompile(`<[^>]+>`)-func extractQATable(htmlStr string) []qaPair {+func extractQATable(htmlStr string, strictPairs bool) []qaPair {if htmlStr == "" {return nil}@@ -209,6 +213,10 @@ func extractQATable(htmlStr string) []qaPair {pairs := make([]qaPair, 0, len(rows))for _, row := range rows {cells := htmlTD.FindAllStringSubmatch(row[1], -1)+ // Python qa.py:365 requires exactly two fields for CSV pairs.+ if strictPairs && len(cells) != 2 {+ continue+ }var texts []stringfor _, cell := range cells {t := html.UnescapeString(htmlTag.ReplaceAllString(cell[1], ""))diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex b1a3ba854d..b2a7f58114 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -127,6 +127,77 @@ func TestQAChunker_HTMLTable(t *testing.T) {}}+func TestQAChunker_CSVStrictPairRejectsThreeCells(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := map[string]any{+ "name": "test.CSV",+ "output_format": "html",+ "html": "<table><tr><td>question</td><td></td><td>extra</td></tr></table>",+ }+ out, err := comp.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke failed: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 0 {+ t.Fatalf("expected 0 chunks, got %d", len(chunks))+ }+}++func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := map[string]any{+ "name": "test.csv",+ "output_format": "html",+ "html": "<table><tr><td>question</td><td>answer</td></tr></table>",+ }+ out, err := comp.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke failed: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("expected 1 chunk, got %d", len(chunks))+ }+ cww, _ := chunks[0]["content_with_weight"].(string)+ if cww != "Question: question\tAnswer: answer" {+ t.Fatalf("unexpected content: %q", cww)+ }+}++// Non-CSV names keep the old "first two non-empty cells" rule on the HTML+// table path. Since #18800 an .xlsx file reaches the chunker as "json", not+// "html", so this covers the shared HTML branch and not the XLSX pipeline.+func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {+ comp, err := NewQAChunker(map[string]any{"lang": "english"})+ if err != nil {+ t.Fatal(err)+ }+ inputs := map[string]any{+ "name": "test.xls",+ "output_format": "html",+ "html": "<table><tr><td>question</td><td></td><td>extra</td></tr></table>",+ }+ out, err := comp.Invoke(t.Context(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke failed: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ if len(chunks) != 1 {+ t.Fatalf("expected 1 chunk, got %d", len(chunks))+ }+ cww, _ := chunks[0]["content_with_weight"].(string)+ if cww != "Question: question\tAnswer: extra" {+ t.Fatalf("unexpected content: %q", cww)+ }+}+func TestQAChunker_RmQAPrefix(t *testing.T) {comp, err := NewQAChunker(map[string]any{"lang": "english"})if err != nil {