RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(ingestion): keep Q&A/Tag chunks from collapsing into one indexed chunk (#18480)
euvre2727dac30c98
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.goindex 4c4b1b8bd2..a9b70eaae2 100644--- a/internal/ingestion/component/chunker/qa.go+++ b/internal/ingestion/component/chunker/qa.go@@ -22,8 +22,9 @@// - HTML (xls, xlsx) → table-based Q&A (first two columns)// - JSON (pdf, docx, xlsx) → text sections via delimiter; table items via extractQATable//-// Every Q&A pair becomes a single chunk with content_with_weight-// formatted as "Question: {q}\tAnswer: {a}".+// Every Q&A pair becomes a single chunk whose text is+// "Question: {q}\tAnswer: {a}" (ingestion renames text to+// content_with_weight at the index boundary).package chunkerimport (@@ -131,11 +132,16 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (mif isMarkdown {answer = renderMarkdown(answer)}+ // Text is the pipeline's canonical chunk carrier: ingestion hashes+ // it into the chunk id and renames it to content_with_weight. A+ // content_with_weight-only chunk would share one empty-text id with+ // every sibling and the index write would collapse all Q&A pairs+ // into a single chunk.chunk := schema.ChunkDoc{- ContentWithWeight: fmt.Sprintf("%s%s\t%s%s", qPrefix, rmQAPrefix(pair.Question), aPrefix, answer),- DocType: "text",- ContentLtks: contentLTKS,- ContentSmLtks: contentSMLTKS,+ Text: fmt.Sprintf("%s%s\t%s%s", qPrefix, rmQAPrefix(pair.Question), aPrefix, answer),+ DocType: "text",+ ContentLtks: contentLTKS,+ ContentSmLtks: contentSMLTKS,}//// index), image id + coordinates carried from the source item.@@ -226,7 +232,9 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {}}if len(texts) >= 2 {- pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1]})+ // RowNum mirrors Python qa.py's enumerate over the extracted+ // pairs (beAdoc(..., row_num=ii)) → top_int.+ pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)})}}return pairsdiff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.goindex b6730bbaf4..aa2f762dab 100644--- a/internal/ingestion/component/chunker/qa_batch2_test.go+++ b/internal/ingestion/component/chunker/qa_batch2_test.go@@ -53,7 +53,7 @@ func TestQAChunker_DefaultLangIsChinese(t *testing.T) {if len(chunks) == 0 {t.Fatalf("no chunks produced")}- cw := chunks[0]["content_with_weight"].(string)+ cw := chunks[0]["text"].(string)if !contains(cw, "问题:") || !contains(cw, "回答:") {t.Errorf("empty lang should default to Chinese prefixes, got %q", cw)}@@ -131,7 +131,7 @@ func TestQAChunker_CarriesImageAndPositions(t *testing.T) {t.Errorf("QA chunk lost upstream _pdf_positions")}// The prefix-stripped content must still be present.- cw, _ := c["content_with_weight"].(string)+ cw, _ := c["text"].(string)if !contains(cw, "A") {t.Errorf("QA content missing answer: %q", cw)}diff --git a/internal/ingestion/component/chunker/qa_id_test.go b/internal/ingestion/component/chunker/qa_id_test.gonew file mode 100644index 0000000000..22f728bc13--- /dev/null+++ b/internal/ingestion/component/chunker/qa_id_test.go@@ -0,0 +1,92 @@+package chunker++import (+ "context"+ "testing"+ "time"++ "ragflow/internal/ingestion/task/indexdoc"+)++// xlsxQATableHTML is the HTML the spreadsheet parser emits for a+// headerless two-column Q&A workbook (renderSheetTables repeats the+// first row as the <th> header; extractQATable still reads it as a pair).+const xlsxQATableHTML = `<table><caption>Sheet1</caption>+<tr><th>门打不开怎么办</th><th>请家人回来开门</th></tr>+<tr><td>门的材质是什么</td><td>蜂窝纸板</td></tr>+<tr><td>怎么确定门的真伪</td><td>提供钢印号</td></tr>+</table>`++// TestQAChunker_HTMLTable_TextCarrierAndTopInt pins the spreadsheet Q&A+// contract the index write depends on: every row becomes its own chunk+// carried by `text` (not content_with_weight, which ingestion renames+// from text — a content_with_weight-only chunk would hash an empty text+// into a shared id and the index write would collapse all rows into one+// chunk), and each chunk carries the pair index in top_int like Python+// qa.py's beAdoc(..., row_num=ii).+func TestQAChunker_HTMLTable_TextCarrierAndTopInt(t *testing.T) {+ inputs := map[string]any{+ "name": "售后常见问题.xlsx",+ "output_format": "html",+ "html": xlsxQATableHTML,+ }+ chunks := qaInvoke(t, inputs)+ if len(chunks) != 3 {+ t.Fatalf("want one chunk per Q&A row, got %d", len(chunks))+ }+ for i, c := range chunks {+ if _, exists := c["content_with_weight"]; exists {+ t.Errorf("chunk %d must carry text, not content_with_weight (ingestion renames): %#v", i, c)+ }+ txt, _ := c["text"].(string)+ if txt == "" || !contains(txt, "问题:") || !contains(txt, "回答:") {+ t.Errorf("chunk %d text = %q, want 问题/回答 pair", i, txt)+ }+ raw, ok := c["top_int"].([]any)+ if !ok || len(raw) != 1 || int(raw[0].(float64)) != i {+ t.Errorf("chunk %d top_int = %#v, want [%d]", i, c["top_int"], i)+ }+ }+}++// TestQAChunker_ChunksGetDistinctIndexIDs runs the QA chunker output+// through the ingestion post-processing that assigns index ids, guarding+// the end-to-end invariant behind the reported bug: an N-row Q&A+// workbook must yield N distinct chunk ids (countDistinctChunkIDs == N),+// not one shared empty-text id with only the last row surviving.+func TestQAChunker_ChunksGetDistinctIndexIDs(t *testing.T) {+ inputs := map[string]any{+ "name": "售后常见问题.xlsx",+ "output_format": "html",+ "html": xlsxQATableHTML,+ }+ comp, err := NewQAChunker(nil)+ if err != nil {+ t.Fatal(err)+ }+ out, err := comp.Invoke(context.Background(), nil, inputs)+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks := indexdoc.NormalizeChunks(out)+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "售后常见问题.xlsx", time.Now()); err != nil {+ t.Fatalf("ProcessChunksForPipeline: %v", err)+ }+ ids := map[string]struct{}{}+ for i, ck := range chunks {+ id, _ := ck["id"].(string)+ if id == "" {+ t.Fatalf("chunk %d has no id", i)+ }+ ids[id] = struct{}{}+ if _, hasText := ck["text"]; hasText {+ t.Errorf("chunk %d: text must be renamed to content_with_weight by ingestion", i)+ }+ if cw, _ := ck["content_with_weight"].(string); cw == "" {+ t.Errorf("chunk %d: content_with_weight empty after rename", i)+ }+ }+ if len(ids) != 3 {+ t.Fatalf("want 3 distinct chunk ids, got %d", len(ids))+ }+}diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex b2a7f58114..61bc0b1195 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -56,7 +56,7 @@ func TestQAChunker_DelimiterTab(t *testing.T) {t.Fatalf("expected 1 chunk, got %d", len(chunks))}chunk := chunks[0]- cww, _ := chunk["content_with_weight"].(string)+ cww, _ := chunk["text"].(string)if cww != "Question: What is Go?\tAnswer: Go is a programming language." {t.Fatalf("unexpected content: %q", cww)}@@ -81,7 +81,7 @@ func TestQAChunker_DelimiterComma(t *testing.T) {t.Fatalf("expected 1 chunk, got %d", len(chunks))}chunk := chunks[0]- cww, _ := chunk["content_with_weight"].(string)+ cww, _ := chunk["text"].(string)if cww != "Question: What is Rust?\tAnswer: Rust is a systems language." {t.Fatalf("unexpected content: %q", cww)}@@ -165,7 +165,7 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if cww != "Question: question\tAnswer: answer" {t.Fatalf("unexpected content: %q", cww)}@@ -192,7 +192,7 @@ func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if cww != "Question: question\tAnswer: extra" {t.Fatalf("unexpected content: %q", cww)}@@ -213,7 +213,7 @@ func TestQAChunker_RmQAPrefix(t *testing.T) {t.Fatalf("Invoke failed: %v", err)}chunks, _ := out["chunks"].([]map[string]any)- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if cww != "Question: What is Go?\tAnswer: Go is a language." {t.Fatalf("prefix not stripped: %q", cww)}@@ -257,7 +257,7 @@ func TestQAChunker_CaseInsensitivePrefix(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if cww != "Question: Hello\tAnswer: World" {t.Fatalf("case-insensitive prefix not stripped: %q", cww)}@@ -281,7 +281,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)// Python qa.py:241 uses `[\t:: ]+`, so a space is a valid separator:// a leading "A"/"Q" followed by a space is stripped.if cww != "Question: language model is useful\tAnswer: How does it work" {@@ -327,7 +327,7 @@ func TestQAChunker_ChineseLang(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if want := "问题:什么是Go?\t回答:Go是一种编程语言。"; cww != want {t.Fatalf("unexpected content: %q, want %q", cww, want)}@@ -351,7 +351,7 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}- cww, _ := chunks[0]["content_with_weight"].(string)+ cww, _ := chunks[0]["text"].(string)if !strings.Contains(cww, "<strong>bold</strong>") &&!strings.Contains(cww, "<b>bold</b>") {t.Fatalf("markdown not rendered to HTML: %q", cww)