Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(ingestion): keep Q&A/Tag chunks from collapsing into one indexed chunk (#18480)

euvre2727dac30c98
diff --git a/internal/ingestion/component/chunker/qa.go b/internal/ingestion/component/chunker/qa.go
index 4c4b1b8bd2..a9b70eaae2 100644
--- a/internal/ingestion/component/chunker/qa.go
+++ b/internal/ingestion/component/chunker/qa.go
@@ -22,8 +22,9 @@
// - HTML (xls, xlsx) → table-based Q&A (first two columns)
// - JSON (pdf, docx, xlsx) → text sections via delimiter; table items via extractQATable
//
-// Every Q&A pair becomes a single chunk with content_with_weight
-// formatted as "Question: {q}\tAnswer: {a}".
+// Every Q&A pair becomes a single chunk whose text is
+// "Question: {q}\tAnswer: {a}" (ingestion renames text to
+// content_with_weight at the index boundary).
package chunker
import (
@@ -131,11 +132,16 @@ func (c *QAChunkerComponent) invoke(_ context.Context, inputs map[string]any) (m
if isMarkdown {
answer = renderMarkdown(answer)
}
+ // Text is the pipeline's canonical chunk carrier: ingestion hashes
+ // it into the chunk id and renames it to content_with_weight. A
+ // content_with_weight-only chunk would share one empty-text id with
+ // every sibling and the index write would collapse all Q&A pairs
+ // into a single chunk.
chunk := schema.ChunkDoc{
- ContentWithWeight: fmt.Sprintf("%s%s\t%s%s", qPrefix, rmQAPrefix(pair.Question), aPrefix, answer),
- DocType: "text",
- ContentLtks: contentLTKS,
- ContentSmLtks: contentSMLTKS,
+ Text: fmt.Sprintf("%s%s\t%s%s", qPrefix, rmQAPrefix(pair.Question), aPrefix, answer),
+ DocType: "text",
+ ContentLtks: contentLTKS,
+ ContentSmLtks: contentSMLTKS,
}
//
// index), image id + coordinates carried from the source item.
@@ -226,7 +232,9 @@ func extractQATable(htmlStr string, strictPairs bool) []qaPair {
}
}
if len(texts) >= 2 {
- pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1]})
+ // RowNum mirrors Python qa.py's enumerate over the extracted
+ // pairs (beAdoc(..., row_num=ii)) → top_int.
+ pairs = append(pairs, qaPair{Question: texts[0], Answer: texts[1], RowNum: len(pairs)})
}
}
return pairs
diff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.go
index b6730bbaf4..aa2f762dab 100644
--- a/internal/ingestion/component/chunker/qa_batch2_test.go
+++ b/internal/ingestion/component/chunker/qa_batch2_test.go
@@ -53,7 +53,7 @@ func TestQAChunker_DefaultLangIsChinese(t *testing.T) {
if len(chunks) == 0 {
t.Fatalf("no chunks produced")
}
- cw := chunks[0]["content_with_weight"].(string)
+ cw := chunks[0]["text"].(string)
if !contains(cw, "问题:") || !contains(cw, "回答:") {
t.Errorf("empty lang should default to Chinese prefixes, got %q", cw)
}
@@ -131,7 +131,7 @@ func TestQAChunker_CarriesImageAndPositions(t *testing.T) {
t.Errorf("QA chunk lost upstream _pdf_positions")
}
// The prefix-stripped content must still be present.
- cw, _ := c["content_with_weight"].(string)
+ cw, _ := c["text"].(string)
if !contains(cw, "A") {
t.Errorf("QA content missing answer: %q", cw)
}
diff --git a/internal/ingestion/component/chunker/qa_id_test.go b/internal/ingestion/component/chunker/qa_id_test.go
new file mode 100644
index 0000000000..22f728bc13
--- /dev/null
+++ b/internal/ingestion/component/chunker/qa_id_test.go
@@ -0,0 +1,92 @@
+package chunker
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "ragflow/internal/ingestion/task/indexdoc"
+)
+
+// xlsxQATableHTML is the HTML the spreadsheet parser emits for a
+// headerless two-column Q&A workbook (renderSheetTables repeats the
+// first row as the <th> header; extractQATable still reads it as a pair).
+const xlsxQATableHTML = `<table><caption>Sheet1</caption>
+<tr><th>门打不开怎么办</th><th>请家人回来开门</th></tr>
+<tr><td>门的材质是什么</td><td>蜂窝纸板</td></tr>
+<tr><td>怎么确定门的真伪</td><td>提供钢印号</td></tr>
+</table>`
+
+// TestQAChunker_HTMLTable_TextCarrierAndTopInt pins the spreadsheet Q&A
+// contract the index write depends on: every row becomes its own chunk
+// carried by `text` (not content_with_weight, which ingestion renames
+// from text — a content_with_weight-only chunk would hash an empty text
+// into a shared id and the index write would collapse all rows into one
+// chunk), and each chunk carries the pair index in top_int like Python
+// qa.py's beAdoc(..., row_num=ii).
+func TestQAChunker_HTMLTable_TextCarrierAndTopInt(t *testing.T) {
+ inputs := map[string]any{
+ "name": "售后常见问题.xlsx",
+ "output_format": "html",
+ "html": xlsxQATableHTML,
+ }
+ chunks := qaInvoke(t, inputs)
+ if len(chunks) != 3 {
+ t.Fatalf("want one chunk per Q&A row, got %d", len(chunks))
+ }
+ for i, c := range chunks {
+ if _, exists := c["content_with_weight"]; exists {
+ t.Errorf("chunk %d must carry text, not content_with_weight (ingestion renames): %#v", i, c)
+ }
+ txt, _ := c["text"].(string)
+ if txt == "" || !contains(txt, "问题:") || !contains(txt, "回答:") {
+ t.Errorf("chunk %d text = %q, want 问题/回答 pair", i, txt)
+ }
+ raw, ok := c["top_int"].([]any)
+ if !ok || len(raw) != 1 || int(raw[0].(float64)) != i {
+ t.Errorf("chunk %d top_int = %#v, want [%d]", i, c["top_int"], i)
+ }
+ }
+}
+
+// TestQAChunker_ChunksGetDistinctIndexIDs runs the QA chunker output
+// through the ingestion post-processing that assigns index ids, guarding
+// the end-to-end invariant behind the reported bug: an N-row Q&A
+// workbook must yield N distinct chunk ids (countDistinctChunkIDs == N),
+// not one shared empty-text id with only the last row surviving.
+func TestQAChunker_ChunksGetDistinctIndexIDs(t *testing.T) {
+ inputs := map[string]any{
+ "name": "售后常见问题.xlsx",
+ "output_format": "html",
+ "html": xlsxQATableHTML,
+ }
+ comp, err := NewQAChunker(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := comp.Invoke(context.Background(), nil, inputs)
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks := indexdoc.NormalizeChunks(out)
+ if _, err := indexdoc.ProcessChunksForPipeline(chunks, "doc-1", "售后常见问题.xlsx", time.Now()); err != nil {
+ t.Fatalf("ProcessChunksForPipeline: %v", err)
+ }
+ ids := map[string]struct{}{}
+ for i, ck := range chunks {
+ id, _ := ck["id"].(string)
+ if id == "" {
+ t.Fatalf("chunk %d has no id", i)
+ }
+ ids[id] = struct{}{}
+ if _, hasText := ck["text"]; hasText {
+ t.Errorf("chunk %d: text must be renamed to content_with_weight by ingestion", i)
+ }
+ if cw, _ := ck["content_with_weight"].(string); cw == "" {
+ t.Errorf("chunk %d: content_with_weight empty after rename", i)
+ }
+ }
+ if len(ids) != 3 {
+ t.Fatalf("want 3 distinct chunk ids, got %d", len(ids))
+ }
+}
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index b2a7f58114..61bc0b1195 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -56,7 +56,7 @@ func TestQAChunker_DelimiterTab(t *testing.T) {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
chunk := chunks[0]
- cww, _ := chunk["content_with_weight"].(string)
+ cww, _ := chunk["text"].(string)
if cww != "Question: What is Go?\tAnswer: Go is a programming language." {
t.Fatalf("unexpected content: %q", cww)
}
@@ -81,7 +81,7 @@ func TestQAChunker_DelimiterComma(t *testing.T) {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
chunk := chunks[0]
- cww, _ := chunk["content_with_weight"].(string)
+ cww, _ := chunk["text"].(string)
if cww != "Question: What is Rust?\tAnswer: Rust is a systems language." {
t.Fatalf("unexpected content: %q", cww)
}
@@ -165,7 +165,7 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if cww != "Question: question\tAnswer: answer" {
t.Fatalf("unexpected content: %q", cww)
}
@@ -192,7 +192,7 @@ func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if cww != "Question: question\tAnswer: extra" {
t.Fatalf("unexpected content: %q", cww)
}
@@ -213,7 +213,7 @@ func TestQAChunker_RmQAPrefix(t *testing.T) {
t.Fatalf("Invoke failed: %v", err)
}
chunks, _ := out["chunks"].([]map[string]any)
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if cww != "Question: What is Go?\tAnswer: Go is a language." {
t.Fatalf("prefix not stripped: %q", cww)
}
@@ -257,7 +257,7 @@ func TestQAChunker_CaseInsensitivePrefix(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if cww != "Question: Hello\tAnswer: World" {
t.Fatalf("case-insensitive prefix not stripped: %q", cww)
}
@@ -281,7 +281,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
// Python qa.py:241 uses `[\t:: ]+`, so a space is a valid separator:
// a leading "A"/"Q" followed by a space is stripped.
if cww != "Question: language model is useful\tAnswer: How does it work" {
@@ -327,7 +327,7 @@ func TestQAChunker_ChineseLang(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if want := "问题:什么是Go?\t回答:Go是一种编程语言。"; cww != want {
t.Fatalf("unexpected content: %q, want %q", cww, want)
}
@@ -351,7 +351,7 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
- cww, _ := chunks[0]["content_with_weight"].(string)
+ cww, _ := chunks[0]["text"].(string)
if !strings.Contains(cww, "<strong>bold</strong>") &&
!strings.Contains(cww, "<b>bold</b>") {
t.Fatalf("markdown not rendered to HTML: %q", cww)