Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Fix template integration tests for typed parser DocItem output (#21141)

xugangqiang8f3dfba875ea
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 6282cad78d..7e00d23aaa 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -437,7 +437,7 @@ func (c *ParserComponent) Outputs() map[string]string {
"name": "string: the upstream file/document name (or doc_id when no name is available).",
"file_type": "string: canonical extension used for parser dispatch (for example pdf, md, xlsx, or other).",
"output_format": "string: always \"json\".",
- "json": "[]map[string]any: canonical structured parser items.",
+ "json": "[]parser.DocItem: canonical structured parser items.",
"lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).",
"file": "map[string]any: backend-produced file metadata, when present.",
"doc_id": "string: source document ID, when present.",
@@ -454,7 +454,7 @@ func (c *ParserComponent) Outputs() map[string]string {
// "name": string (from inputs["name"], file.name, or doc_id),
// "file_type": string (canonical extension used for parser dispatch),
// "output_format": "json",
-// "json": []map[string]any,
+// "json": []parser.DocItem,
// "lang": string (from inputs["lang"]; e.g. English, Dutch),
// "_created_time": RFC3339Nano (via TrackElapsed),
// "_elapsed_time": float64 seconds (via TrackElapsed),
diff --git a/internal/ingestion/pipeline/helpers_test.go b/internal/ingestion/pipeline/helpers_test.go
index 300cc67606..7de2e78215 100644
--- a/internal/ingestion/pipeline/helpers_test.go
+++ b/internal/ingestion/pipeline/helpers_test.go
@@ -23,6 +23,7 @@ import (
"testing"
"time"
+ "ragflow/internal/parser/parser"
"ragflow/internal/tokenizer"
)
@@ -105,10 +106,10 @@ func floatSliceFromAny(t *testing.T, v any) []float64 {
}
}
-func joinJSONItemTexts(items []map[string]any) string {
+func joinJSONItemTexts(items []parser.DocItem) string {
var parts []string
for _, item := range items {
- text, _ := item["text"].(string)
+ text, _ := item.ToMap()["text"].(string)
text = strings.TrimSpace(text)
if text == "" {
continue
diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.go
index b6ad120e83..0881c549e2 100644
--- a/internal/ingestion/pipeline/real_storage_integration_test.go
+++ b/internal/ingestion/pipeline/real_storage_integration_test.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/entity"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
+ "ragflow/internal/parser/parser"
"ragflow/internal/server"
"ragflow/internal/server/config"
"ragflow/internal/storage"
@@ -125,8 +126,9 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
// the global token-size budget (chunk_token_size=512, no backtick
// delimiter): this matches Python's _merge_text_chunks_by_token_size and
// is the intended behaviour. TextParser emits one normalized json item;
- // GeneralChunker owns the delimiter split and collapses the atoms.
- wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}
+ // the blank line between the paragraphs is preserved verbatim in the
+ // merged chunk text (no whitespace collapsing).
+ wantChunkTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}
if len(chunks) != len(wantChunkTexts) {
t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))
}
@@ -167,13 +169,13 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
if got := parserState["output_format"]; got != "json" {
t.Fatalf("parser output_format = %v, want json", got)
}
- jsonItems, ok := parserState["json"].([]map[string]any)
+ jsonItems, ok := parserState["json"].([]parser.DocItem)
if !ok || len(jsonItems) != 1 {
t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])
}
wantParserTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}
for i, wantText := range wantParserTexts {
- if got := jsonItems[i]["text"]; got != wantText {
+ if got := jsonItems[i].ToMap()["text"]; got != wantText {
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
}
}
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index e47ad602eb..31cd18920d 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -41,6 +41,7 @@ import (
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
"ragflow/internal/ingestion/testutil"
+ "ragflow/internal/parser/parser"
"ragflow/internal/storage"
"github.com/signintech/gopdf"
@@ -106,7 +107,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
- wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}
+ wantChunkTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}
if len(chunks) != len(wantChunkTexts) {
t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))
}
@@ -191,7 +192,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
}
payload := terminalPayloadFromRunOutput(t, out, terminalIDs[0])
- wantMergedText := "Alpha paragraph.\nBeta paragraph."
+ wantMergedText := "Alpha paragraph.\n\nBeta paragraph."
assertTokenizerTerminalChunk(t, payload, filename, wantMergedText)
state := stateFromRunOutput(t, out)
@@ -215,11 +216,11 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
if got := parserState["output_format"]; got != "json" {
t.Fatalf("parser output_format = %v, want json", got)
}
- jsonItems, ok := parserState["json"].([]map[string]any)
+ jsonItems, ok := parserState["json"].([]parser.DocItem)
if !ok || len(jsonItems) != 1 {
t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])
}
- if got := jsonItems[0]["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {
+ if got := jsonItems[0].ToMap()["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {
t.Fatalf("parser json[0].text = %v, want normalized full text", got)
}
chunkerState, ok := state["OneChunker:DryDrinksVisit"]
@@ -317,9 +318,9 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)
if got := fileState["page_count"]; got != fixture.PageCount {
t.Fatalf("parser file.page_count = %v, want %d", got, fixture.PageCount)
}
- jsonItems, ok := parserState["json"].([]map[string]any)
+ jsonItems, ok := parserState["json"].([]parser.DocItem)
if !ok || len(jsonItems) == 0 {
- t.Fatalf("parser json = %T/%v, want non-empty []map[string]any", parserState["json"], parserState["json"])
+ t.Fatalf("parser json = %T/%v, want non-empty []parser.DocItem", parserState["json"], parserState["json"])
}
parserJoined := joinJSONItemTexts(jsonItems)
assertNormalizedContainsAll(t, parserJoined, fixture.ExpectContains...)
@@ -410,13 +411,13 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
if !ok {
t.Fatal("missing Parser:HipSignsRhyme state")
}
- jsonItems, ok := parserState["json"].([]map[string]any)
- if !ok || len(jsonItems) != 6 {
- t.Fatalf("parser json = %T/%v, want 6 items", parserState["json"], parserState["json"])
+ jsonItems, ok := parserState["json"].([]parser.DocItem)
+ if !ok || len(jsonItems) != 1 {
+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])
}
- wantParserTexts := []string{"PART ONE", "Intro paragraph.", "Section 1", "Detail paragraph.", "PART TWO", "Tail paragraph."}
+ wantParserTexts := []string{"PART ONE\n\nIntro paragraph.\n\nSection 1\n\nDetail paragraph.\n\nPART TWO\n\nTail paragraph."}
for i, wantText := range wantParserTexts {
- if got := jsonItems[i]["text"]; got != wantText {
+ if got := jsonItems[i].ToMap()["text"]; got != wantText {
t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)
}
}