RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Fix template integration tests for typed parser DocItem output (#21141)
xugangqiang8f3dfba875ea
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 6282cad78d..7e00d23aaa 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -437,7 +437,7 @@ func (c *ParserComponent) Outputs() map[string]string {"name": "string: the upstream file/document name (or doc_id when no name is available).","file_type": "string: canonical extension used for parser dispatch (for example pdf, md, xlsx, or other).","output_format": "string: always \"json\".",- "json": "[]map[string]any: canonical structured parser items.",+ "json": "[]parser.DocItem: canonical structured parser items.","lang": "string: the language for tokenization (e.g. English, Dutch, Chinese).","file": "map[string]any: backend-produced file metadata, when present.","doc_id": "string: source document ID, when present.",@@ -454,7 +454,7 @@ func (c *ParserComponent) Outputs() map[string]string {// "name": string (from inputs["name"], file.name, or doc_id),// "file_type": string (canonical extension used for parser dispatch),// "output_format": "json",-// "json": []map[string]any,+// "json": []parser.DocItem,// "lang": string (from inputs["lang"]; e.g. English, Dutch),// "_created_time": RFC3339Nano (via TrackElapsed),// "_elapsed_time": float64 seconds (via TrackElapsed),diff --git a/internal/ingestion/pipeline/helpers_test.go b/internal/ingestion/pipeline/helpers_test.goindex 300cc67606..7de2e78215 100644--- a/internal/ingestion/pipeline/helpers_test.go+++ b/internal/ingestion/pipeline/helpers_test.go@@ -23,6 +23,7 @@ import ("testing""time"+ "ragflow/internal/parser/parser""ragflow/internal/tokenizer")@@ -105,10 +106,10 @@ func floatSliceFromAny(t *testing.T, v any) []float64 {}}-func joinJSONItemTexts(items []map[string]any) string {+func joinJSONItemTexts(items []parser.DocItem) string {var parts []stringfor _, item := range items {- text, _ := item["text"].(string)+ text, _ := item.ToMap()["text"].(string)text = strings.TrimSpace(text)if text == "" {continuediff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.goindex b6ad120e83..0881c549e2 100644--- a/internal/ingestion/pipeline/real_storage_integration_test.go+++ b/internal/ingestion/pipeline/real_storage_integration_test.go@@ -23,6 +23,7 @@ import ("ragflow/internal/entity"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker"+ "ragflow/internal/parser/parser""ragflow/internal/server""ragflow/internal/server/config""ragflow/internal/storage"@@ -125,8 +126,9 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {// the global token-size budget (chunk_token_size=512, no backtick// delimiter): this matches Python's _merge_text_chunks_by_token_size and// is the intended behaviour. TextParser emits one normalized json item;- // GeneralChunker owns the delimiter split and collapses the atoms.- wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}+ // the blank line between the paragraphs is preserved verbatim in the+ // merged chunk text (no whitespace collapsing).+ wantChunkTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}if len(chunks) != len(wantChunkTexts) {t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))}@@ -167,13 +169,13 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {if got := parserState["output_format"]; got != "json" {t.Fatalf("parser output_format = %v, want json", got)}- jsonItems, ok := parserState["json"].([]map[string]any)+ jsonItems, ok := parserState["json"].([]parser.DocItem)if !ok || len(jsonItems) != 1 {t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])}wantParserTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}for i, wantText := range wantParserTexts {- if got := jsonItems[i]["text"]; got != wantText {+ if got := jsonItems[i].ToMap()["text"]; got != wantText {t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)}}diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex e47ad602eb..31cd18920d 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -41,6 +41,7 @@ import (componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker""ragflow/internal/ingestion/testutil"+ "ragflow/internal/parser/parser""ragflow/internal/storage""github.com/signintech/gopdf"@@ -106,7 +107,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}- wantChunkTexts := []string{"Alpha paragraph.\nBeta paragraph."}+ wantChunkTexts := []string{"Alpha paragraph.\n\nBeta paragraph."}if len(chunks) != len(wantChunkTexts) {t.Fatalf("len(chunks) = %d, want %d", len(chunks), len(wantChunkTexts))}@@ -191,7 +192,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {}payload := terminalPayloadFromRunOutput(t, out, terminalIDs[0])- wantMergedText := "Alpha paragraph.\nBeta paragraph."+ wantMergedText := "Alpha paragraph.\n\nBeta paragraph."assertTokenizerTerminalChunk(t, payload, filename, wantMergedText)state := stateFromRunOutput(t, out)@@ -215,11 +216,11 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {if got := parserState["output_format"]; got != "json" {t.Fatalf("parser output_format = %v, want json", got)}- jsonItems, ok := parserState["json"].([]map[string]any)+ jsonItems, ok := parserState["json"].([]parser.DocItem)if !ok || len(jsonItems) != 1 {t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])}- if got := jsonItems[0]["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {+ if got := jsonItems[0].ToMap()["text"]; got != "Alpha paragraph.\n\nBeta paragraph." {t.Fatalf("parser json[0].text = %v, want normalized full text", got)}chunkerState, ok := state["OneChunker:DryDrinksVisit"]@@ -317,9 +318,9 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)if got := fileState["page_count"]; got != fixture.PageCount {t.Fatalf("parser file.page_count = %v, want %d", got, fixture.PageCount)}- jsonItems, ok := parserState["json"].([]map[string]any)+ jsonItems, ok := parserState["json"].([]parser.DocItem)if !ok || len(jsonItems) == 0 {- t.Fatalf("parser json = %T/%v, want non-empty []map[string]any", parserState["json"], parserState["json"])+ t.Fatalf("parser json = %T/%v, want non-empty []parser.DocItem", parserState["json"], parserState["json"])}parserJoined := joinJSONItemTexts(jsonItems)assertNormalizedContainsAll(t, parserJoined, fixture.ExpectContains...)@@ -410,13 +411,13 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {if !ok {t.Fatal("missing Parser:HipSignsRhyme state")}- jsonItems, ok := parserState["json"].([]map[string]any)- if !ok || len(jsonItems) != 6 {- t.Fatalf("parser json = %T/%v, want 6 items", parserState["json"], parserState["json"])+ jsonItems, ok := parserState["json"].([]parser.DocItem)+ if !ok || len(jsonItems) != 1 {+ t.Fatalf("parser json = %T/%v, want 1 item", parserState["json"], parserState["json"])}- wantParserTexts := []string{"PART ONE", "Intro paragraph.", "Section 1", "Detail paragraph.", "PART TWO", "Tail paragraph."}+ wantParserTexts := []string{"PART ONE\n\nIntro paragraph.\n\nSection 1\n\nDetail paragraph.\n\nPART TWO\n\nTail paragraph."}for i, wantText := range wantParserTexts {- if got := jsonItems[i]["text"]; got != wantText {+ if got := jsonItems[i].ToMap()["text"]; got != wantText {t.Fatalf("parser json[%d].text = %v, want %q", i, got, wantText)}}