RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Add typed envelope accessors for ingestion component boundaries (#21273)
xugangqiang31296803d4da
diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.goindex 765c5aa7ca..c08bcab6a2 100644--- a/internal/ingestion/component/chunker/bare_delimiter_test.go+++ b/internal/ingestion/component/chunker/bare_delimiter_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -50,7 +51,7 @@ func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[strinif msg, ok := out["_ERROR"].(string); ok && msg != "" {t.Fatalf("Go returned _ERROR: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}@@ -122,7 +123,7 @@ func TestBareDelimiterSplitsJSON(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if got := joinedText(chunks); got != text {t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))}diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.goindex 78efdacd4e..3d75f0859c 100644--- a/internal/ingestion/component/chunker/custom_delim_test.go+++ b/internal/ingestion/component/chunker/custom_delim_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""slices""testing")@@ -31,7 +32,7 @@ func invokeTokenChunks(t *testing.T, params, input map[string]any) []map[string]if msg, ok := out["_ERROR"].(string); ok && msg != "" {t.Fatalf("Go returned _ERROR: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.goindex 459776305a..c7fe552049 100644--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go@@ -33,6 +33,7 @@ import ("strings""testing"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/chunk")@@ -185,7 +186,7 @@ func TestTokenChunker_BacktickEndSplitsOnlyAtLowercase(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])got := make([]string, 0, len(chunks))for _, ck := range chunks {text, _ := ck["text"].(string)@@ -225,7 +226,7 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])got := make([]string, 0, len(chunks))for _, ck := range chunks {text, _ := ck["text"].(string)diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.goindex 20e9924574..0ba17f858f 100644--- a/internal/ingestion/component/chunker/general_merge_test.go+++ b/internal/ingestion/component/chunker/general_merge_test.go@@ -487,7 +487,7 @@ func generalTextInput(text string) map[string]any {func outputChunks(t *testing.T, out map[string]any) []map[string]any {t.Helper()- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])}diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.goindex 093e99f5cf..a037cd72fb 100644--- a/internal/ingestion/component/chunker/general_test.go+++ b/internal/ingestion/component/chunker/general_test.go@@ -299,7 +299,7 @@ func TestGeneralChunkerPreservesSingleParserUnit(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("chunks = %#v, want one chunk", out["chunks"])}diff --git a/internal/ingestion/component/chunker/golden_parity_test.go b/internal/ingestion/component/chunker/golden_parity_test.goindex 2b982fdf2b..73c57fe35e 100644--- a/internal/ingestion/component/chunker/golden_parity_test.go+++ b/internal/ingestion/component/chunker/golden_parity_test.go@@ -46,6 +46,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema""github.com/google/go-cmp/cmp")@@ -230,7 +231,7 @@ func invokeChunker(t *testing.T, tc parityCase) []map[string]any {if msg, ok := out["_ERROR"].(string); ok && msg != "" {t.Fatalf("Go returned _ERROR while Python succeeded: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.goindex 1da22158b6..d9fec8f499 100644--- a/internal/ingestion/component/chunker/group_test.go+++ b/internal/ingestion/component/chunker/group_test.go@@ -191,7 +191,7 @@ func TestTitleFamilyChunkersRecognizeHeadingsAfterFirstParserLine(t *testing.T)if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("got %d chunks, want one per heading section", len(chunks))}@@ -216,7 +216,7 @@ func TestGroupTitleChunker_InvokeEmptyInput(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Errorf("chunks = %d, want 0", len(chunks))}@@ -240,7 +240,7 @@ func TestGroupTitleChunker_Headings_ASCII(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -263,7 +263,7 @@ func TestGroupTitleChunker_RootChunkAsHeading_StillSingleGroup(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -311,7 +311,7 @@ func TestGroupChunker_StructuredMetadata(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])found := falsefor _, ck := range chunks {if dt, _ := ck["doc_type_kwd"].(string); dt == "image" {@@ -352,7 +352,7 @@ func TestGroupChunker_MergesPDFPositionsAndRemovesTags(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("no chunks emitted")}@@ -400,7 +400,7 @@ func TestGroupTitleChunker_InvokeDeterministic(t *testing.T) {if err != nil {t.Fatalf("Invoke run %d: %v", run, err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])texts := make([]string, len(chunks))for i, ck := range chunks {texts[i], _ = ck["text"].(string)diff --git a/internal/ingestion/component/chunker/hierarchy_test.go b/internal/ingestion/component/chunker/hierarchy_test.goindex c421eb1f82..158628e1be 100644--- a/internal/ingestion/component/chunker/hierarchy_test.go+++ b/internal/ingestion/component/chunker/hierarchy_test.go@@ -21,6 +21,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema")func TestHierarchyTitleChunker_Registered(t *testing.T) {@@ -87,7 +88,7 @@ func TestHierarchyTitleChunker_InvokeEmptyInput(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Errorf("chunks = %d, want 0", len(chunks))}@@ -110,7 +111,7 @@ func TestHierarchyTitleChunker_NestedHeadings(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -165,7 +166,7 @@ func TestHierarchyChunker_StructuredMetadata(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])found := falsefor _, ck := range chunks {if dt, _ := ck["doc_type_kwd"].(string); dt == "image" {@@ -199,7 +200,7 @@ func TestHierarchyTitleChunker_InvokeDeterministic(t *testing.T) {if err != nil {t.Fatalf("Invoke run %d: %v", run, err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])texts := make([]string, len(chunks))for i, ck := range chunks {texts[i], _ = ck["text"].(string)@@ -252,7 +253,7 @@ func TestHierarchyTitleChunker_ColonTitlePromotion(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -284,7 +285,7 @@ func TestHierarchyTitleChunker_ColonTitleShortLine_Negative(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -319,7 +320,7 @@ func TestHierarchyTitleChunker_ShortNumericLineFilter(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -395,7 +396,7 @@ func TestHierarchyTitleChunker_ColonTitlePromotion_CJK_EdgeCase(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -433,7 +434,7 @@ func TestHierarchyTitleChunker_ShortSingleCJKLineFilter(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -472,7 +473,7 @@ func TestHierarchyTitleChunker_LayoutHeadingFallback(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -513,7 +514,7 @@ func TestHierarchyTitleChunker_ConsecutiveNonTextOrder(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])idx := func(sub string) int {for i, ck := range chunks {if text, _ := ck["text"].(string); strings.Contains(text, sub) {diff --git a/internal/ingestion/component/chunker/image_upload_test.go b/internal/ingestion/component/chunker/image_upload_test.goindex 747081d579..ef60ba1154 100644--- a/internal/ingestion/component/chunker/image_upload_test.go+++ b/internal/ingestion/component/chunker/image_upload_test.go@@ -29,6 +29,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/component/schema"parserlib "ragflow/internal/parser/parser""gorm.io/gorm"@@ -295,7 +296,7 @@ func TestImageUploadDecorator_EndToEnd(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatalf("no chunks in output")}@@ -372,7 +373,7 @@ func TestImageUploadDecorator_DebugSkipsUpload(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatalf("expected chunks in output, got %#v", out)}@@ -426,7 +427,7 @@ func TestImageUploadDecorator_DebugCapsChunks(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks missing or wrong type: %#v", out["chunks"])}@@ -463,7 +464,7 @@ func TestImageUploadDecorator_NoCapKeepsAll(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 5 {t.Fatalf("len(chunks) = %d, want 5 (no cap should keep all)", len(chunks))}diff --git a/internal/ingestion/component/chunker/json_global_merge_test.go b/internal/ingestion/component/chunker/json_global_merge_test.goindex 0e2d619fc9..3bcd7d74de 100644--- a/internal/ingestion/component/chunker/json_global_merge_test.go+++ b/internal/ingestion/component/chunker/json_global_merge_test.go@@ -31,6 +31,7 @@ package chunker// boundary like Python's _merge_text_chunks_by_token_size.import ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -66,7 +67,7 @@ func TestJSONGlobalMergeMatchesPython(t *testing.T) {if msg, _ := out["_ERROR"].(string); msg != "" {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("want 1 chunk (global merge across items), got %d", len(chunks))}@@ -119,7 +120,7 @@ func TestJSONSingleItemSubSplitUnderHardCap(t *testing.T) {if msg, _ := out["_ERROR"].(string); msg != "" {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) < 2 {t.Fatalf("over-budget item must be split, got %d chunk(s)", len(chunks))}diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.goindex e896728fa1..097d366d48 100644--- a/internal/ingestion/component/chunker/manual_test.go+++ b/internal/ingestion/component/chunker/manual_test.go@@ -23,6 +23,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema")// manualChunkerInput is a small builder for a structured (output_format=chunks)@@ -76,7 +77,7 @@ func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent {// manualChunkTexts pulls the ordered chunk texts out of a chunker output map.func manualChunkTexts(out map[string]any) []string {- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])out2 := make([]string, 0, len(chunks))for _, ck := range chunks {if s, _ := ck["text"].(string); s != "" {@@ -118,7 +119,7 @@ func TestManualChunker_Empty(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Errorf("chunks = %d, want 0", len(chunks))}@@ -155,8 +156,8 @@ func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) {t.Fatalf("manual Invoke: %v", err)}- gChunks, _ := gOut["chunks"].([]map[string]any)- mChunks, _ := mOut["chunks"].([]map[string]any)+ gChunks, _ := schema.ChunkCarrierToMaps(gOut["chunks"])+ mChunks, _ := schema.ChunkCarrierToMaps(mOut["chunks"])if !reflect.DeepEqual(gChunks, mChunks) {t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks)}@@ -253,7 +254,7 @@ func TestManualChunker_MergesPositions(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])found := falsefor _, ck := range chunks {text, _ := ck["text"].(string)diff --git a/internal/ingestion/component/chunker/one_tag_test.go b/internal/ingestion/component/chunker/one_tag_test.goindex 123ea8fad9..5e8306c2fc 100644--- a/internal/ingestion/component/chunker/one_tag_test.go+++ b/internal/ingestion/component/chunker/one_tag_test.go@@ -32,7 +32,7 @@ func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {{Text: "Sentence two@@1\t2\t3\t4##"},}out := emitOneFromItems(items, nil)- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("want 1 chunk, got %d", len(chunks))}@@ -49,7 +49,7 @@ func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {// path also strips coordinate tags before emitting.func TestOneChunkerStripsCoordTagsText(t *testing.T) {out := emitOne("Hello world@@1\t2\t3\t4##", "text")- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("want 1 chunk, got %d", len(chunks))}diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.goindex f96adf3db5..6aa0d9c746 100644--- a/internal/ingestion/component/chunker/one_test.go+++ b/internal/ingestion/component/chunker/one_test.go@@ -18,6 +18,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -32,7 +33,7 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {if err != nil {t.Fatalf("OneChunker.Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks not []map[string]any: %T", out["chunks"])}diff --git a/internal/ingestion/component/chunker/paddleocr_test.go b/internal/ingestion/component/chunker/paddleocr_test.goindex d4ea0371af..9b57caa5cf 100644--- a/internal/ingestion/component/chunker/paddleocr_test.go+++ b/internal/ingestion/component/chunker/paddleocr_test.go@@ -71,7 +71,7 @@ func TestPaddleOCRHighlightsThroughPipeline(t *testing.T) {if err != nil {t.Fatal(err)}- chunks := output["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(output["chunks"])if len(chunks) == 0 {t.Fatal("no chunks")}diff --git a/internal/ingestion/component/chunker/page_test.go b/internal/ingestion/component/chunker/page_test.goindex 337e1248cf..447bf9a208 100644--- a/internal/ingestion/component/chunker/page_test.go+++ b/internal/ingestion/component/chunker/page_test.go@@ -18,6 +18,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -32,7 +33,7 @@ func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {if err != nil {t.Fatalf("PageChunker.Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks not []map[string]any: %T", out["chunks"])}diff --git a/internal/ingestion/component/chunker/python_parity_test.go b/internal/ingestion/component/chunker/python_parity_test.goindex a94210a1e8..9ff3367ca6 100644--- a/internal/ingestion/component/chunker/python_parity_test.go+++ b/internal/ingestion/component/chunker/python_parity_test.go@@ -35,6 +35,7 @@ import ("testing""ragflow/internal/agent/runtime"+ "ragflow/internal/ingestion/component/schema")// invokeAsTokenChunker constructs a TokenChunker component and@@ -187,21 +188,11 @@ func chunksFromOutput(t *testing.T, out map[string]any) []map[string]any {if !ok {t.Fatalf("output missing 'chunks' key; got keys=%v", keysOf(out))}- switch v := raw.(type) {- case []map[string]any:- return v- case []any:- out := make([]map[string]any, 0, len(v))- for _, it := range v {- if m, ok := it.(map[string]any); ok {- out = append(out, m)- }- }- return out- default:- t.Fatalf("chunks: unexpected type %T", raw)- return nil+ maps, ok := schema.ChunkCarrierToMaps(raw)+ if !ok {+ t.Fatalf("chunks: unexpected carrier type %T", raw)}+ return maps}func keysOf(m map[string]any) []string {diff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.goindex aa2f762dab..8daa3c588f 100644--- a/internal/ingestion/component/chunker/qa_batch2_test.go+++ b/internal/ingestion/component/chunker/qa_batch2_test.go@@ -18,6 +18,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -34,7 +35,7 @@ func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("Invoke did not return []map chunks: %#v", out["chunks"])}diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.goindex 70b139eca8..583ee33d5b 100644--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go@@ -5,6 +5,7 @@ import ("strings""testing"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser")@@ -28,7 +29,7 @@ func TestCSVQAFirstRowIsData(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)}@@ -53,7 +54,7 @@ func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("malformed row created or removed a pair: %#v", chunks)}@@ -80,7 +81,7 @@ func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("malformed HTML row created a QA pair: %#v", chunks)}diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.goindex 9fdc5e3fb4..fcf73456d3 100644--- a/internal/ingestion/component/chunker/qa_test.go+++ b/internal/ingestion/component/chunker/qa_test.go@@ -66,7 +66,7 @@ func TestQAChunker_DelimiterTab(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -91,7 +91,7 @@ func TestQAChunker_DelimiterComma(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -116,7 +116,7 @@ func TestQAChunker_Markdown(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("expected 2 chunks, got %d", len(chunks))}@@ -136,7 +136,7 @@ func TestQAChunker_HTMLTable(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("expected 2 chunks, got %d", len(chunks))}@@ -156,7 +156,7 @@ func TestQAChunker_CSVStrictPairRejectsThreeCells(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Fatalf("expected 0 chunks, got %d", len(chunks))}@@ -176,7 +176,7 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -203,7 +203,7 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Fatalf("chunks = %#v, want malformed CSV row rejected", chunks)}@@ -226,7 +226,7 @@ func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -250,7 +250,7 @@ func TestQAChunker_RmQAPrefix(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])cww, _ := chunks[0]["text"].(string)if cww != "Question: What is Go?\tAnswer: Go is a language." {t.Fatalf("prefix not stripped: %q", cww)@@ -271,7 +271,7 @@ func TestQAChunker_Empty(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Fatalf("expected 0 chunks, got %d", len(chunks))}@@ -291,7 +291,7 @@ func TestQAChunker_CaseInsensitivePrefix(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -315,7 +315,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -341,7 +341,7 @@ func TestQAChunker_HeadingNoTrailingSpace(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -361,7 +361,7 @@ func TestQAChunker_ChineseLang(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -385,7 +385,7 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -415,7 +415,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {if err != nil {t.Fatalf("Invoke failed: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 3 {t.Fatalf("expected 3 chunks, got %d", len(chunks))}@@ -449,7 +449,7 @@ func TestQAChunkerSpreadsheetWireTreatsFirstRowAsQAData(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 2 {t.Fatalf("chunks = %#v, want both QA rows", out["chunks"])}diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.goindex 661c30aeef..51e076c9f5 100644--- a/internal/ingestion/component/chunker/register.go+++ b/internal/ingestion/component/chunker/register.go@@ -31,6 +31,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/ingestion/component/globals"+ "ragflow/internal/ingestion/component/schema""gorm.io/gorm")@@ -68,8 +69,25 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs mif err != nil {return nil, err}- chunks, ok := out["chunks"].([]map[string]any)- if !ok || len(chunks) == 0 {+ // Accept the legacy []map[string]any carrier verbatim (no round-trip, so+ // the id-stamp/upload logic below sees the exact maps the chunker emitted).+ // The typed []ChunkDoc carrier is normalized via GetChunks, which also+ // bridges []any-of-map, so this choke point no longer silently skips the+ // whole id-stamp/upload pass on a carrier-type mismatch.+ var chunks []map[string]any+ switch v := out["chunks"].(type) {+ case []map[string]any:+ chunks = v+ default:+ if typed, ok := schema.GetChunks(out); ok {+ chunks = schema.ChunkDocsToMaps(typed)+ // Write the normalized carrier back so the in-place id-stamp and+ // image-drop mutations below propagate to the emitted envelope+ // (the legacy []map[string]any path shares the reference already).+ out["chunks"] = chunks+ }+ }+ if len(chunks) == 0 {return out, nil}diff --git a/internal/ingestion/component/chunker/register_envelope_test.go b/internal/ingestion/component/chunker/register_envelope_test.gonew file mode 100644index 0000000000..d03fa58efd--- /dev/null+++ b/internal/ingestion/component/chunker/register_envelope_test.go@@ -0,0 +1,68 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package chunker++import (+ "context"+ "testing"++ "ragflow/internal/ingestion/component/schema"++ "gorm.io/gorm"+)++// typedChunksInner is a minimal runtime.Component fake that returns a fixed+// output map carrying a typed []ChunkDoc under "chunks".+type typedChunksInner struct {+ out map[string]any+}++func (t *typedChunksInner) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {+ return t.out, nil+}++// TestImageUploadDecorator_ProcessesTypedChunks locks the R2 regression: the+// imageUploadDecorator is the single choke point every chunker variant flows+// through. When the upstream carrier is the typed []ChunkDoc form, the+// decorator must normalize it (not silently skip), stamp chunk ids, and+// re-emit the []map[string]any wire form. Before migration the bare+// .([]map[string]any) assertion failed on a typed carrier, so the whole+// id-stamp/upload pass was skipped.+//+// kb_id is intentionally absent, so this exercises the dry-run branch (drop+// image bytes, no MinIO upload).+func TestImageUploadDecorator_ProcessesTypedChunks(t *testing.T) {+ inner := &typedChunksInner{out: map[string]any{+ "chunks": []schema.ChunkDoc{+ {Text: "hello", Image: "img/1.png"}, // no id yet+ },+ }}+ dec := &imageUploadDecorator{inner: inner}++ out, err := dec.Invoke(context.Background(), nil, map[string]any{})+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])+ if !ok {+ t.Fatalf("out[chunks] type = %T, want []map[string]any (typed carrier must be normalized)", out["chunks"])+ }+ if len(chunks) != 1 {+ t.Fatalf("len = %d, want 1", len(chunks))+ }+ if id, _ := chunks[0]["id"].(string); id == "" {+ t.Fatalf("chunk id not stamped: decorator skipped the typed carrier? chunk=%+v", chunks[0])+ }+}diff --git a/internal/ingestion/component/chunker/table_html_text_test.go b/internal/ingestion/component/chunker/table_html_text_test.goindex 917374dea1..5ade4fb167 100644--- a/internal/ingestion/component/chunker/table_html_text_test.go+++ b/internal/ingestion/component/chunker/table_html_text_test.go@@ -23,6 +23,7 @@ import ("testing""ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema")// generalTableChunksOf drives GeneralChunker.Invoke over structured TableData@@ -41,7 +42,7 @@ func generalTableChunksOf(t *testing.T, params map[string]any, items ...map[striif err != nil {t.Fatalf("GeneralChunker.Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("output chunks type = %T, want []map[string]any", out["chunks"])}diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.goindex 82e5189b9f..43eb78b510 100644--- a/internal/ingestion/component/chunker/table_test.go+++ b/internal/ingestion/component/chunker/table_test.go@@ -22,6 +22,7 @@ import ("strings""testing"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/tableutil")@@ -36,7 +37,7 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {if err != nil {t.Fatalf("TableChunker.Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks not []map[string]any: %T", out["chunks"])}diff --git a/internal/ingestion/component/chunker/textcode_parity_test.go b/internal/ingestion/component/chunker/textcode_parity_test.goindex 04daf104d0..c39a25719b 100644--- a/internal/ingestion/component/chunker/textcode_parity_test.go+++ b/internal/ingestion/component/chunker/textcode_parity_test.go@@ -24,6 +24,7 @@ import ("strings""testing"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser")@@ -108,7 +109,7 @@ func TestTextParserGeneralChunkerParity(t *testing.T) {if msg, ok := out["_ERROR"].(string); ok && msg != "" {t.Fatalf("GeneralChunker returned _ERROR: %s", msg)}- got, ok := out["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("GeneralChunker chunks has type %T", out["chunks"])}diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.goindex 2db586ed10..c7aa1323e6 100644--- a/internal/ingestion/component/chunker/title_structured_media_test.go+++ b/internal/ingestion/component/chunker/title_structured_media_test.go@@ -26,6 +26,7 @@ import ("ragflow/internal/agent/runtime""ragflow/internal/common""ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/tableutil")@@ -79,7 +80,7 @@ func titleFamilyChunks(t *testing.T, comp runtime.Component, items []map[string]if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}@@ -204,7 +205,7 @@ func TestTitleFamilyEmptyTextImageUploadsThroughDecorator(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("chunks = %d, want 1", len(chunks))}@@ -255,7 +256,7 @@ func TestTitleFamilyCaptionlessImagesGetDistinctIDs(t *testing.T) {if err != nil {t.Fatalf("decorated Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("chunks = %d, want 2", len(chunks))}@@ -288,7 +289,7 @@ func TestTitleFamilyReportsDecodeFailure(t *testing.T) {if msg, _ := out["_ERROR"].(string); !strings.HasPrefix(msg, "Input error:") {t.Fatalf("_ERROR = %v, want an input error", out["_ERROR"])}- if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 0 {+ if chunks, _ := schema.ChunkCarrierToMaps(out["chunks"]); len(chunks) != 0 {t.Fatalf("chunks = %#v, want none", chunks)}})diff --git a/internal/ingestion/component/chunker/title_test.go b/internal/ingestion/component/chunker/title_test.goindex a1b0a4c083..5f12477833 100644--- a/internal/ingestion/component/chunker/title_test.go+++ b/internal/ingestion/component/chunker/title_test.go@@ -98,7 +98,7 @@ func TestTitleChunker_InvokeEmptyInput(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Errorf("chunks = %d, want 0", len(chunks))}@@ -136,7 +136,7 @@ func TestTitleChunker_Headings_ASCII(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -165,7 +165,7 @@ func TestTitleChunker_NoHeadings_FallsBack(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -218,7 +218,7 @@ func TestTitleChunker_InvokeDeterministic(t *testing.T) {if err != nil {t.Fatalf("Invoke run %d: %v", run, err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])texts := make([]string, len(chunks))for i, ck := range chunks {texts[i], _ = ck["text"].(string)diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.goindex d35cf8fb2a..e24b44c51e 100644--- a/internal/ingestion/component/chunker/title_token_cap_test.go+++ b/internal/ingestion/component/chunker/title_token_cap_test.go@@ -445,7 +445,7 @@ func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {if err != nil {t.Fatalf("invokeGroup: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -469,7 +469,7 @@ func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {if err != nil {t.Fatalf("invokeGroup: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected cap=128 to re-split the oversized body, got %d", len(chunks))}@@ -486,7 +486,7 @@ func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -503,7 +503,7 @@ func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {if err != nil {t.Fatalf("invokeGroup: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) != 1 {t.Fatalf("cap=0 must keep 1 chunk, got %d", len(chunks))}@@ -524,7 +524,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {if err != nil {t.Fatalf("invokeGroup: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected merged split, got %d", len(chunks))}@@ -553,7 +553,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -593,7 +593,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -639,7 +639,7 @@ func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -677,7 +677,7 @@ func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T)if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}@@ -718,7 +718,7 @@ func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 2 {t.Fatalf("expected parent chunk + split leaf pieces, got %d", len(chunks))}@@ -761,7 +761,7 @@ func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {if err != nil {t.Fatalf("invokeHierarchy: %v", err)}- chunks := got["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])if len(chunks) <= 1 {t.Fatalf("expected split, got %d", len(chunks))}diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.goindex 5b87758bde..abbe0cc394 100644--- a/internal/ingestion/component/chunker/token_children_delim_test.go+++ b/internal/ingestion/component/chunker/token_children_delim_test.go@@ -1,6 +1,7 @@package chunkerimport (+ "ragflow/internal/ingestion/component/schema""testing")@@ -24,7 +25,7 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 3 {t.Fatalf("chunk count: want 3 got %d (%v)", len(chunks), chunkTexts(chunks))}@@ -64,7 +65,7 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])// The children delimiter "###" is dropped from each child.want := []string{"sec one", "sec two", "sec three"}if len(chunks) != len(want) {@@ -92,7 +93,7 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks2, _ := out2["chunks"].([]map[string]any)+ chunks2, _ := schema.ChunkCarrierToMaps(out2["chunks"])// The inner "###" matches and is DROPPED, leaving the surrounding// backticks as literal text.backtickWrapped := []string{"a `", "` b"}@@ -123,7 +124,7 @@ func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])// The children delimiter ". " is retained on each child.want := []string{"alpha one. ", "alpha two. ", "alpha three."}if len(chunks) != len(want) {diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.goindex acd3c5e36f..c846f44776 100644--- a/internal/ingestion/component/chunker/token_delim_contract_test.go+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go@@ -1,6 +1,7 @@package chunkerimport (+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -29,7 +30,7 @@ func TestTokenChunker_BareDelimiterHonored(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatalf("no chunks produced")}@@ -73,7 +74,7 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])// Custom (backtick) delimiters are DROPPED, mirroring Python's// naive_merge has_custom branch: "段落" is split away, not preserved.want := []string{"第一部分", "第二部分"}@@ -104,7 +105,7 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks2, _ := out2["chunks"].([]map[string]any)+ chunks2, _ := schema.ChunkCarrierToMaps(out2["chunks"])if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" {t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2))}diff --git a/internal/ingestion/component/chunker/token_delim_lossless_test.go b/internal/ingestion/component/chunker/token_delim_lossless_test.goindex f636277518..c4d06972aa 100644--- a/internal/ingestion/component/chunker/token_delim_lossless_test.go+++ b/internal/ingestion/component/chunker/token_delim_lossless_test.go@@ -28,6 +28,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -58,7 +59,7 @@ func driveChunker(t *testing.T, params map[string]any, input map[string]any) []mif msg, _ := out["_ERROR"].(string); msg != "" {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}diff --git a/internal/ingestion/component/chunker/token_item_separator_test.go b/internal/ingestion/component/chunker/token_item_separator_test.goindex 4d9719932e..e0c95acd9f 100644--- a/internal/ingestion/component/chunker/token_item_separator_test.go+++ b/internal/ingestion/component/chunker/token_item_separator_test.go@@ -17,6 +17,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -74,7 +75,7 @@ func invokeTokenChunkerJSON(t *testing.T, texts ...string) string {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("chunks = %#v, want one merged chunk", out["chunks"])}diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.goindex ab38e0d4a2..ded584de81 100644--- a/internal/ingestion/component/chunker/token_merge_parity_test.go+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go@@ -187,7 +187,7 @@ func invokeTokenChunker(t *testing.T, text string, chunkTokenSize int, overlappeif msg, ok := out["_ERROR"].(string); ok && msg != "" {t.Fatalf("TokenChunker returned _ERROR: %s", msg)}- raw, _ := out["chunks"].([]map[string]any)+ raw, _ := schema.ChunkCarrierToMaps(out["chunks"])texts := make([]string, 0, len(raw))for _, c := range raw {if s, ok := c["text"].(string); ok {diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.goindex 50fcb46041..1d25a4beba 100644--- a/internal/ingestion/component/chunker/token_overlap_test.go+++ b/internal/ingestion/component/chunker/token_overlap_test.go@@ -2,6 +2,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -56,7 +57,7 @@ func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatalf("expected chunks, got none")}diff --git a/internal/ingestion/component/chunker/token_oversize_split_test.go b/internal/ingestion/component/chunker/token_oversize_split_test.goindex bfcceff323..71f10f50e6 100644--- a/internal/ingestion/component/chunker/token_oversize_split_test.go+++ b/internal/ingestion/component/chunker/token_oversize_split_test.go@@ -16,6 +16,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -59,7 +60,7 @@ func TestTokenChunker_OversizeUnitSplit(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks missing or wrong type: %T", out["chunks"])}@@ -128,7 +129,7 @@ func TestTokenChunker_OversizeUnitSplitAfterInBudgetUnit(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks missing or wrong type: %T", out["chunks"])}diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.goindex de20ef6af2..6608565e80 100644--- a/internal/ingestion/component/chunker/token_strict_cap_test.go+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go@@ -114,7 +114,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {}tc := comp.(*TokenChunkerComponent)out := tc.mergeByTokenSize(b.String(), nil, nil)- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) < 2 {t.Fatalf("want multiple chunks, got %d", len(chunks))}@@ -139,7 +139,7 @@ func TestMergeByTokenSize_OversizedBoundarylessRunSplit(t *testing.T) {t.Fatalf("NewTokenChunker: %v", err)}out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) < 2 {t.Fatalf("oversized run must be split, got %d chunk(s)", len(chunks))}@@ -176,7 +176,7 @@ func TestInvokeTextPayload_HardCapEndToEnd(t *testing.T) {if errMsg, _ := out["_ERROR"].(string); errMsg != "" {t.Fatalf("Invoke error payload: %s", errMsg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatalf("expected chunks, got %#v", out)}@@ -216,7 +216,7 @@ func TestInvokeJSONPayload_HardCapEndToEnd(t *testing.T) {if errMsg, _ := out["_ERROR"].(string); errMsg != "" {t.Fatalf("Invoke error payload: %s", errMsg)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) < 2 {t.Fatalf("want multiple chunks, got %d", len(chunks))}diff --git a/internal/ingestion/component/chunker/token_tag_test.go b/internal/ingestion/component/chunker/token_tag_test.goindex bca95db810..ef2d58792d 100644--- a/internal/ingestion/component/chunker/token_tag_test.go+++ b/internal/ingestion/component/chunker/token_tag_test.go@@ -15,6 +15,7 @@ package chunkerimport ("context"+ "ragflow/internal/ingestion/component/schema""strings""testing")@@ -40,7 +41,7 @@ func TestTokenChunker_TextPath_StripsParserTags(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -74,7 +75,7 @@ func TestTokenChunker_JSONPath_StripsParserTags(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.goindex d69f1ec25d..2fbc33bd9e 100644--- a/internal/ingestion/component/chunker/token_test.go+++ b/internal/ingestion/component/chunker/token_test.go@@ -86,7 +86,7 @@ func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])}@@ -119,7 +119,7 @@ func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("chunks = %#v, want data segment and header-only segment", chunks)}@@ -148,7 +148,7 @@ func TestTokenChunker_InvokeDelimMode_BasicChunking(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -182,7 +182,7 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("chunks = %d, want 2: %v", len(chunks), chunks)}@@ -228,7 +228,7 @@ func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) {if got, want := out["output_format"], "chunks"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) < 1 {t.Errorf("chunks = %d, want >=1", len(chunks))}@@ -254,7 +254,7 @@ func TestTokenChunker_InvokeChildrenDelim(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -284,7 +284,7 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -340,7 +340,7 @@ func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) == 0 {t.Fatalf("chunks missing: %#v", out["chunks"])}@@ -390,7 +390,7 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks: want []map[string]any, got %T", out["chunks"])}@@ -650,7 +650,7 @@ func TestTokenChunker_InvokeDeterministic(t *testing.T) {if err != nil {t.Fatalf("Invoke run %d: %v", run, err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])fp := fingerprint{count: len(chunks)}if len(chunks) > 0 {fp.first, _ = chunks[0]["text"].(string)@@ -703,7 +703,7 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))}@@ -776,7 +776,7 @@ func TestTokenChunker_PrefersUpstreamChunks(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("chunks: want >=1, got 0")}@@ -970,7 +970,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) {c.param.ChunkTokenSize = 128c.param.OverlappedPercent = 0out := c.mergeByTokenSize(text, nil, nil)- raw, ok := out["chunks"].([]map[string]any)+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)}@@ -1015,7 +1015,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) {c.param.ChunkTokenSize = 128c.param.OverlappedPercent = 0out := c.mergeByTokenSize("A\n\n\nB", nil, nil)- raw, ok := out["chunks"].([]map[string]any)+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)}@@ -1044,7 +1044,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) {c.param.OverlappedPercent = 0text := "第一句。第二句。第三句。第四句。第五句。"out := c.mergeByTokenSize(text, nil, nil)- raw, ok := out["chunks"].([]map[string]any)+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)}@@ -1075,7 +1075,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {block := strings.Repeat("知识库检索增强生成技术", 7) // 70 charstext := block + "\n\n" + blockout := c.mergeByTokenSize(text, nil, nil)- raw, ok := out["chunks"].([]map[string]any)+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("mergeByTokenSize output missing chunks: %v", out)}diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.goindex 946983df85..4c85241678 100644--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go@@ -8,6 +8,7 @@ import ("github.com/xuri/excelize/v2"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser")@@ -59,7 +60,7 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])return chunks}diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.goindex 4e3e76aa34..3c13debdeb 100644--- a/internal/ingestion/component/extractor.go+++ b/internal/ingestion/component/extractor.go@@ -644,6 +644,8 @@ func extractorChunkList(v any) ([]map[string]any, bool) {switch list := v.(type) {case []map[string]any:return list, true+ case []schema.ChunkDoc:+ return schema.ChunkDocsToMaps(list), truecase []any:out := make([]map[string]any, 0, len(list))for _, item := range list {diff --git a/internal/ingestion/component/extractor_call_timeout_test.go b/internal/ingestion/component/extractor_call_timeout_test.goindex 2f2a2852e0..bf7cd6220f 100644--- a/internal/ingestion/component/extractor_call_timeout_test.go+++ b/internal/ingestion/component/extractor_call_timeout_test.go@@ -85,7 +85,7 @@ func TestExtractorChatTimeoutSkipsSlowChunk(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("chunks = %d, want 2", len(chunks))}@@ -222,7 +222,7 @@ func TestExtractorPartialTimeoutKeepsCompleted(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("chunks = %d, want 2", len(chunks))}diff --git a/internal/ingestion/component/extractor_envelope_test.go b/internal/ingestion/component/extractor_envelope_test.gonew file mode 100644index 0000000000..1d966e60c6--- /dev/null+++ b/internal/ingestion/component/extractor_envelope_test.go@@ -0,0 +1,41 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package component++import (+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestExtractorChunkList_AcceptsTypedChunkDocs: once the upstream chunker emits+// the typed []ChunkDoc carrier, the extractor's input reader must accept it+// (converting to the map wire form) instead of returning ok=false, which would+// silently drop every upstream chunk from the extraction context.+func TestExtractorChunkList_AcceptsTypedChunkDocs(t *testing.T) {+ got, ok := extractorChunkList([]schema.ChunkDoc{{Text: "a", ID: "1"}, {Text: "b", ID: "2"}})+ if !ok {+ t.Fatal("extractorChunkList(typed []ChunkDoc) ok=false, want true")+ }+ if len(got) != 2 || got[0]["text"] != "a" || got[1]["text"] != "b" {+ t.Fatalf("typed carrier mismatch: %+v", got)+ }++ // Legacy []map[string]any must still be accepted verbatim.+ got2, ok2 := extractorChunkList([]map[string]any{{"text": "x"}})+ if !ok2 || len(got2) != 1 || got2[0]["text"] != "x" {+ t.Fatalf("legacy []map carrier: ok=%v got=%+v", ok2, got2)+ }+}diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.goindex 7f8c231afc..e7b604c509 100644--- a/internal/ingestion/component/extractor_tag_test.go+++ b/internal/ingestion/component/extractor_tag_test.go@@ -58,7 +58,7 @@ func TestExtractorTags_NoTagFileID(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %d", len(chunks))}@@ -81,7 +81,7 @@ func TestExtractorTags_NoLLMID(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if _, ok := chunks[0][common.TAG_FLD]; ok {t.Fatal("tag_feas should not be set without tag_file_id")}@@ -111,7 +111,7 @@ func TestExtractorTags_WithKeywords(t *testing.T) {if err != nil {t.Fatal(err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if chunks[0][common.TAG_FLD] != nil {t.Fatal("tag_feas should not be set without tag_file_id")}diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.goindex cb85b202be..e3195ce08b 100644--- a/internal/ingestion/component/extractor_test.go+++ b/internal/ingestion/component/extractor_test.go@@ -150,7 +150,7 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks key missing or wrong shape: %T", out["chunks"])}@@ -186,7 +186,7 @@ func TestExtractorComponent_SkipsMediaChunksWithoutTextAndUsesContext(t *testingif calls := stub.Calls(); calls != 1 {t.Fatalf("LLM calls = %d, want one call for the context-bearing media chunk", calls)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 2 {t.Fatalf("chunks = %#v, want two input chunks", out["chunks"])}@@ -265,7 +265,7 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if s, _ := chunks[0]["summary"].(string); s != "recovered" {t.Errorf("summary = %q, want recovered", s)}@@ -306,7 +306,7 @@ func TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks missing or wrong shape")}@@ -331,7 +331,7 @@ func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("chunks malformed: %v", out["chunks"])}@@ -536,7 +536,7 @@ func TestExtractorComponent_Invoke_ToggleOffSkipsExtraction(t *testing.T) {if calls := stub.Calls(); calls != 0 {t.Errorf("LLM calls = %d, want 0 for a switched-off keywords group", calls)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if _, exists := chunks[0]["important_kwd"]; exists {t.Errorf("switched-off keywords should not enrich the chunk: %#v", chunks[0])}@@ -1191,7 +1191,7 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 2 {t.Fatalf("expected 2 chunks, got %v", out["chunks"])}@@ -1601,7 +1601,7 @@ func TestExtractorModularPromptsExecution(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -1714,7 +1714,7 @@ func TestExtractorModularMetadataExecution(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -1758,7 +1758,7 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -1813,7 +1813,7 @@ func TestExtractorCustomSummarySystemPrompt(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -1871,7 +1871,7 @@ func TestExtractorCustomKeywordsAndQuestionsSystemPrompt(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -2016,7 +2016,7 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {t.Errorf("expected 0 LLM calls when summary is disabled, got %d", stub.Calls())}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk output, got %v", out)}@@ -2166,7 +2166,7 @@ func TestExtractorBuiltInDoesNotCallLLM(t *testing.T) {if stub.calls.Load() != 0 {t.Fatalf("built_in-only must not call LLM, got %d calls, requests=%v", stub.calls.Load(), stub.requests)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %v", out)}@@ -2201,7 +2201,7 @@ func TestExtractorEnabledFalseDoesNotCallLLM(t *testing.T) {if stub.calls.Load() != 0 {t.Fatalf("enabled=false must not call LLM, got %d", stub.calls.Load())}- if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 1 {+ if chunks, _ := schema.ChunkCarrierToMaps(out["chunks"]); len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %v", out)}}@@ -2234,7 +2234,7 @@ func TestExtractor_KeywordsThenTagsSynergy(t *testing.T) {t.Fatalf("Invoke: %v", err)}- chunks, ok := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("expected 1 chunk, got %v", out)}@@ -2318,7 +2318,8 @@ func TestExtractor_CallTextCached_NoRedis_FailOpen(t *testing.T) {if calls := stub.Calls(); calls != 3 {t.Fatalf("Expected 3 LLM calls, got %d", calls)}- ck := out["chunks"].([]map[string]any)[0]+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])+ ck := chunks[0]if sum, ok := ck["summary"].(string); !ok || sum != "This is a summary without Redis." {t.Errorf("got summary %v, want 'This is a summary without Redis.'", ck["summary"])}diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.goindex c5da9b7d53..a6117b0dbb 100644--- a/internal/ingestion/component/knowledge_compiler/component.go+++ b/internal/ingestion/component/knowledge_compiler/component.go@@ -749,9 +749,11 @@ func metaStringSlice(m map[string]any, key string) []string {// headless / manual chaining reads them from the component output map, so they// must be forwarded when present.func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]any {- // Accept both the []any and []map[string]any chunk carriers (the chunker- // emits the latter; buildInputs already handles both). Without this, the- // original source chunks would be dropped when the carrier is []map[string]any.+ // Accept the []any and []map[string]any chunk carriers verbatim (the+ // legacy chunker emits the latter; buildInputs already handles both). The+ // default arm accepts the typed []ChunkDoc carrier via GetChunks, which+ // also bridges legacy forms, so the upstream chunks are never silently+ // dropped on a type mismatch.var raw []anyswitch v := inputs["chunks"].(type) {case []any:@@ -762,7 +764,13 @@ func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]araw = append(raw, m)}default:- clog.Warn("knowledge_compiler: mergeChunks: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))+ if typed, ok := schema.GetChunks(inputs); ok {+ for _, c := range typed {+ raw = append(raw, c.ToMap())+ }+ } else {+ clog.Warn("knowledge_compiler: mergeChunks: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))+ }}merged := make([]any, 0, len(raw)+len(compiled))for _, r := range raw {@@ -811,7 +819,8 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, erro// The upstream pipeline hands chunks over as a []any of map[string]any in// some paths and as a []map[string]any in others (the chunker emits the// latter). Accept both so the knowledge compiler never silently drops the- // whole upstream output on a type mismatch.+ // whole upstream output on a type mismatch. The typed []ChunkDoc carrier is+ // handled via GetChunks, which also bridges the legacy forms.var raw []map[string]anyswitch v := inputs["chunks"].(type) {case []any:@@ -826,7 +835,11 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, errocase []map[string]any:raw = vdefault:- clog.Warn("knowledge_compiler: buildInputs: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))+ if typed, ok := schema.GetChunks(inputs); ok {+ raw = schema.ChunkDocsToMaps(typed)+ } else {+ clog.Warn("knowledge_compiler: buildInputs: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))+ }}clog.Info("knowledge_compiler: buildInputs: accepted chunks from inputs[chunks]", zap.Int("chunks", len(raw)))for _, m := range raw {diff --git a/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.go b/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.gonew file mode 100644index 0000000000..2af99d947d--- /dev/null+++ b/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.go@@ -0,0 +1,76 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package knowledge_compiler++import (+ "testing"++ "ragflow/internal/ingestion/component/knowledge_compiler/common"+ "ragflow/internal/ingestion/component/schema"+)++// TestKnowledgeCompiler_BuildInputsAcceptsTypedChunkDocs: once chunkers emit+// typed []ChunkDoc through the envelope, buildInputs must accept the typed+// carrier in addition to the legacy []map[string]any / []any forms, without+// dropping every chunk on a type mismatch.+func TestKnowledgeCompiler_BuildInputsAcceptsTypedChunkDocs(t *testing.T) {+ in, err := buildInputs(map[string]any{+ "chunks": []schema.ChunkDoc{+ {ID: "c1", Text: "甲"},+ {ID: "c2", Text: "乙"},+ },+ }, common.Param{})+ if err != nil {+ t.Fatalf("buildInputs (typed []ChunkDoc): %v", err)+ }+ if len(in.Chunks) != 2 {+ t.Fatalf("buildInputs produced %d chunks, want 2 (typed []ChunkDoc carrier)", len(in.Chunks))+ }+ if in.Chunks[0].ID != "c1" || in.Chunks[0].Text != "甲" {+ t.Fatalf("chunk[0] = %+v, want id=c1 text=甲", in.Chunks[0])+ }+}++// TestKnowledgeCompiler_MergeChunksAcceptsTypedChunkDocs: mergeChunks must+// forward the upstream typed []ChunkDoc carrier (converted to the wire map+// form) ahead of the freshly compiled chunks, rather than silently dropping+// the upstream chunks on an unexpected type.+func TestKnowledgeCompiler_MergeChunksAcceptsTypedChunkDocs(t *testing.T) {+ out := mergeChunks(map[string]any{+ "chunks": []schema.ChunkDoc{{ID: "c1", Text: "甲"}},+ }, []schema.ChunkDoc{{ID: "d1", Text: "编译"}})++ merged, ok := out["chunks"].([]any)+ if !ok {+ t.Fatalf("mergeChunks output chunks type = %T, want []any", out["chunks"])+ }+ if len(merged) != 2 {+ t.Fatalf("mergeChunks produced %d chunks, want 2 (1 upstream + 1 compiled)", len(merged))+ }+ first, ok := merged[0].(map[string]any)+ if !ok {+ t.Fatalf("merged[0] type = %T, want map[string]any", merged[0])+ }+ if first["id"] != "c1" || first["text"] != "甲" {+ t.Fatalf("merged[0] = %+v, want upstream chunk id=c1 text=甲 (must not be dropped)", first)+ }+ second, ok := merged[1].(map[string]any)+ if !ok {+ t.Fatalf("merged[1] type = %T, want map[string]any", merged[1])+ }+ if second["id"] != "d1" {+ t.Fatalf("merged[1] = %+v, want compiled chunk id=d1", second)+ }+}diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex 89a2c988c3..ad631378a4 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -173,21 +173,70 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {}func (d ChunkDoc) MarshalJSON() ([]byte, error) {+ // Delegate to wireMap so the typed []ChunkDoc form and the []map[string]any+ // wire form (ToMap / ChunkDocsToMaps) serialize identically by construction.+ // Propagate marshal failures (e.g. invalid JSON in the RawMessage position+ // fields) instead of silently emitting an empty object.+ m, err := d.wireMapErr()+ if err != nil {+ return nil, err+ }+ return json.Marshal(m)+}++// wireMap is the error-tolerant wrapper used by ToMap, which cannot return an+// error. On a marshal/unmarshal failure it falls back to an empty map,+// preserving the historical ToMap behavior.+func (d ChunkDoc) wireMap() map[string]any {+ m, err := d.wireMapErr()+ if err != nil {+ return map[string]any{}+ }+ return m+}++// wireMapErr is the single source of truth for a ChunkDoc's wire representation,+// shared by ToMap and MarshalJSON. It marshals the alias (a distinct type+// without a MarshalJSON method) to avoid infinite recursion, then applies the+// canonical normalization: Extra values are decoded, "text" is always emitted+// (even for media-only chunks, so chunk identity/image upload share one wire+// contract), the stale content_with_weight key is dropped, and structured+// positions are decoded to their canonical Go form.+func (d ChunkDoc) wireMapErr() (map[string]any, error) {type alias ChunkDoc- base, err := json.Marshal(alias(d))+ b, err := json.Marshal(alias(d))if err != nil {return nil, err}- var raw map[string]json.RawMessage- if err := json.Unmarshal(base, &raw); err != nil {+ var out map[string]any+ if err := json.Unmarshal(b, &out); err != nil {return nil, err}- for k, v := range d.Extra {- if _, exists := raw[k]; !exists {- raw[k] = v+ for k, raw := range d.Extra {+ // ck_type is a retired parser key (#21259); a stray Extra entry must+ // not resurrect it through the wire form.+ if k == "ck_type" {+ continue+ }+ // Fill-missing-only: typed fields already emitted by the alias marshal+ // (id, doc_type_kwd, table, ...) take precedence over a colliding Extra+ // key. UnmarshalJSON copies the incoming "id" into Extra (its deletion+ // whitelist omits "id"), so an unconditional assignment would let a stale+ // Extra copy shadow the real typed value through the GetChunks bridge.+ if _, exists := out[k]; exists {+ continue}+ out[k] = decodeExtraValue(raw)}- return json.Marshal(raw)+ out["text"] = d.Text+ delete(out, "content_with_weight")+ if len(d.PDFPositions) > 0 {+ out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)+ }+ if len(d.Positions) > 0 {+ out["positions"] = decodeStructuredValue(d.Positions)+ }+ return out, nil}// ChunkDocFromMap decodes a free-form map into a typed ChunkDoc.@@ -284,29 +333,7 @@ func chunkDocFromDocItem(it parser.DocItem) (ChunkDoc, error) {// ToMap converts a typed chunk back to the generic map form used by// runtime.Component boundaries.func (d ChunkDoc) ToMap() map[string]any {- b, err := json.Marshal(d)- if err != nil {- return map[string]any{}- }- var out map[string]any- if err := json.Unmarshal(b, &out); err != nil {- return map[string]any{}- }- for k, raw := range d.Extra {- out[k] = decodeExtraValue(raw)- }- // Every pre-index chunk has one canonical text field. Keep the key even- // for media-only chunks so chunk identity and image upload use the same- // wire contract; contextual text remains explicit retrieval metadata.- out["text"] = d.Text- delete(out, "content_with_weight")- if len(d.PDFPositions) > 0 {- out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)- }- if len(d.Positions) > 0 {- out["positions"] = decodeStructuredValue(d.Positions)- }- return out+ return d.wireMap()}// ChunkDocsToMaps converts a typed chunk slice to the generic map form.diff --git a/internal/ingestion/component/schema/envelope.go b/internal/ingestion/component/schema/envelope.gonew file mode 100644index 0000000000..75ad651423--- /dev/null+++ b/internal/ingestion/component/schema/envelope.go@@ -0,0 +1,79 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++// Canonical envelope keys. The canvas Component.Invoke contract keeps the+// envelope as map[string]any, but each well-known key carries a single typed+// payload. The "chunks" key (chunker/tokenizer output) has a typed accessor,+// GetChunks, so downstream components read []ChunkDoc instead of bare type+// assertions, eliminating the silent type-mismatch skips that the untyped map+// form allowed. The "json" key (parser output) remains a plain wire key.+//+// These live in the schema package because it owns ChunkDoc and is imported by+// every component subpackage (component, chunker, knowledge_compiler) without+// introducing an import cycle.+const (+ KeyParseItems = "json"+ KeyChunks = "chunks"+)++// GetChunks reads the canonical "chunks" envelope key. It accepts either a+// typed []ChunkDoc or the legacy []map[string]any / []any form via+// ChunkDocsFromAny, so migration can proceed producer-by-producer without a+// flag day. A missing key, nil input, or a non-chunk carrier returns+// (nil, false). For the []map[string]any form a malformed entry rejects the+// whole carrier (nil, false) rather than being skipped; []any non-map entries+// are skipped. It never panics.+func GetChunks(in map[string]any) ([]ChunkDoc, bool) {+ if in == nil {+ return nil, false+ }+ v, ok := in[KeyChunks]+ if !ok {+ return nil, false+ }+ docs, ok, err := ChunkDocsFromAny(v)+ if err != nil || !ok {+ return nil, false+ }+ return docs, true+}++// ChunkCarrierToMaps returns a "chunks" carrier value as []map[string]any,+// bridging every form losslessly: []map[string]any is returned verbatim (no+// ChunkDoc round-trip, so non-ChunkDoc fields like content_with_weight+// survive), []ChunkDoc is converted via ChunkDocsToMaps, and []any is+// unwrapped to its map elements. Any other value, or a []any carrying a+// non-map element, returns (nil, false).+func ChunkCarrierToMaps(v any) ([]map[string]any, bool) {+ switch t := v.(type) {+ case []map[string]any:+ return t, true+ case []ChunkDoc:+ return ChunkDocsToMaps(t), true+ case []any:+ out := make([]map[string]any, 0, len(t))+ for _, it := range t {+ m, ok := it.(map[string]any)+ if !ok {+ return nil, false+ }+ out = append(out, m)+ }+ return out, true+ default:+ return nil, false+ }+}diff --git a/internal/ingestion/component/schema/envelope_parity_test.go b/internal/ingestion/component/schema/envelope_parity_test.gonew file mode 100644index 0000000000..3f1f161072--- /dev/null+++ b/internal/ingestion/component/schema/envelope_parity_test.go@@ -0,0 +1,142 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++import (+ "encoding/json"+ "reflect"+ "testing"++ "ragflow/internal/entity"+)++// TestChunksSerializationParity is the R1 regression gate: the []ChunkDoc wire+// form and the []map[string]any wire form (ChunkDocsToMaps) must serialize+// byte-semantically identically. Any+// code that json.Marshal's the envelope (e.g. the agent canvas marshalling+// node outputs) must produce JSON byte-semantically identical to the historical+// []map[string]any wire form produced by ChunkDocsToMaps. If this drifts,+// external consumers of the serialized envelope observe a shape change.+func TestChunksSerializationParity(t *testing.T) {+ cases := map[string]ChunkDoc{+ "simple text": {+ Text: "hello world",+ DocType: "text",+ ID: "id-1",+ },+ "empty text media-only": {+ Text: "",+ Image: "img/1.png",+ ImgID: "imgid-1",+ DocType: "image",+ },+ "with extra": {+ Text: "body",+ Extra: map[string]json.RawMessage{+ "custom_key": json.RawMessage(`"v"`),+ "page_number": json.RawMessage(`3`),+ },+ },+ "with positions": {+ Text: "positioned",+ PDFPositions: json.RawMessage(`[[1,2,3,4]]`),+ Positions: json.RawMessage(`[[10,20,30,40]]`),+ },+ "with table data": {+ Text: "table chunk",+ TableData: &entity.TableData{},+ },+ "token nums pointer": {+ Text: "counted",+ TKNums: intptr(42),+ },+ }+ for name, d := range cases {+ t.Run(name, func(t *testing.T) {+ typedJSON, err := json.Marshal([]ChunkDoc{d})+ if err != nil {+ t.Fatalf("marshal []ChunkDoc: %v", err)+ }+ mapJSON, err := json.Marshal(ChunkDocsToMaps([]ChunkDoc{d}))+ if err != nil {+ t.Fatalf("marshal ChunkDocsToMaps: %v", err)+ }+ assertSemanticJSONEqual(t, typedJSON, mapJSON)+ })+ }+}++// assertSemanticJSONEqual compares two JSON documents for structural equality,+// ignoring object key order (which is not semantically meaningful to consumers).+func assertSemanticJSONEqual(t *testing.T, a, b []byte) {+ t.Helper()+ var av, bv any+ if err := json.Unmarshal(a, &av); err != nil {+ t.Fatalf("unmarshal A (%s): %v", a, err)+ }+ if err := json.Unmarshal(b, &bv); err != nil {+ t.Fatalf("unmarshal B (%s): %v", b, err)+ }+ if !reflect.DeepEqual(av, bv) {+ t.Fatalf("serialization parity mismatch:\n typed []ChunkDoc = %s\n map wire form = %s", a, b)+ }+}++// TestChunkDocMarshalJSON_PropagatesError locks that a marshal failure (e.g.+// invalid JSON bytes in the RawMessage position fields) is surfaced as an+// error, not silently swallowed into an empty object. The pre-refactor+// MarshalJSON propagated the error; the wireMap refactor must preserve that,+// otherwise a corrupt chunk serializes as {} with no signal.+func TestChunkDocMarshalJSON_PropagatesError(t *testing.T) {+ cases := map[string]ChunkDoc{+ "invalid PDFPositions": {Text: "x", PDFPositions: json.RawMessage("{not json")},+ "invalid Positions": {Text: "x", Positions: json.RawMessage("[[1,2")},+ }+ for name, d := range cases {+ t.Run(name, func(t *testing.T) {+ if _, err := json.Marshal(d); err == nil {+ t.Fatalf("expected marshal error for %s, got nil (error swallowed)", name)+ }+ })+ }+}++// TestChunkDoc_RetiredCKTypeNotLeaked locks #21259's ck_type retirement+// contract through the refactored wireMap path: a ck_type that ends up in+// ChunkDoc.Extra (e.g. set programmatically) must not be re-emitted by ToMap /+// ChunkDocsToMaps, while legitimate Extra keys survive.+func TestChunkDoc_RetiredCKTypeNotLeaked(t *testing.T) {+ d := ChunkDoc{+ Text: "body",+ Extra: map[string]json.RawMessage{+ "ck_type": json.RawMessage(`"text"`), // retired parser key+ "custom": json.RawMessage(`"keep"`),+ },+ }+ m := d.ToMap()+ if _, leaked := m["ck_type"]; leaked {+ t.Fatalf("retired ck_type leaked from Extra through ToMap: %+v", m)+ }+ if m["custom"] != "keep" {+ t.Fatalf("legitimate Extra key dropped: %+v", m)+ }+ // Same guarantee through the map wire form used by the envelope bridge.+ maps := ChunkDocsToMaps([]ChunkDoc{d})+ if _, leaked := maps[0]["ck_type"]; leaked {+ t.Fatalf("retired ck_type leaked through ChunkDocsToMaps: %+v", maps[0])+ }+}++func intptr(n int) *int { return &n }diff --git a/internal/ingestion/component/schema/envelope_test.go b/internal/ingestion/component/schema/envelope_test.gonew file mode 100644index 0000000000..4a0adbd76c--- /dev/null+++ b/internal/ingestion/component/schema/envelope_test.go@@ -0,0 +1,133 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++import (+ "testing"+)++func TestPutGetChunks_RoundTrip(t *testing.T) {+ docs := []ChunkDoc{{Text: "a", ID: "1"}, {Text: "b", ID: "2"}}+ out := map[string]any{KeyChunks: docs}++ got, ok := GetChunks(out)+ if !ok {+ t.Fatal("GetChunks ok=false, want true")+ }+ if len(got) != 2 {+ t.Fatalf("len=%d, want 2", len(got))+ }+ if got[0].Text != "a" || got[1].Text != "b" {+ t.Fatalf("content mismatch: %+v", got)+ }+}++func TestGetChunks_LegacyMapBridge(t *testing.T) {+ legacy := []map[string]any{+ {"text": "a", "id": "1"},+ {"text": "b", "id": "2"},+ }+ out := map[string]any{KeyChunks: legacy}++ got, ok := GetChunks(out)+ if !ok {+ t.Fatal("GetChunks ok=false for legacy []map[string]any, want true")+ }+ if len(got) != 2 {+ t.Fatalf("len=%d, want 2", len(got))+ }+ if got[0].Text != "a" || got[1].Text != "b" {+ t.Fatalf("content mismatch: %+v", got)+ }+ if got[0].ID != "1" {+ t.Fatalf("id field lost through bridge: %+v", got[0])+ }+}++func TestGetChunks_EmptyAndNil(t *testing.T) {+ // nil input+ if _, ok := GetChunks(nil); ok {+ t.Fatal("nil input should return ok=false")+ }+ // absent key+ if _, ok := GetChunks(map[string]any{}); ok {+ t.Fatal("absent key should return ok=false")+ }+ // typed empty slice -> ok=true, len 0+ out := map[string]any{KeyChunks: []ChunkDoc{}}+ got, ok := GetChunks(out)+ if !ok {+ t.Fatal("empty typed []ChunkDoc should return ok=true")+ }+ if len(got) != 0 {+ t.Fatalf("empty typed len=%d, want 0", len(got))+ }+ // legacy empty slice -> ok=true, len 0+ out2 := map[string]any{KeyChunks: []map[string]any{}}+ got2, ok2 := GetChunks(out2)+ if !ok2 {+ t.Fatal("empty legacy []map[string]any should return ok=true")+ }+ if len(got2) != 0 {+ t.Fatalf("empty legacy len=%d, want 0", len(got2))+ }+}++func TestGetChunks_Garbage(t *testing.T) {+ // A slice of non-map values must not panic and must not yield partial+ // chunks. ChunkDocsFromAny skips non-map entries, so we accept either+ // (empty, true) or (nil, false) as safe.+ out := map[string]any{KeyChunks: []any{"not-a-map", 42}}+ got, ok := GetChunks(out)+ if ok && len(got) != 0 {+ t.Fatalf("garbage produced %d chunks, want 0", len(got))+ }+ // A scalar value for the key is not a chunk carrier.+ out2 := map[string]any{KeyChunks: 42}+ if _, ok2 := GetChunks(out2); ok2 {+ t.Fatal("scalar chunks value should return ok=false")+ }+}++func TestChunkCarrierToMaps(t *testing.T) {+ // []map[string]any is returned verbatim — no ChunkDoc round-trip, so+ // non-ChunkDoc fields (content_with_weight) survive.+ maps := []map[string]any{{"id": "c1", "content_with_weight": "X", "text": "t"}}+ got, ok := ChunkCarrierToMaps(maps)+ if !ok || len(got) != 1 {+ t.Fatalf("[]map carrier: ok=%v len=%d", ok, len(got))+ }+ if got[0]["content_with_weight"] != "X" {+ t.Fatalf("[]map carrier lost content_with_weight: %+v", got[0])+ }+ // []ChunkDoc is converted to the map wire form.+ got2, ok2 := ChunkCarrierToMaps([]ChunkDoc{{Text: "a", ID: "1"}})+ if !ok2 || len(got2) != 1 || got2[0]["text"] != "a" {+ t.Fatalf("[]ChunkDoc carrier: ok=%v got=%+v", ok2, got2)+ }+ // []any of maps is unwrapped.+ got3, ok3 := ChunkCarrierToMaps([]any{map[string]any{"id": "z"}})+ if !ok3 || len(got3) != 1 || got3[0]["id"] != "z" {+ t.Fatalf("[]any carrier: ok=%v got=%+v", ok3, got3)+ }+ // []any containing a non-map element is rejected wholesale (matches toChunkMaps).+ if _, ok4 := ChunkCarrierToMaps([]any{"not-a-map"}); ok4 {+ t.Fatal("[]any with non-map element should be ok=false")+ }+ // A scalar / unrelated value is not a chunk carrier.+ if _, ok5 := ChunkCarrierToMaps(42); ok5 {+ t.Fatal("scalar value should be ok=false")+ }+}diff --git a/internal/ingestion/component/schema/wiremap_extra_precedence_test.go b/internal/ingestion/component/schema/wiremap_extra_precedence_test.gonew file mode 100644index 0000000000..087412f8d9--- /dev/null+++ b/internal/ingestion/component/schema/wiremap_extra_precedence_test.go@@ -0,0 +1,124 @@+// Copyright 2025 Infiniflow, Inc. All rights reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++import (+ "encoding/json"+ "testing"+)++// R1: a typed field (id) must not be clobbered by a colliding Extra key.+// Regression for yuzhichang's P1 review on PR #21273: wireMapErr previously+// overwrote typed fields unconditionally, so a ChunkDoc whose Extra carried a+// colliding "id" (which ChunkDocFromMap/UnmarshalJSON populate from legacy+// []map[string]any input) emitted the wrong id into indexing.+func TestWireMap_PreservesTypedIDOverExtra(t *testing.T) {+ d := ChunkDoc{+ ID: "real-id",+ Text: "body",+ Extra: map[string]json.RawMessage{"id": json.RawMessage(`"stale-id"`)},+ }+ m := d.ToMap()+ if got := m["id"]; got != "real-id" {+ t.Fatalf("ToMap id = %v, want %q (typed field must win over Extra)", got, "real-id")+ }+}++// R3 (MarshalJSON path): MarshalJSON delegates to wireMapErr, so the same+// precedence must hold on the JSON layer.+func TestMarshalJSON_PreservesTypedIDOverExtra(t *testing.T) {+ d := ChunkDoc{+ ID: "real-id",+ Text: "body",+ Extra: map[string]json.RawMessage{"id": json.RawMessage(`"stale-id"`)},+ }+ b, err := json.Marshal(d)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ var out map[string]any+ if err := json.Unmarshal(b, &out); err != nil {+ t.Fatalf("unmarshal result: %v", err)+ }+ if got := out["id"]; got != "real-id" {+ t.Fatalf("marshaled id = %v, want %q", got, "real-id")+ }+}++// R2: fill-missing-only must still ADD Extra keys that are not typed fields.+func TestWireMap_AddsNonTypedExtraKeys(t *testing.T) {+ d := ChunkDoc{+ ID: "real-id",+ Text: "body",+ Extra: map[string]json.RawMessage{"custom_key": json.RawMessage(`"v"`)},+ }+ m := d.ToMap()+ if got := m["custom_key"]; got != "v" {+ t.Fatalf("custom Extra key = %v, want %q (non-typed Extra keys must survive)", got, "v")+ }+ if got := m["id"]; got != "real-id" {+ t.Fatalf("id = %v, want %q", got, "real-id")+ }+}++// R4/R5 corner: empty and nil Extra must not panic and must still emit typed+// fields.+func TestWireMap_EmptyAndNilExtra(t *testing.T) {+ d1 := ChunkDoc{ID: "a", Text: "t", Extra: map[string]json.RawMessage{}}+ if got := d1.ToMap()["id"]; got != "a" {+ t.Fatalf("empty Extra id = %v, want a", got)+ }+ d2 := ChunkDoc{ID: "b", Text: "t"}+ if got := d2.ToMap()["id"]; got != "b" {+ t.Fatalf("nil Extra id = %v, want b", got)+ }+}++// R4: ck_type in Extra must still be skipped (unrelated to the precedence fix,+// locked so the change does not resurrect the retired key).+func TestWireMap_SkipsRetiredCKType(t *testing.T) {+ d := ChunkDoc{+ ID: "a",+ Text: "t",+ Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"text"`)},+ }+ m := d.ToMap()+ if _, ok := m["ck_type"]; ok {+ t.Fatalf("ck_type leaked: %+v", m)+ }+}++// R-bridge (reviewer's exact scenario): a legacy []map[string]any chunk whose+// id is later mutated must keep the mutated id through the GetChunks -> ToMap+// bridge, not the stale Extra copy that UnmarshalJSON populated. UnmarshalJSON's+// deletion whitelist omits "id", so decoding a legacy map copies id into both+// ChunkDoc.ID and ChunkDoc.Extra["id"]; a subsequent re-id step that updates+// only the typed field must not be shadowed by the stale Extra copy.+func TestGetChunksBridge_PreservesMutatedID(t *testing.T) {+ legacy := []map[string]any{{"id": "original-id", "text": "body"}}+ out := map[string]any{KeyChunks: legacy}++ typed, ok := GetChunks(out)+ if !ok || len(typed) != 1 {+ t.Fatalf("GetChunks ok=%v len=%d, want ok=true len=1", ok, len(typed))+ }+ // Simulate a re-id step that updates the typed field but leaves Extra stale.+ typed[0].ID = "reassigned-id"++ maps := ChunkDocsToMaps(typed)+ if got := maps[0]["id"]; got != "reassigned-id" {+ t.Fatalf("bridged id = %v, want %q (mutated typed id must survive the Extra copy)", got, "reassigned-id")+ }+}diff --git a/internal/ingestion/component/slice3_integration_test.go b/internal/ingestion/component/slice3_integration_test.goindex 0a6d546c45..820485314d 100644--- a/internal/ingestion/component/slice3_integration_test.go+++ b/internal/ingestion/component/slice3_integration_test.go@@ -22,6 +22,7 @@ package componentimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -44,7 +45,7 @@ func TestTokenizer_DropsContentWithWeightOnlyChunks(t *testing.T) {if err != nil {t.Fatalf("Tokenizer.Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 0 {t.Fatalf("content_with_weight-only chunks must be dropped, got %d chunk(s)", len(chunks))}@@ -70,7 +71,7 @@ func TestTokenizer_DoesNotChangeChunkText(t *testing.T) {if err != nil {t.Fatalf("Tokenizer.Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) == 0 {t.Fatal("no chunks emitted")}diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.goindex 7eee084374..4ec5efd757 100644--- a/internal/ingestion/component/tokenizer_nonpersist_test.go+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go@@ -24,6 +24,7 @@ package componentimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -77,7 +78,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {if got := debugOut["embedding_token_consumption"]; got != nil {t.Errorf("embedding_token_consumption = %v under debug; must be omitted when embedding is skipped", got)}- got, ok := debugOut["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(debugOut["chunks"])if !ok || len(got) != 2 {t.Fatalf("chunks malformed under debug: %v", debugOut["chunks"])}@@ -96,7 +97,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {if got := kbOut["embedding_token_consumption"]; got == nil {t.Error("embedding_token_consumption missing with kb_id set; embedding accounting must run")}- kbChunks, ok := kbOut["chunks"].([]map[string]any)+ kbChunks, ok := schema.ChunkCarrierToMaps(kbOut["chunks"])if !ok || len(kbChunks) != 2 {t.Fatalf("chunks malformed with kb_id: %v", kbOut["chunks"])}diff --git a/internal/ingestion/component/tokenizer_table_tag_test.go b/internal/ingestion/component/tokenizer_table_tag_test.goindex f9201c4dda..5652ab7983 100644--- a/internal/ingestion/component/tokenizer_table_tag_test.go+++ b/internal/ingestion/component/tokenizer_table_tag_test.go@@ -20,6 +20,7 @@ package componentimport ("context"+ "ragflow/internal/ingestion/component/schema""testing")@@ -50,7 +51,7 @@ func TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk(t *testing.T) {if len(stub.callInputs) != 2 || len(stub.callInputs[1]) != 1 || stub.callInputs[1][0] != "real text" {t.Fatalf("content call inputs = %v, want [real text]", stub.callInputs)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("output chunks = %d, want 2 (a skipped chunk stays in the payload)", len(chunks))}diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.goindex bc4861bf89..01dd954a3d 100644--- a/internal/ingestion/component/tokenizer_test.go+++ b/internal/ingestion/component/tokenizer_test.go@@ -76,7 +76,7 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {t.Fatalf("Invoke: %v", err)}- gotChunks, ok := out["chunks"].([]map[string]any)+ gotChunks, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok {t.Fatalf("chunks type = %T, want []map[string]any", out["chunks"])}@@ -142,7 +142,7 @@ func TestTokenizerComponent_Invoke_Unicode(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- gotChunks, _ := out["chunks"].([]map[string]any)+ gotChunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(gotChunks) != len(inputs) {t.Fatalf("chunks len = %d, want %d", len(gotChunks), len(inputs))}@@ -172,7 +172,7 @@ func TestTokenizerComponent_Invoke_TextPayload(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) != 1 {t.Fatalf("chunks len = %d, want 1", len(got))}@@ -198,7 +198,7 @@ func TestTokenizerComponent_Invoke_JSONPayload(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) != 2 {t.Fatalf("chunks len = %d, want 2", len(got))}@@ -253,7 +253,7 @@ func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {if out["embedding_token_consumption"] != nil {t.Errorf("embedding_token_consumption should be absent, got %v", out["embedding_token_consumption"])}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) == 0 || got[0]["content_ltks"] == nil {t.Errorf("content_ltks missing: %v", got)}@@ -282,7 +282,7 @@ func TestTokenizerComponent_Invoke_KeywordSplitCommaOnly(t *testing.T) {if err != nil {t.Fatalf("Invoke(%q): %v", keywords, err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) != 1 {t.Fatalf("chunks len = %d, want 1", len(got))}@@ -325,7 +325,7 @@ func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) != 1 {t.Fatalf("chunks len = %d, want 1", len(got))}@@ -472,7 +472,7 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {t.Errorf("elapsed %v exceeds 5s ceiling", elapsed)}- got, ok := out["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(got) != 1 {t.Fatalf("chunks output malformed: %v", out["chunks"])}@@ -513,7 +513,7 @@ func TestTokenizerComponent_Embedding_MergesTitleAndContentVectors(t *testing.T)if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])want0 := []float64{1.9, 3.8}want1 := []float64{3.7, 5.6}if !floatSliceClose(got[0]["q_2_vec"].([]float64), want0) {@@ -549,7 +549,7 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])want := []float64{3.5, 3.5}if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)@@ -583,7 +583,7 @@ func TestTokenizerComponent_Embedding_EmptyNameWarnsAndUsesContentVector(t *testif got := logs.FilterMessageSnippet("empty name provided from upstream").Len(); got != 1 {t.Fatalf("empty-name warnings = %d, want 1 (entries: %v)", got, logs.All())}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])want := []float64{2, 4}if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)@@ -665,7 +665,7 @@ func TestTokenizerComponent_Embedding_SkipsEmptyCleanedTextsButReturnsZeroWhenAlif got := out["embedding_token_consumption"]; got != 0 {t.Fatalf("embedding_token_consumption = %v, want 0", got)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])for i, ck := range got {if _, ok := ck["q_2_vec"]; ok {t.Fatalf("chunk[%d] should not have vector: %v", i, ck)@@ -762,8 +762,16 @@ func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testinif err != nil {t.Fatalf("Invoke B: %v", err)}- vecA := outA["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)- vecB := outB["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)+ chunksA, okA := schema.ChunkCarrierToMaps(outA["chunks"])+ if !okA || len(chunksA) == 0 {+ t.Fatalf("A: no chunks emitted: %v", outA["chunks"])+ }+ vecA, _ := chunksA[0]["q_2_vec"].([]float64)+ chunksB, okB := schema.ChunkCarrierToMaps(outB["chunks"])+ if !okB || len(chunksB) == 0 {+ t.Fatalf("B: no chunks emitted: %v", outB["chunks"])+ }+ vecB, _ := chunksB[0]["q_2_vec"].([]float64)if reflect.DeepEqual(vecA, vecB) {t.Fatalf("instance resolvers leaked: vecA=%v vecB=%v", vecA, vecB)}diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.goindex 2db6eeb52b..233e241753 100644--- a/internal/ingestion/component/tokenizer_unit_test.go+++ b/internal/ingestion/component/tokenizer_unit_test.go@@ -174,7 +174,10 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])+ if !ok {+ t.Fatalf("chunks carrier missing or invalid: %v", out["chunks"])+ }if len(chunks) != 0 {t.Errorf("chunks len = %d, want 0", len(chunks))}@@ -201,7 +204,10 @@ func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks, _ := out["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])+ if !ok {+ t.Fatalf("chunks carrier missing or invalid: %v", out["chunks"])+ }if len(chunks) != 0 {t.Errorf("chunks len = %d, want 0", len(chunks))}@@ -225,7 +231,7 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, _ := out["chunks"].([]map[string]any)+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(got) != 1 {t.Fatalf("chunks len = %d, want 1", len(got))}@@ -264,7 +270,7 @@ func TestTokenizerComponent_FullTextIncludesMediaContext(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if got := chunks[0]["content_ltks"]; got != "above table body below" {t.Fatalf("content_ltks = %q, want contextual text", got)}@@ -555,7 +561,7 @@ func TestChunkOrderInt_EmbeddingOnly(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("want 2 chunks, got %d", len(chunks))}@@ -598,7 +604,7 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("want 1 chunk (5 phantoms filtered), got %d: %#v", len(chunks), chunks)}@@ -628,7 +634,7 @@ func TestTokenizerComponent_KeepsContextOnlyMediaChunk(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 1 {t.Fatalf("chunks = %#v, want context-bearing media chunk retained", chunks)}@@ -689,7 +695,7 @@ func TestChunkOrderInt_AllPathsUnconditional(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != tc.wantChunks {t.Fatalf("want %d chunks, got %d: %#v", tc.wantChunks, len(chunks), chunks)}@@ -742,7 +748,7 @@ func TestChunkOrderInt_A5FilterKeepsSequenceContiguous(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- chunks := out["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])if len(chunks) != 2 {t.Fatalf("want 2 surviving chunks, got %d: %#v", len(chunks), chunks)}@@ -790,7 +796,7 @@ func TestTokenizerComponent_ImportantKwd_CommaOnly(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, ok := out["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(got) != 1 {t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])}@@ -843,7 +849,7 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, ok := out["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(got) != 1 {t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])}@@ -874,7 +880,7 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {if err != nil {t.Fatalf("Invoke(empty keywords): %v", err)}- gotEmpty, ok := outEmpty["chunks"].([]map[string]any)+ gotEmpty, ok := schema.ChunkCarrierToMaps(outEmpty["chunks"])if !ok || len(gotEmpty) != 1 {t.Fatalf("empty chunks = %v, want 1 chunk", outEmpty["chunks"])}@@ -958,7 +964,7 @@ func TestTokenizerComponent_ImportantKwd_Bounded(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- got, ok := out["chunks"].([]map[string]any)+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])if !ok || len(got) != 1 {t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])}diff --git a/internal/ingestion/pipeline/image_ocr_test.go b/internal/ingestion/pipeline/image_ocr_test.goindex 351cab397e..42856485a1 100644--- a/internal/ingestion/pipeline/image_ocr_test.go+++ b/internal/ingestion/pipeline/image_ocr_test.go@@ -36,6 +36,7 @@ import ("ragflow/internal/entity""ragflow/internal/ingestion/component""ragflow/internal/ingestion/component/chunker"+ "ragflow/internal/ingestion/component/schema")type imageOCREmbedder struct{ texts []string }@@ -193,7 +194,7 @@ func TestImageOCRSavedConfigToSearchableChunk(t *testing.T) {if err != nil {t.Fatal(err)}- chunks := indexable["chunks"].([]map[string]any)+ chunks, _ := schema.ChunkCarrierToMaps(indexable["chunks"])if len(chunks) != 1 || chunks[0]["text"] != text {t.Fatalf("searchable OCR text was lost: %+v", chunks)}diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.goindex 0881c549e2..3d5744d0d8 100644--- a/internal/ingestion/pipeline/real_storage_integration_test.go+++ b/internal/ingestion/pipeline/real_storage_integration_test.go@@ -23,6 +23,7 @@ import ("ragflow/internal/entity"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser""ragflow/internal/server""ragflow/internal/server/config"@@ -118,7 +119,7 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -187,7 +188,7 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {if got := chunkerState["output_format"]; got != "chunks" {t.Fatalf("chunker output_format = %v, want chunks", got)}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.goindex 31cd18920d..77f34d9798 100644--- a/internal/ingestion/pipeline/template_integration_test.go+++ b/internal/ingestion/pipeline/template_integration_test.go@@ -40,6 +40,7 @@ import (doctype "ragflow/internal/deepdoc/parser/type"componentpkg "ragflow/internal/ingestion/component"_ "ragflow/internal/ingestion/component/chunker"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/ingestion/testutil""ragflow/internal/parser/parser""ragflow/internal/storage"@@ -103,7 +104,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -141,7 +142,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {if got := chunkerState["output_format"]; got != "chunks" {t.Fatalf("chunker output_format = %v, want chunks", got)}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}@@ -230,7 +231,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {if got := chunkerState["output_format"]; got != "chunks" {t.Fatalf("chunker output_format = %v, want chunks", got)}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != 1 {t.Fatalf("chunker chunks = %T/%v, want 1 item", chunkerState["chunks"], chunkerState["chunks"])}@@ -277,7 +278,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok || len(chunks) != 1 {t.Fatalf("chunks = %T/%v, want 1 merged chunk", payload["chunks"], payload["chunks"])}@@ -329,7 +330,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)if !ok {t.Fatal("missing OneChunker:DryDrinksVisit state")}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != 1 {t.Fatalf("chunker chunks = %T/%v, want 1 item", chunkerState["chunks"], chunkerState["chunks"])}@@ -376,7 +377,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -425,7 +426,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {if !ok {t.Fatal("missing ManualChunker:NineInsectsFind state")}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}@@ -474,7 +475,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -508,7 +509,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {if !ok {t.Fatal("missing TitleChunker:SpicyKeysKick state")}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}@@ -557,7 +558,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -589,7 +590,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {if !ok {t.Fatal("missing TitleChunker:SparklySchoolsTravel state")}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}@@ -638,7 +639,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}@@ -675,7 +676,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {if !ok {t.Fatal("missing TitleChunker:GrumpyGarlicsBake state")}- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])if !ok || len(chunkerChunks) != len(wantChunkTexts) {t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))}@@ -741,7 +742,7 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok || len(chunks) == 0 {t.Fatalf("chunks = %T/%v, want non-empty []map[string]any", payload["chunks"], payload["chunks"])}@@ -1020,7 +1021,7 @@ func assertTokenizerTerminalChunk(t *testing.T, payload map[string]any, name, waif got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}diff --git a/internal/ingestion/task/indexdoc/normalize.go b/internal/ingestion/task/indexdoc/normalize.goindex 0395d1b553..aee0b78ed7 100644--- a/internal/ingestion/task/indexdoc/normalize.go+++ b/internal/ingestion/task/indexdoc/normalize.go@@ -20,6 +20,7 @@ import ("fmt""ragflow/internal/common"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser")@@ -33,6 +34,15 @@ func NormalizeChunks(output map[string]any) []map[string]any {if chunks, ok := output["chunks"].([]map[string]any); ok {return DeepCopyChunks(chunks)}+ // Typed []ChunkDoc carrier (post-migration chunker/tokenizer output):+ // normalize it to the map wire form. Match the concrete typed slice only —+ // do NOT use schema.GetChunks here, because it also accepts the legacy+ // []any carrier and would round-trip it through ChunkDoc, dropping+ // non-ChunkDoc fields (e.g. content_with_weight). The legacy []any carrier+ // must stay on the value-preserving toChunkMaps path below.+ if typed, ok := output["chunks"].([]schema.ChunkDoc); ok {+ return DeepCopyChunks(schema.ChunkDocsToMaps(typed))+ }if chunks, ok := toChunkMaps(output["chunks"]); ok {return DeepCopyChunks(chunks)}diff --git a/internal/ingestion/task/indexdoc/normalize_test.go b/internal/ingestion/task/indexdoc/normalize_test.goindex 8555f2807b..b1ecfcfa5b 100644--- a/internal/ingestion/task/indexdoc/normalize_test.go+++ b/internal/ingestion/task/indexdoc/normalize_test.go@@ -3,6 +3,7 @@ package indexdocimport ("testing"+ "ragflow/internal/ingestion/component/schema""ragflow/internal/parser/parser")@@ -26,6 +27,54 @@ func TestNormalizeChunks_ChunksFormat(t *testing.T) {}}+// TestNormalizeChunks_TypedChunkDocCarrier: once a chunker/tokenizer emits the+// typed []ChunkDoc carrier, NormalizeChunks must bridge it to the map wire form+// instead of silently dropping every chunk (which would yield zero index+// writes).+func TestNormalizeChunks_TypedChunkDocCarrier(t *testing.T) {+ input := map[string]any{+ "chunks": []schema.ChunkDoc{+ {Text: "typed-a", ID: "1"},+ {Text: "typed-b", ID: "2"},+ },+ }+ result := NormalizeChunks(input)+ if len(result) != 2 {+ t.Fatalf("len = %d, want 2 (typed []ChunkDoc carrier)", len(result))+ }+ if result[0]["text"] != "typed-a" {+ t.Errorf("result[0][\"text\"] = %q, want \"typed-a\"", result[0]["text"])+ }+}++// TestNormalizeChunks_PreservesAnyCarrierFields locks the legacy []any-of-map+// carrier path: such a carrier must be preserved verbatim (matching the+// historical toChunkMaps behavior), NOT round-tripped through ChunkDoc. The+// ChunkDoc round-trip drops non-ChunkDoc fields (e.g. content_with_weight) and+// forces an empty "text" key, which the index path would then persist as an+// empty content_with_weight.+func TestNormalizeChunks_PreservesAnyCarrierFields(t *testing.T) {+ input := map[string]any{+ "chunks": []any{+ map[string]any{"id": "c1", "content_with_weight": "hello world"},+ map[string]any{"id": "c2", "text": "body", "content_with_weight": "rich text"},+ },+ }+ result := NormalizeChunks(input)+ if len(result) != 2 {+ t.Fatalf("len = %d, want 2", len(result))+ }+ if result[0]["content_with_weight"] != "hello world" {+ t.Errorf("result[0] content_with_weight lost: %+v", result[0])+ }+ if _, hasText := result[0]["text"]; hasText {+ t.Errorf("result[0] must not gain a spurious empty text key: %+v", result[0])+ }+ if result[1]["text"] != "body" || result[1]["content_with_weight"] != "rich text" {+ t.Errorf("result[1] fields lost: %+v", result[1])+ }+}+func TestNormalizeChunks_JSONFormat(t *testing.T) {input := map[string]any{"json": []map[string]any{diff --git a/internal/ingestion/task/pipeline_run_test.go b/internal/ingestion/task/pipeline_run_test.goindex 41c47ac7ae..1df0e8278b 100644--- a/internal/ingestion/task/pipeline_run_test.go+++ b/internal/ingestion/task/pipeline_run_test.go@@ -23,6 +23,7 @@ import ("ragflow/internal/dao""ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema"pipelinepkg "ragflow/internal/ingestion/pipeline")@@ -72,7 +73,7 @@ func TestExtractPipelinePayload_UnwrapsSingleTerminalOutput(t *testing.T) {if got := payload["output_format"]; got != "chunks" {t.Fatalf("output_format = %v, want chunks", got)}- chunks, ok := payload["chunks"].([]map[string]any)+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])if !ok {t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])}