Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Add typed envelope accessors for ingestion component boundaries (#21273)

xugangqiang31296803d4da
diff --git a/internal/ingestion/component/chunker/bare_delimiter_test.go b/internal/ingestion/component/chunker/bare_delimiter_test.go
index 765c5aa7ca..c08bcab6a2 100644
--- a/internal/ingestion/component/chunker/bare_delimiter_test.go
+++ b/internal/ingestion/component/chunker/bare_delimiter_test.go
@@ -2,6 +2,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -50,7 +51,7 @@ func invokeText(t *testing.T, c *TokenChunkerComponent, text string) []map[strin
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
t.Fatalf("Go returned _ERROR: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
@@ -122,7 +123,7 @@ func TestBareDelimiterSplitsJSON(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if got := joinedText(chunks); got != text {
t.Errorf("joined text: want %q got %q (chunks=%v)", text, got, chunkTexts(chunks))
}
diff --git a/internal/ingestion/component/chunker/custom_delim_test.go b/internal/ingestion/component/chunker/custom_delim_test.go
index 78efdacd4e..3d75f0859c 100644
--- a/internal/ingestion/component/chunker/custom_delim_test.go
+++ b/internal/ingestion/component/chunker/custom_delim_test.go
@@ -2,6 +2,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"slices"
"testing"
)
@@ -31,7 +32,7 @@ func invokeTokenChunks(t *testing.T, params, input map[string]any) []map[string]
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
t.Fatalf("Go returned _ERROR: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
diff --git a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
index 459776305a..c7fe552049 100644
--- a/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
+++ b/internal/ingestion/component/chunker/delimiter_case_sensitive_test.go
@@ -33,6 +33,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/chunk"
)
@@ -185,7 +186,7 @@ func TestTokenChunker_BacktickEndSplitsOnlyAtLowercase(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
got := make([]string, 0, len(chunks))
for _, ck := range chunks {
text, _ := ck["text"].(string)
@@ -225,7 +226,7 @@ func TestTokenChunker_BacktickASplitsOnlyAtLowercase(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
got := make([]string, 0, len(chunks))
for _, ck := range chunks {
text, _ := ck["text"].(string)
diff --git a/internal/ingestion/component/chunker/general_merge_test.go b/internal/ingestion/component/chunker/general_merge_test.go
index 20e9924574..0ba17f858f 100644
--- a/internal/ingestion/component/chunker/general_merge_test.go
+++ b/internal/ingestion/component/chunker/general_merge_test.go
@@ -487,7 +487,7 @@ func generalTextInput(text string) map[string]any {
func outputChunks(t *testing.T, out map[string]any) []map[string]any {
t.Helper()
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/general_test.go b/internal/ingestion/component/chunker/general_test.go
index 093e99f5cf..a037cd72fb 100644
--- a/internal/ingestion/component/chunker/general_test.go
+++ b/internal/ingestion/component/chunker/general_test.go
@@ -299,7 +299,7 @@ func TestGeneralChunkerPreservesSingleParserUnit(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("chunks = %#v, want one chunk", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/golden_parity_test.go b/internal/ingestion/component/chunker/golden_parity_test.go
index 2b982fdf2b..73c57fe35e 100644
--- a/internal/ingestion/component/chunker/golden_parity_test.go
+++ b/internal/ingestion/component/chunker/golden_parity_test.go
@@ -46,6 +46,7 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
"github.com/google/go-cmp/cmp"
)
@@ -230,7 +231,7 @@ func invokeChunker(t *testing.T, tc parityCase) []map[string]any {
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
t.Fatalf("Go returned _ERROR while Python succeeded: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go
index 1da22158b6..d9fec8f499 100644
--- a/internal/ingestion/component/chunker/group_test.go
+++ b/internal/ingestion/component/chunker/group_test.go
@@ -191,7 +191,7 @@ func TestTitleFamilyChunkersRecognizeHeadingsAfterFirstParserLine(t *testing.T)
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("got %d chunks, want one per heading section", len(chunks))
}
@@ -216,7 +216,7 @@ func TestGroupTitleChunker_InvokeEmptyInput(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Errorf("chunks = %d, want 0", len(chunks))
}
@@ -240,7 +240,7 @@ func TestGroupTitleChunker_Headings_ASCII(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -263,7 +263,7 @@ func TestGroupTitleChunker_RootChunkAsHeading_StillSingleGroup(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -311,7 +311,7 @@ func TestGroupChunker_StructuredMetadata(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
found := false
for _, ck := range chunks {
if dt, _ := ck["doc_type_kwd"].(string); dt == "image" {
@@ -352,7 +352,7 @@ func TestGroupChunker_MergesPDFPositionsAndRemovesTags(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("no chunks emitted")
}
@@ -400,7 +400,7 @@ func TestGroupTitleChunker_InvokeDeterministic(t *testing.T) {
if err != nil {
t.Fatalf("Invoke run %d: %v", run, err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
texts := make([]string, len(chunks))
for i, ck := range chunks {
texts[i], _ = ck["text"].(string)
diff --git a/internal/ingestion/component/chunker/hierarchy_test.go b/internal/ingestion/component/chunker/hierarchy_test.go
index c421eb1f82..158628e1be 100644
--- a/internal/ingestion/component/chunker/hierarchy_test.go
+++ b/internal/ingestion/component/chunker/hierarchy_test.go
@@ -21,6 +21,7 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
)
func TestHierarchyTitleChunker_Registered(t *testing.T) {
@@ -87,7 +88,7 @@ func TestHierarchyTitleChunker_InvokeEmptyInput(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Errorf("chunks = %d, want 0", len(chunks))
}
@@ -110,7 +111,7 @@ func TestHierarchyTitleChunker_NestedHeadings(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -165,7 +166,7 @@ func TestHierarchyChunker_StructuredMetadata(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
found := false
for _, ck := range chunks {
if dt, _ := ck["doc_type_kwd"].(string); dt == "image" {
@@ -199,7 +200,7 @@ func TestHierarchyTitleChunker_InvokeDeterministic(t *testing.T) {
if err != nil {
t.Fatalf("Invoke run %d: %v", run, err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
texts := make([]string, len(chunks))
for i, ck := range chunks {
texts[i], _ = ck["text"].(string)
@@ -252,7 +253,7 @@ func TestHierarchyTitleChunker_ColonTitlePromotion(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -284,7 +285,7 @@ func TestHierarchyTitleChunker_ColonTitleShortLine_Negative(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -319,7 +320,7 @@ func TestHierarchyTitleChunker_ShortNumericLineFilter(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -395,7 +396,7 @@ func TestHierarchyTitleChunker_ColonTitlePromotion_CJK_EdgeCase(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -433,7 +434,7 @@ func TestHierarchyTitleChunker_ShortSingleCJKLineFilter(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -472,7 +473,7 @@ func TestHierarchyTitleChunker_LayoutHeadingFallback(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -513,7 +514,7 @@ func TestHierarchyTitleChunker_ConsecutiveNonTextOrder(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
idx := func(sub string) int {
for i, ck := range chunks {
if text, _ := ck["text"].(string); strings.Contains(text, sub) {
diff --git a/internal/ingestion/component/chunker/image_upload_test.go b/internal/ingestion/component/chunker/image_upload_test.go
index 747081d579..ef60ba1154 100644
--- a/internal/ingestion/component/chunker/image_upload_test.go
+++ b/internal/ingestion/component/chunker/image_upload_test.go
@@ -29,6 +29,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/ingestion/component/schema"
parserlib "ragflow/internal/parser/parser"
"gorm.io/gorm"
@@ -295,7 +296,7 @@ func TestImageUploadDecorator_EndToEnd(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatalf("no chunks in output")
}
@@ -372,7 +373,7 @@ func TestImageUploadDecorator_DebugSkipsUpload(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatalf("expected chunks in output, got %#v", out)
}
@@ -426,7 +427,7 @@ func TestImageUploadDecorator_DebugCapsChunks(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks missing or wrong type: %#v", out["chunks"])
}
@@ -463,7 +464,7 @@ func TestImageUploadDecorator_NoCapKeepsAll(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 5 {
t.Fatalf("len(chunks) = %d, want 5 (no cap should keep all)", len(chunks))
}
diff --git a/internal/ingestion/component/chunker/json_global_merge_test.go b/internal/ingestion/component/chunker/json_global_merge_test.go
index 0e2d619fc9..3bcd7d74de 100644
--- a/internal/ingestion/component/chunker/json_global_merge_test.go
+++ b/internal/ingestion/component/chunker/json_global_merge_test.go
@@ -31,6 +31,7 @@ package chunker
// boundary like Python's _merge_text_chunks_by_token_size.
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -66,7 +67,7 @@ func TestJSONGlobalMergeMatchesPython(t *testing.T) {
if msg, _ := out["_ERROR"].(string); msg != "" {
t.Fatalf("TokenChunker returned _ERROR: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("want 1 chunk (global merge across items), got %d", len(chunks))
}
@@ -119,7 +120,7 @@ func TestJSONSingleItemSubSplitUnderHardCap(t *testing.T) {
if msg, _ := out["_ERROR"].(string); msg != "" {
t.Fatalf("TokenChunker returned _ERROR: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) < 2 {
t.Fatalf("over-budget item must be split, got %d chunk(s)", len(chunks))
}
diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go
index e896728fa1..097d366d48 100644
--- a/internal/ingestion/component/chunker/manual_test.go
+++ b/internal/ingestion/component/chunker/manual_test.go
@@ -23,6 +23,7 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
)
// manualChunkerInput is a small builder for a structured (output_format=chunks)
@@ -76,7 +77,7 @@ func mustManual(t *testing.T, params map[string]any) *ManualChunkerComponent {
// manualChunkTexts pulls the ordered chunk texts out of a chunker output map.
func manualChunkTexts(out map[string]any) []string {
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
out2 := make([]string, 0, len(chunks))
for _, ck := range chunks {
if s, _ := ck["text"].(string); s != "" {
@@ -118,7 +119,7 @@ func TestManualChunker_Empty(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Errorf("chunks = %d, want 0", len(chunks))
}
@@ -155,8 +156,8 @@ func TestManualChunker_NoPositionsEqualsGroupChunker(t *testing.T) {
t.Fatalf("manual Invoke: %v", err)
}
- gChunks, _ := gOut["chunks"].([]map[string]any)
- mChunks, _ := mOut["chunks"].([]map[string]any)
+ gChunks, _ := schema.ChunkCarrierToMaps(gOut["chunks"])
+ mChunks, _ := schema.ChunkCarrierToMaps(mOut["chunks"])
if !reflect.DeepEqual(gChunks, mChunks) {
t.Fatalf("ManualChunker output diverges from GroupTitleChunker on no-coordinate input\n group: %#v\n manual: %#v", gChunks, mChunks)
}
@@ -253,7 +254,7 @@ func TestManualChunker_MergesPositions(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
found := false
for _, ck := range chunks {
text, _ := ck["text"].(string)
diff --git a/internal/ingestion/component/chunker/one_tag_test.go b/internal/ingestion/component/chunker/one_tag_test.go
index 123ea8fad9..5e8306c2fc 100644
--- a/internal/ingestion/component/chunker/one_tag_test.go
+++ b/internal/ingestion/component/chunker/one_tag_test.go
@@ -32,7 +32,7 @@ func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {
{Text: "Sentence two@@1\t2\t3\t4##"},
}
out := emitOneFromItems(items, nil)
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("want 1 chunk, got %d", len(chunks))
}
@@ -49,7 +49,7 @@ func TestOneChunkerStripsCoordTagsJSON(t *testing.T) {
// path also strips coordinate tags before emitting.
func TestOneChunkerStripsCoordTagsText(t *testing.T) {
out := emitOne("Hello world@@1\t2\t3\t4##", "text")
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("want 1 chunk, got %d", len(chunks))
}
diff --git a/internal/ingestion/component/chunker/one_test.go b/internal/ingestion/component/chunker/one_test.go
index f96adf3db5..6aa0d9c746 100644
--- a/internal/ingestion/component/chunker/one_test.go
+++ b/internal/ingestion/component/chunker/one_test.go
@@ -18,6 +18,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -32,7 +33,7 @@ func oneChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
if err != nil {
t.Fatalf("OneChunker.Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks not []map[string]any: %T", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/paddleocr_test.go b/internal/ingestion/component/chunker/paddleocr_test.go
index d4ea0371af..9b57caa5cf 100644
--- a/internal/ingestion/component/chunker/paddleocr_test.go
+++ b/internal/ingestion/component/chunker/paddleocr_test.go
@@ -71,7 +71,7 @@ func TestPaddleOCRHighlightsThroughPipeline(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks := output["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(output["chunks"])
if len(chunks) == 0 {
t.Fatal("no chunks")
}
diff --git a/internal/ingestion/component/chunker/page_test.go b/internal/ingestion/component/chunker/page_test.go
index 337e1248cf..447bf9a208 100644
--- a/internal/ingestion/component/chunker/page_test.go
+++ b/internal/ingestion/component/chunker/page_test.go
@@ -18,6 +18,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -32,7 +33,7 @@ func slideChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
if err != nil {
t.Fatalf("PageChunker.Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks not []map[string]any: %T", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/python_parity_test.go b/internal/ingestion/component/chunker/python_parity_test.go
index a94210a1e8..9ff3367ca6 100644
--- a/internal/ingestion/component/chunker/python_parity_test.go
+++ b/internal/ingestion/component/chunker/python_parity_test.go
@@ -35,6 +35,7 @@ import (
"testing"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/ingestion/component/schema"
)
// invokeAsTokenChunker constructs a TokenChunker component and
@@ -187,21 +188,11 @@ func chunksFromOutput(t *testing.T, out map[string]any) []map[string]any {
if !ok {
t.Fatalf("output missing 'chunks' key; got keys=%v", keysOf(out))
}
- switch v := raw.(type) {
- case []map[string]any:
- return v
- case []any:
- out := make([]map[string]any, 0, len(v))
- for _, it := range v {
- if m, ok := it.(map[string]any); ok {
- out = append(out, m)
- }
- }
- return out
- default:
- t.Fatalf("chunks: unexpected type %T", raw)
- return nil
+ maps, ok := schema.ChunkCarrierToMaps(raw)
+ if !ok {
+ t.Fatalf("chunks: unexpected carrier type %T", raw)
}
+ return maps
}
func keysOf(m map[string]any) []string {
diff --git a/internal/ingestion/component/chunker/qa_batch2_test.go b/internal/ingestion/component/chunker/qa_batch2_test.go
index aa2f762dab..8daa3c588f 100644
--- a/internal/ingestion/component/chunker/qa_batch2_test.go
+++ b/internal/ingestion/component/chunker/qa_batch2_test.go
@@ -18,6 +18,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -34,7 +35,7 @@ func qaInvoke(t *testing.T, inputs map[string]any) []map[string]any {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("Invoke did not return []map chunks: %#v", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
index 70b139eca8..583ee33d5b 100644
--- a/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
+++ b/internal/ingestion/component/chunker/qa_spreadsheet_contract_test.go
@@ -5,6 +5,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
)
@@ -28,7 +29,7 @@ func TestCSVQAFirstRowIsData(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("expected both QA rows, got %d chunks: %#v", len(chunks), chunks)
}
@@ -53,7 +54,7 @@ func TestCSVTypedRowsFoldMalformedRowIntoPreviousAnswer(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("malformed row created or removed a pair: %#v", chunks)
}
@@ -80,7 +81,7 @@ func TestCSVJSONHTMLTableUsesStrictContinuationRules(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("malformed HTML row created a QA pair: %#v", chunks)
}
diff --git a/internal/ingestion/component/chunker/qa_test.go b/internal/ingestion/component/chunker/qa_test.go
index 9fdc5e3fb4..fcf73456d3 100644
--- a/internal/ingestion/component/chunker/qa_test.go
+++ b/internal/ingestion/component/chunker/qa_test.go
@@ -66,7 +66,7 @@ func TestQAChunker_DelimiterTab(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -91,7 +91,7 @@ func TestQAChunker_DelimiterComma(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -116,7 +116,7 @@ func TestQAChunker_Markdown(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d", len(chunks))
}
@@ -136,7 +136,7 @@ func TestQAChunker_HTMLTable(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %d", len(chunks))
}
@@ -156,7 +156,7 @@ func TestQAChunker_CSVStrictPairRejectsThreeCells(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Fatalf("expected 0 chunks, got %d", len(chunks))
}
@@ -176,7 +176,7 @@ func TestQAChunker_CSVStrictPairAcceptsTwoCells(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -203,7 +203,7 @@ func TestQAChunker_JSONCSVNameUsesStrictRowShapeWithoutFileType(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Fatalf("chunks = %#v, want malformed CSV row rejected", chunks)
}
@@ -226,7 +226,7 @@ func TestQAChunker_NonCSVHTMLThreeCellsKeepsFirstTwo(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -250,7 +250,7 @@ func TestQAChunker_RmQAPrefix(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
cww, _ := chunks[0]["text"].(string)
if cww != "Question: What is Go?\tAnswer: Go is a language." {
t.Fatalf("prefix not stripped: %q", cww)
@@ -271,7 +271,7 @@ func TestQAChunker_Empty(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Fatalf("expected 0 chunks, got %d", len(chunks))
}
@@ -291,7 +291,7 @@ func TestQAChunker_CaseInsensitivePrefix(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -315,7 +315,7 @@ func TestQAChunker_PrefixSpaceSeparatorStrips(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -341,7 +341,7 @@ func TestQAChunker_HeadingNoTrailingSpace(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -361,7 +361,7 @@ func TestQAChunker_ChineseLang(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -385,7 +385,7 @@ func TestQAChunker_MarkdownRendersHTML(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -415,7 +415,7 @@ func TestQAChunker_XLSXJSONRegression(t *testing.T) {
if err != nil {
t.Fatalf("Invoke failed: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 3 {
t.Fatalf("expected 3 chunks, got %d", len(chunks))
}
@@ -449,7 +449,7 @@ func TestQAChunkerSpreadsheetWireTreatsFirstRowAsQAData(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 2 {
t.Fatalf("chunks = %#v, want both QA rows", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/register.go b/internal/ingestion/component/chunker/register.go
index 661c30aeef..51e076c9f5 100644
--- a/internal/ingestion/component/chunker/register.go
+++ b/internal/ingestion/component/chunker/register.go
@@ -31,6 +31,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/ingestion/component/globals"
+ "ragflow/internal/ingestion/component/schema"
"gorm.io/gorm"
)
@@ -68,8 +69,25 @@ func (d *imageUploadDecorator) Invoke(ctx context.Context, db *gorm.DB, inputs m
if err != nil {
return nil, err
}
- chunks, ok := out["chunks"].([]map[string]any)
- if !ok || len(chunks) == 0 {
+ // Accept the legacy []map[string]any carrier verbatim (no round-trip, so
+ // the id-stamp/upload logic below sees the exact maps the chunker emitted).
+ // The typed []ChunkDoc carrier is normalized via GetChunks, which also
+ // bridges []any-of-map, so this choke point no longer silently skips the
+ // whole id-stamp/upload pass on a carrier-type mismatch.
+ var chunks []map[string]any
+ switch v := out["chunks"].(type) {
+ case []map[string]any:
+ chunks = v
+ default:
+ if typed, ok := schema.GetChunks(out); ok {
+ chunks = schema.ChunkDocsToMaps(typed)
+ // Write the normalized carrier back so the in-place id-stamp and
+ // image-drop mutations below propagate to the emitted envelope
+ // (the legacy []map[string]any path shares the reference already).
+ out["chunks"] = chunks
+ }
+ }
+ if len(chunks) == 0 {
return out, nil
}
diff --git a/internal/ingestion/component/chunker/register_envelope_test.go b/internal/ingestion/component/chunker/register_envelope_test.go
new file mode 100644
index 0000000000..d03fa58efd
--- /dev/null
+++ b/internal/ingestion/component/chunker/register_envelope_test.go
@@ -0,0 +1,68 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunker
+
+import (
+ "context"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+
+ "gorm.io/gorm"
+)
+
+// typedChunksInner is a minimal runtime.Component fake that returns a fixed
+// output map carrying a typed []ChunkDoc under "chunks".
+type typedChunksInner struct {
+ out map[string]any
+}
+
+func (t *typedChunksInner) Invoke(ctx context.Context, db *gorm.DB, inputs map[string]any) (map[string]any, error) {
+ return t.out, nil
+}
+
+// TestImageUploadDecorator_ProcessesTypedChunks locks the R2 regression: the
+// imageUploadDecorator is the single choke point every chunker variant flows
+// through. When the upstream carrier is the typed []ChunkDoc form, the
+// decorator must normalize it (not silently skip), stamp chunk ids, and
+// re-emit the []map[string]any wire form. Before migration the bare
+// .([]map[string]any) assertion failed on a typed carrier, so the whole
+// id-stamp/upload pass was skipped.
+//
+// kb_id is intentionally absent, so this exercises the dry-run branch (drop
+// image bytes, no MinIO upload).
+func TestImageUploadDecorator_ProcessesTypedChunks(t *testing.T) {
+ inner := &typedChunksInner{out: map[string]any{
+ "chunks": []schema.ChunkDoc{
+ {Text: "hello", Image: "img/1.png"}, // no id yet
+ },
+ }}
+ dec := &imageUploadDecorator{inner: inner}
+
+ out, err := dec.Invoke(context.Background(), nil, map[string]any{})
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
+ if !ok {
+ t.Fatalf("out[chunks] type = %T, want []map[string]any (typed carrier must be normalized)", out["chunks"])
+ }
+ if len(chunks) != 1 {
+ t.Fatalf("len = %d, want 1", len(chunks))
+ }
+ if id, _ := chunks[0]["id"].(string); id == "" {
+ t.Fatalf("chunk id not stamped: decorator skipped the typed carrier? chunk=%+v", chunks[0])
+ }
+}
diff --git a/internal/ingestion/component/chunker/table_html_text_test.go b/internal/ingestion/component/chunker/table_html_text_test.go
index 917374dea1..5ade4fb167 100644
--- a/internal/ingestion/component/chunker/table_html_text_test.go
+++ b/internal/ingestion/component/chunker/table_html_text_test.go
@@ -23,6 +23,7 @@ import (
"testing"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
)
// generalTableChunksOf drives GeneralChunker.Invoke over structured TableData
@@ -41,7 +42,7 @@ func generalTableChunksOf(t *testing.T, params map[string]any, items ...map[stri
if err != nil {
t.Fatalf("GeneralChunker.Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("output chunks type = %T, want []map[string]any", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/table_test.go b/internal/ingestion/component/chunker/table_test.go
index 82e5189b9f..43eb78b510 100644
--- a/internal/ingestion/component/chunker/table_test.go
+++ b/internal/ingestion/component/chunker/table_test.go
@@ -22,6 +22,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/tableutil"
)
@@ -36,7 +37,7 @@ func tableChunksOf(t *testing.T, inputs map[string]any) []map[string]any {
if err != nil {
t.Fatalf("TableChunker.Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks not []map[string]any: %T", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/textcode_parity_test.go b/internal/ingestion/component/chunker/textcode_parity_test.go
index 04daf104d0..c39a25719b 100644
--- a/internal/ingestion/component/chunker/textcode_parity_test.go
+++ b/internal/ingestion/component/chunker/textcode_parity_test.go
@@ -24,6 +24,7 @@ import (
"strings"
"testing"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
)
@@ -108,7 +109,7 @@ func TestTextParserGeneralChunkerParity(t *testing.T) {
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
t.Fatalf("GeneralChunker returned _ERROR: %s", msg)
}
- got, ok := out["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("GeneralChunker chunks has type %T", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.go
index 2db586ed10..c7aa1323e6 100644
--- a/internal/ingestion/component/chunker/title_structured_media_test.go
+++ b/internal/ingestion/component/chunker/title_structured_media_test.go
@@ -26,6 +26,7 @@ import (
"ragflow/internal/agent/runtime"
"ragflow/internal/common"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/tableutil"
)
@@ -79,7 +80,7 @@ func titleFamilyChunks(t *testing.T, comp runtime.Component, items []map[string]
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
@@ -204,7 +205,7 @@ func TestTitleFamilyEmptyTextImageUploadsThroughDecorator(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("chunks = %d, want 1", len(chunks))
}
@@ -255,7 +256,7 @@ func TestTitleFamilyCaptionlessImagesGetDistinctIDs(t *testing.T) {
if err != nil {
t.Fatalf("decorated Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("chunks = %d, want 2", len(chunks))
}
@@ -288,7 +289,7 @@ func TestTitleFamilyReportsDecodeFailure(t *testing.T) {
if msg, _ := out["_ERROR"].(string); !strings.HasPrefix(msg, "Input error:") {
t.Fatalf("_ERROR = %v, want an input error", out["_ERROR"])
}
- if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 0 {
+ if chunks, _ := schema.ChunkCarrierToMaps(out["chunks"]); len(chunks) != 0 {
t.Fatalf("chunks = %#v, want none", chunks)
}
})
diff --git a/internal/ingestion/component/chunker/title_test.go b/internal/ingestion/component/chunker/title_test.go
index a1b0a4c083..5f12477833 100644
--- a/internal/ingestion/component/chunker/title_test.go
+++ b/internal/ingestion/component/chunker/title_test.go
@@ -98,7 +98,7 @@ func TestTitleChunker_InvokeEmptyInput(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Errorf("chunks = %d, want 0", len(chunks))
}
@@ -136,7 +136,7 @@ func TestTitleChunker_Headings_ASCII(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -165,7 +165,7 @@ func TestTitleChunker_NoHeadings_FallsBack(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -218,7 +218,7 @@ func TestTitleChunker_InvokeDeterministic(t *testing.T) {
if err != nil {
t.Fatalf("Invoke run %d: %v", run, err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
texts := make([]string, len(chunks))
for i, ck := range chunks {
texts[i], _ = ck["text"].(string)
diff --git a/internal/ingestion/component/chunker/title_token_cap_test.go b/internal/ingestion/component/chunker/title_token_cap_test.go
index d35cf8fb2a..e24b44c51e 100644
--- a/internal/ingestion/component/chunker/title_token_cap_test.go
+++ b/internal/ingestion/component/chunker/title_token_cap_test.go
@@ -445,7 +445,7 @@ func TestTitleCap_GroupPipeline_RespectsCap(t *testing.T) {
if err != nil {
t.Fatalf("invokeGroup: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -469,7 +469,7 @@ func TestTitleCap_GroupPipeline_ValidatedInRangeCap(t *testing.T) {
if err != nil {
t.Fatalf("invokeGroup: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected cap=128 to re-split the oversized body, got %d", len(chunks))
}
@@ -486,7 +486,7 @@ func TestTitleCap_HierarchyPipeline_RespectsCap(t *testing.T) {
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -503,7 +503,7 @@ func TestTitleCap_GroupPipeline_CapZeroNoop(t *testing.T) {
if err != nil {
t.Fatalf("invokeGroup: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) != 1 {
t.Fatalf("cap=0 must keep 1 chunk, got %d", len(chunks))
}
@@ -524,7 +524,7 @@ func TestTitleCap_GroupPipeline_MultiRecordMerged(t *testing.T) {
if err != nil {
t.Fatalf("invokeGroup: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected merged split, got %d", len(chunks))
}
@@ -553,7 +553,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepPositions(t *testing.T) {
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -593,7 +593,7 @@ func TestTitleCap_HierarchyPipeline_SubChunksKeepHeadingChain(t *testing.T) {
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -639,7 +639,7 @@ func TestTitleCap_HierarchyPipeline_RootAsHeadingKeepsChain(t *testing.T) {
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -677,7 +677,7 @@ func TestTitleCap_HierarchyPipeline_AncestorBodyKeepsLeadingChain(t *testing.T)
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
@@ -718,7 +718,7 @@ func TestTitleCap_HierarchyPipeline_IncludeHeadingContentKeepsTitles(t *testing.
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 2 {
t.Fatalf("expected parent chunk + split leaf pieces, got %d", len(chunks))
}
@@ -761,7 +761,7 @@ func TestTitleCap_HierarchyPipeline_NearCapPrefixFallsBack(t *testing.T) {
if err != nil {
t.Fatalf("invokeHierarchy: %v", err)
}
- chunks := got["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(got["chunks"])
if len(chunks) <= 1 {
t.Fatalf("expected split, got %d", len(chunks))
}
diff --git a/internal/ingestion/component/chunker/token_children_delim_test.go b/internal/ingestion/component/chunker/token_children_delim_test.go
index 5b87758bde..abbe0cc394 100644
--- a/internal/ingestion/component/chunker/token_children_delim_test.go
+++ b/internal/ingestion/component/chunker/token_children_delim_test.go
@@ -1,6 +1,7 @@
package chunker
import (
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -24,7 +25,7 @@ func TestTokenChunker_ChildrenDelimiterDroppedJSON(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 3 {
t.Fatalf("chunk count: want 3 got %d (%v)", len(chunks), chunkTexts(chunks))
}
@@ -64,7 +65,7 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
// The children delimiter "###" is dropped from each child.
want := []string{"sec one", "sec two", "sec three"}
if len(chunks) != len(want) {
@@ -92,7 +93,7 @@ func TestTokenChunker_ChildrenDelimiterBacktickStripped(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks2, _ := out2["chunks"].([]map[string]any)
+ chunks2, _ := schema.ChunkCarrierToMaps(out2["chunks"])
// The inner "###" matches and is DROPPED, leaving the surrounding
// backticks as literal text.
backtickWrapped := []string{"a `", "` b"}
@@ -123,7 +124,7 @@ func TestTokenChunker_ChildrenDelimiterDroppedText(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
// The children delimiter ". " is retained on each child.
want := []string{"alpha one. ", "alpha two. ", "alpha three."}
if len(chunks) != len(want) {
diff --git a/internal/ingestion/component/chunker/token_delim_contract_test.go b/internal/ingestion/component/chunker/token_delim_contract_test.go
index acd3c5e36f..c846f44776 100644
--- a/internal/ingestion/component/chunker/token_delim_contract_test.go
+++ b/internal/ingestion/component/chunker/token_delim_contract_test.go
@@ -1,6 +1,7 @@
package chunker
import (
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -29,7 +30,7 @@ func TestTokenChunker_BareDelimiterHonored(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatalf("no chunks produced")
}
@@ -73,7 +74,7 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
// Custom (backtick) delimiters are DROPPED, mirroring Python's
// naive_merge has_custom branch: "段落" is split away, not preserved.
want := []string{"第一部分", "第二部分"}
@@ -104,7 +105,7 @@ func TestTokenChunker_MultiByteBacktickDelimiter(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks2, _ := out2["chunks"].([]map[string]any)
+ chunks2, _ := schema.ChunkCarrierToMaps(out2["chunks"])
if len(chunks2) != 2 || chunks2[0]["text"].(string) != "A" || chunks2[1]["text"].(string) != "B" {
t.Errorf("longest delimiter not preferred: got %v", chunkTexts(chunks2))
}
diff --git a/internal/ingestion/component/chunker/token_delim_lossless_test.go b/internal/ingestion/component/chunker/token_delim_lossless_test.go
index f636277518..c4d06972aa 100644
--- a/internal/ingestion/component/chunker/token_delim_lossless_test.go
+++ b/internal/ingestion/component/chunker/token_delim_lossless_test.go
@@ -28,6 +28,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -58,7 +59,7 @@ func driveChunker(t *testing.T, params map[string]any, input map[string]any) []m
if msg, _ := out["_ERROR"].(string); msg != "" {
t.Fatalf("TokenChunker returned _ERROR: %s", msg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
diff --git a/internal/ingestion/component/chunker/token_item_separator_test.go b/internal/ingestion/component/chunker/token_item_separator_test.go
index 4d9719932e..e0c95acd9f 100644
--- a/internal/ingestion/component/chunker/token_item_separator_test.go
+++ b/internal/ingestion/component/chunker/token_item_separator_test.go
@@ -17,6 +17,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -74,7 +75,7 @@ func invokeTokenChunkerJSON(t *testing.T, texts ...string) string {
t.Fatalf("TokenChunker returned _ERROR: %s", msg)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("chunks = %#v, want one merged chunk", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/token_merge_parity_test.go b/internal/ingestion/component/chunker/token_merge_parity_test.go
index ab38e0d4a2..ded584de81 100644
--- a/internal/ingestion/component/chunker/token_merge_parity_test.go
+++ b/internal/ingestion/component/chunker/token_merge_parity_test.go
@@ -187,7 +187,7 @@ func invokeTokenChunker(t *testing.T, text string, chunkTokenSize int, overlappe
if msg, ok := out["_ERROR"].(string); ok && msg != "" {
t.Fatalf("TokenChunker returned _ERROR: %s", msg)
}
- raw, _ := out["chunks"].([]map[string]any)
+ raw, _ := schema.ChunkCarrierToMaps(out["chunks"])
texts := make([]string, 0, len(raw))
for _, c := range raw {
if s, ok := c["text"].(string); ok {
diff --git a/internal/ingestion/component/chunker/token_overlap_test.go b/internal/ingestion/component/chunker/token_overlap_test.go
index 50fcb46041..1d25a4beba 100644
--- a/internal/ingestion/component/chunker/token_overlap_test.go
+++ b/internal/ingestion/component/chunker/token_overlap_test.go
@@ -2,6 +2,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -56,7 +57,7 @@ func TestTokenChunker_TextOverlapPreservesInterLineSpace(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatalf("expected chunks, got none")
}
diff --git a/internal/ingestion/component/chunker/token_oversize_split_test.go b/internal/ingestion/component/chunker/token_oversize_split_test.go
index bfcceff323..71f10f50e6 100644
--- a/internal/ingestion/component/chunker/token_oversize_split_test.go
+++ b/internal/ingestion/component/chunker/token_oversize_split_test.go
@@ -16,6 +16,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -59,7 +60,7 @@ func TestTokenChunker_OversizeUnitSplit(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks missing or wrong type: %T", out["chunks"])
}
@@ -128,7 +129,7 @@ func TestTokenChunker_OversizeUnitSplitAfterInBudgetUnit(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks missing or wrong type: %T", out["chunks"])
}
diff --git a/internal/ingestion/component/chunker/token_strict_cap_test.go b/internal/ingestion/component/chunker/token_strict_cap_test.go
index de20ef6af2..6608565e80 100644
--- a/internal/ingestion/component/chunker/token_strict_cap_test.go
+++ b/internal/ingestion/component/chunker/token_strict_cap_test.go
@@ -114,7 +114,7 @@ func TestMergeByTokenSize_TextPathStrictCap(t *testing.T) {
}
tc := comp.(*TokenChunkerComponent)
out := tc.mergeByTokenSize(b.String(), nil, nil)
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) < 2 {
t.Fatalf("want multiple chunks, got %d", len(chunks))
}
@@ -139,7 +139,7 @@ func TestMergeByTokenSize_OversizedBoundarylessRunSplit(t *testing.T) {
t.Fatalf("NewTokenChunker: %v", err)
}
out := comp.(*TokenChunkerComponent).mergeByTokenSize(long, nil, nil)
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) < 2 {
t.Fatalf("oversized run must be split, got %d chunk(s)", len(chunks))
}
@@ -176,7 +176,7 @@ func TestInvokeTextPayload_HardCapEndToEnd(t *testing.T) {
if errMsg, _ := out["_ERROR"].(string); errMsg != "" {
t.Fatalf("Invoke error payload: %s", errMsg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatalf("expected chunks, got %#v", out)
}
@@ -216,7 +216,7 @@ func TestInvokeJSONPayload_HardCapEndToEnd(t *testing.T) {
if errMsg, _ := out["_ERROR"].(string); errMsg != "" {
t.Fatalf("Invoke error payload: %s", errMsg)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) < 2 {
t.Fatalf("want multiple chunks, got %d", len(chunks))
}
diff --git a/internal/ingestion/component/chunker/token_tag_test.go b/internal/ingestion/component/chunker/token_tag_test.go
index bca95db810..ef2d58792d 100644
--- a/internal/ingestion/component/chunker/token_tag_test.go
+++ b/internal/ingestion/component/chunker/token_tag_test.go
@@ -15,6 +15,7 @@ package chunker
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"strings"
"testing"
)
@@ -40,7 +41,7 @@ func TestTokenChunker_TextPath_StripsParserTags(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -74,7 +75,7 @@ func TestTokenChunker_JSONPath_StripsParserTags(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
diff --git a/internal/ingestion/component/chunker/token_test.go b/internal/ingestion/component/chunker/token_test.go
index d69f1ec25d..2fbc33bd9e 100644
--- a/internal/ingestion/component/chunker/token_test.go
+++ b/internal/ingestion/component/chunker/token_test.go
@@ -86,7 +86,7 @@ func TestTokenChunkerPreservesSpreadsheetSegmentBoundaries(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", out["chunks"])
}
@@ -119,7 +119,7 @@ func TestTokenChunkerPreservesHeaderOnlySheetAlongsideDataSheet(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("chunks = %#v, want data segment and header-only segment", chunks)
}
@@ -148,7 +148,7 @@ func TestTokenChunker_InvokeDelimMode_BasicChunking(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -182,7 +182,7 @@ func TestTokenChunker_DelimNeverStandaloneChunk(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("chunks = %d, want 2: %v", len(chunks), chunks)
}
@@ -228,7 +228,7 @@ func TestTokenChunker_InvokeTokenSize_FallbackToMerge(t *testing.T) {
if got, want := out["output_format"], "chunks"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) < 1 {
t.Errorf("chunks = %d, want >=1", len(chunks))
}
@@ -254,7 +254,7 @@ func TestTokenChunker_InvokeChildrenDelim(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -284,7 +284,7 @@ func TestTokenChunker_InvokeJSONPayload(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -340,7 +340,7 @@ func TestTokenChunker_InvokeJSONPayload_IndexesHeaderOnlyEmail(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) == 0 {
t.Fatalf("chunks missing: %#v", out["chunks"])
}
@@ -390,7 +390,7 @@ func TestTokenChunker_InvokeJSONPayload_KeepsNonTextStandalone(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks: want []map[string]any, got %T", out["chunks"])
}
@@ -650,7 +650,7 @@ func TestTokenChunker_InvokeDeterministic(t *testing.T) {
if err != nil {
t.Fatalf("Invoke run %d: %v", run, err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
fp := fingerprint{count: len(chunks)}
if len(chunks) > 0 {
fp.first, _ = chunks[0]["text"].(string)
@@ -703,7 +703,7 @@ func TestTokenChunker_SpreadsheetTablePreservesSelectionRange(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("chunks = %d, want one unsplit table item", len(chunks))
}
@@ -776,7 +776,7 @@ func TestTokenChunker_PrefersUpstreamChunks(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("chunks: want >=1, got 0")
}
@@ -970,7 +970,7 @@ func TestMergeByTokenSize_CRLFNormalization(t *testing.T) {
c.param.ChunkTokenSize = 128
c.param.OverlappedPercent = 0
out := c.mergeByTokenSize(text, nil, nil)
- raw, ok := out["chunks"].([]map[string]any)
+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
}
@@ -1015,7 +1015,7 @@ func TestMergeByTokenSize_PreservesBlankLines(t *testing.T) {
c.param.ChunkTokenSize = 128
c.param.OverlappedPercent = 0
out := c.mergeByTokenSize("A\n\n\nB", nil, nil)
- raw, ok := out["chunks"].([]map[string]any)
+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
}
@@ -1044,7 +1044,7 @@ func TestMergeByTokenSize_OversizeDropsDelimiters(t *testing.T) {
c.param.OverlappedPercent = 0
text := "第一句。第二句。第三句。第四句。第五句。"
out := c.mergeByTokenSize(text, nil, nil)
- raw, ok := out["chunks"].([]map[string]any)
+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
}
@@ -1075,7 +1075,7 @@ func TestMergeByTokenSize_OversizeDropsBlankLines(t *testing.T) {
block := strings.Repeat("知识库检索增强生成技术", 7) // 70 chars
text := block + "\n\n" + block
out := c.mergeByTokenSize(text, nil, nil)
- raw, ok := out["chunks"].([]map[string]any)
+ raw, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("mergeByTokenSize output missing chunks: %v", out)
}
diff --git a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
index 946983df85..4c85241678 100644
--- a/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
+++ b/internal/ingestion/component/chunker/xlsx_qa_regression_test.go
@@ -8,6 +8,7 @@ import (
"github.com/xuri/excelize/v2"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
)
@@ -59,7 +60,7 @@ func qaChunksFromXLSX(t *testing.T, data []byte) []map[string]any {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
return chunks
}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 4e3e76aa34..3c13debdeb 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -644,6 +644,8 @@ func extractorChunkList(v any) ([]map[string]any, bool) {
switch list := v.(type) {
case []map[string]any:
return list, true
+ case []schema.ChunkDoc:
+ return schema.ChunkDocsToMaps(list), true
case []any:
out := make([]map[string]any, 0, len(list))
for _, item := range list {
diff --git a/internal/ingestion/component/extractor_call_timeout_test.go b/internal/ingestion/component/extractor_call_timeout_test.go
index 2f2a2852e0..bf7cd6220f 100644
--- a/internal/ingestion/component/extractor_call_timeout_test.go
+++ b/internal/ingestion/component/extractor_call_timeout_test.go
@@ -85,7 +85,7 @@ func TestExtractorChatTimeoutSkipsSlowChunk(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("chunks = %d, want 2", len(chunks))
}
@@ -222,7 +222,7 @@ func TestExtractorPartialTimeoutKeepsCompleted(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("chunks = %d, want 2", len(chunks))
}
diff --git a/internal/ingestion/component/extractor_envelope_test.go b/internal/ingestion/component/extractor_envelope_test.go
new file mode 100644
index 0000000000..1d966e60c6
--- /dev/null
+++ b/internal/ingestion/component/extractor_envelope_test.go
@@ -0,0 +1,41 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package component
+
+import (
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestExtractorChunkList_AcceptsTypedChunkDocs: once the upstream chunker emits
+// the typed []ChunkDoc carrier, the extractor's input reader must accept it
+// (converting to the map wire form) instead of returning ok=false, which would
+// silently drop every upstream chunk from the extraction context.
+func TestExtractorChunkList_AcceptsTypedChunkDocs(t *testing.T) {
+ got, ok := extractorChunkList([]schema.ChunkDoc{{Text: "a", ID: "1"}, {Text: "b", ID: "2"}})
+ if !ok {
+ t.Fatal("extractorChunkList(typed []ChunkDoc) ok=false, want true")
+ }
+ if len(got) != 2 || got[0]["text"] != "a" || got[1]["text"] != "b" {
+ t.Fatalf("typed carrier mismatch: %+v", got)
+ }
+
+ // Legacy []map[string]any must still be accepted verbatim.
+ got2, ok2 := extractorChunkList([]map[string]any{{"text": "x"}})
+ if !ok2 || len(got2) != 1 || got2[0]["text"] != "x" {
+ t.Fatalf("legacy []map carrier: ok=%v got=%+v", ok2, got2)
+ }
+}
diff --git a/internal/ingestion/component/extractor_tag_test.go b/internal/ingestion/component/extractor_tag_test.go
index 7f8c231afc..e7b604c509 100644
--- a/internal/ingestion/component/extractor_tag_test.go
+++ b/internal/ingestion/component/extractor_tag_test.go
@@ -58,7 +58,7 @@ func TestExtractorTags_NoTagFileID(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %d", len(chunks))
}
@@ -81,7 +81,7 @@ func TestExtractorTags_NoLLMID(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if _, ok := chunks[0][common.TAG_FLD]; ok {
t.Fatal("tag_feas should not be set without tag_file_id")
}
@@ -111,7 +111,7 @@ func TestExtractorTags_WithKeywords(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if chunks[0][common.TAG_FLD] != nil {
t.Fatal("tag_feas should not be set without tag_file_id")
}
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index cb85b202be..e3195ce08b 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -150,7 +150,7 @@ func TestExtractorComponent_Invoke_HappyPath(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks key missing or wrong shape: %T", out["chunks"])
}
@@ -186,7 +186,7 @@ func TestExtractorComponent_SkipsMediaChunksWithoutTextAndUsesContext(t *testing
if calls := stub.Calls(); calls != 1 {
t.Fatalf("LLM calls = %d, want one call for the context-bearing media chunk", calls)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 2 {
t.Fatalf("chunks = %#v, want two input chunks", out["chunks"])
}
@@ -265,7 +265,7 @@ func TestExtractorComponent_Invoke_RetrySucceeds(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if s, _ := chunks[0]["summary"].(string); s != "recovered" {
t.Errorf("summary = %q, want recovered", s)
}
@@ -306,7 +306,7 @@ func TestExtractorComponent_Invoke_EmptyChunksReturnsEmpty(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks missing or wrong shape")
}
@@ -331,7 +331,7 @@ func TestExtractorComponent_Invoke_JSONListInput(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("chunks malformed: %v", out["chunks"])
}
@@ -536,7 +536,7 @@ func TestExtractorComponent_Invoke_ToggleOffSkipsExtraction(t *testing.T) {
if calls := stub.Calls(); calls != 0 {
t.Errorf("LLM calls = %d, want 0 for a switched-off keywords group", calls)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if _, exists := chunks[0]["important_kwd"]; exists {
t.Errorf("switched-off keywords should not enrich the chunk: %#v", chunks[0])
}
@@ -1191,7 +1191,7 @@ func TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions(t *testing.T)
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 2 {
t.Fatalf("expected 2 chunks, got %v", out["chunks"])
}
@@ -1601,7 +1601,7 @@ func TestExtractorModularPromptsExecution(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -1714,7 +1714,7 @@ func TestExtractorModularMetadataExecution(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -1758,7 +1758,7 @@ func TestExtractorDefaultSummaryPromptInjection(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -1813,7 +1813,7 @@ func TestExtractorCustomSummarySystemPrompt(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -1871,7 +1871,7 @@ func TestExtractorCustomKeywordsAndQuestionsSystemPrompt(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -2016,7 +2016,7 @@ func TestExtractorDisabledSummarySkipsCall(t *testing.T) {
t.Errorf("expected 0 LLM calls when summary is disabled, got %d", stub.Calls())
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk output, got %v", out)
}
@@ -2166,7 +2166,7 @@ func TestExtractorBuiltInDoesNotCallLLM(t *testing.T) {
if stub.calls.Load() != 0 {
t.Fatalf("built_in-only must not call LLM, got %d calls, requests=%v", stub.calls.Load(), stub.requests)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %v", out)
}
@@ -2201,7 +2201,7 @@ func TestExtractorEnabledFalseDoesNotCallLLM(t *testing.T) {
if stub.calls.Load() != 0 {
t.Fatalf("enabled=false must not call LLM, got %d", stub.calls.Load())
}
- if chunks, _ := out["chunks"].([]map[string]any); len(chunks) != 1 {
+ if chunks, _ := schema.ChunkCarrierToMaps(out["chunks"]); len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %v", out)
}
}
@@ -2234,7 +2234,7 @@ func TestExtractor_KeywordsThenTagsSynergy(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- chunks, ok := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("expected 1 chunk, got %v", out)
}
@@ -2318,7 +2318,8 @@ func TestExtractor_CallTextCached_NoRedis_FailOpen(t *testing.T) {
if calls := stub.Calls(); calls != 3 {
t.Fatalf("Expected 3 LLM calls, got %d", calls)
}
- ck := out["chunks"].([]map[string]any)[0]
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
+ ck := chunks[0]
if sum, ok := ck["summary"].(string); !ok || sum != "This is a summary without Redis." {
t.Errorf("got summary %v, want 'This is a summary without Redis.'", ck["summary"])
}
diff --git a/internal/ingestion/component/knowledge_compiler/component.go b/internal/ingestion/component/knowledge_compiler/component.go
index c5da9b7d53..a6117b0dbb 100644
--- a/internal/ingestion/component/knowledge_compiler/component.go
+++ b/internal/ingestion/component/knowledge_compiler/component.go
@@ -749,9 +749,11 @@ func metaStringSlice(m map[string]any, key string) []string {
// headless / manual chaining reads them from the component output map, so they
// must be forwarded when present.
func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]any {
- // Accept both the []any and []map[string]any chunk carriers (the chunker
- // emits the latter; buildInputs already handles both). Without this, the
- // original source chunks would be dropped when the carrier is []map[string]any.
+ // Accept the []any and []map[string]any chunk carriers verbatim (the
+ // legacy chunker emits the latter; buildInputs already handles both). The
+ // default arm accepts the typed []ChunkDoc carrier via GetChunks, which
+ // also bridges legacy forms, so the upstream chunks are never silently
+ // dropped on a type mismatch.
var raw []any
switch v := inputs["chunks"].(type) {
case []any:
@@ -762,7 +764,13 @@ func mergeChunks(inputs map[string]any, compiled []schema.ChunkDoc) map[string]a
raw = append(raw, m)
}
default:
- clog.Warn("knowledge_compiler: mergeChunks: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))
+ if typed, ok := schema.GetChunks(inputs); ok {
+ for _, c := range typed {
+ raw = append(raw, c.ToMap())
+ }
+ } else {
+ clog.Warn("knowledge_compiler: mergeChunks: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))
+ }
}
merged := make([]any, 0, len(raw)+len(compiled))
for _, r := range raw {
@@ -811,7 +819,8 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, erro
// The upstream pipeline hands chunks over as a []any of map[string]any in
// some paths and as a []map[string]any in others (the chunker emits the
// latter). Accept both so the knowledge compiler never silently drops the
- // whole upstream output on a type mismatch.
+ // whole upstream output on a type mismatch. The typed []ChunkDoc carrier is
+ // handled via GetChunks, which also bridges the legacy forms.
var raw []map[string]any
switch v := inputs["chunks"].(type) {
case []any:
@@ -826,7 +835,11 @@ func buildInputs(inputs map[string]any, param common.Param) (common.Inputs, erro
case []map[string]any:
raw = v
default:
- clog.Warn("knowledge_compiler: buildInputs: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))
+ if typed, ok := schema.GetChunks(inputs); ok {
+ raw = schema.ChunkDocsToMaps(typed)
+ } else {
+ clog.Warn("knowledge_compiler: buildInputs: unexpected chunks type", zap.String("type", fmt.Sprintf("%T", inputs["chunks"])))
+ }
}
clog.Info("knowledge_compiler: buildInputs: accepted chunks from inputs[chunks]", zap.Int("chunks", len(raw)))
for _, m := range raw {
diff --git a/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.go b/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.go
new file mode 100644
index 0000000000..2af99d947d
--- /dev/null
+++ b/internal/ingestion/component/knowledge_compiler/envelope_typed_chunks_test.go
@@ -0,0 +1,76 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package knowledge_compiler
+
+import (
+ "testing"
+
+ "ragflow/internal/ingestion/component/knowledge_compiler/common"
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestKnowledgeCompiler_BuildInputsAcceptsTypedChunkDocs: once chunkers emit
+// typed []ChunkDoc through the envelope, buildInputs must accept the typed
+// carrier in addition to the legacy []map[string]any / []any forms, without
+// dropping every chunk on a type mismatch.
+func TestKnowledgeCompiler_BuildInputsAcceptsTypedChunkDocs(t *testing.T) {
+ in, err := buildInputs(map[string]any{
+ "chunks": []schema.ChunkDoc{
+ {ID: "c1", Text: "甲"},
+ {ID: "c2", Text: "乙"},
+ },
+ }, common.Param{})
+ if err != nil {
+ t.Fatalf("buildInputs (typed []ChunkDoc): %v", err)
+ }
+ if len(in.Chunks) != 2 {
+ t.Fatalf("buildInputs produced %d chunks, want 2 (typed []ChunkDoc carrier)", len(in.Chunks))
+ }
+ if in.Chunks[0].ID != "c1" || in.Chunks[0].Text != "甲" {
+ t.Fatalf("chunk[0] = %+v, want id=c1 text=甲", in.Chunks[0])
+ }
+}
+
+// TestKnowledgeCompiler_MergeChunksAcceptsTypedChunkDocs: mergeChunks must
+// forward the upstream typed []ChunkDoc carrier (converted to the wire map
+// form) ahead of the freshly compiled chunks, rather than silently dropping
+// the upstream chunks on an unexpected type.
+func TestKnowledgeCompiler_MergeChunksAcceptsTypedChunkDocs(t *testing.T) {
+ out := mergeChunks(map[string]any{
+ "chunks": []schema.ChunkDoc{{ID: "c1", Text: "甲"}},
+ }, []schema.ChunkDoc{{ID: "d1", Text: "编译"}})
+
+ merged, ok := out["chunks"].([]any)
+ if !ok {
+ t.Fatalf("mergeChunks output chunks type = %T, want []any", out["chunks"])
+ }
+ if len(merged) != 2 {
+ t.Fatalf("mergeChunks produced %d chunks, want 2 (1 upstream + 1 compiled)", len(merged))
+ }
+ first, ok := merged[0].(map[string]any)
+ if !ok {
+ t.Fatalf("merged[0] type = %T, want map[string]any", merged[0])
+ }
+ if first["id"] != "c1" || first["text"] != "甲" {
+ t.Fatalf("merged[0] = %+v, want upstream chunk id=c1 text=甲 (must not be dropped)", first)
+ }
+ second, ok := merged[1].(map[string]any)
+ if !ok {
+ t.Fatalf("merged[1] type = %T, want map[string]any", merged[1])
+ }
+ if second["id"] != "d1" {
+ t.Fatalf("merged[1] = %+v, want compiled chunk id=d1", second)
+ }
+}
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index 89a2c988c3..ad631378a4 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -173,21 +173,70 @@ func (d *ChunkDoc) UnmarshalJSON(data []byte) error {
}
func (d ChunkDoc) MarshalJSON() ([]byte, error) {
+ // Delegate to wireMap so the typed []ChunkDoc form and the []map[string]any
+ // wire form (ToMap / ChunkDocsToMaps) serialize identically by construction.
+ // Propagate marshal failures (e.g. invalid JSON in the RawMessage position
+ // fields) instead of silently emitting an empty object.
+ m, err := d.wireMapErr()
+ if err != nil {
+ return nil, err
+ }
+ return json.Marshal(m)
+}
+
+// wireMap is the error-tolerant wrapper used by ToMap, which cannot return an
+// error. On a marshal/unmarshal failure it falls back to an empty map,
+// preserving the historical ToMap behavior.
+func (d ChunkDoc) wireMap() map[string]any {
+ m, err := d.wireMapErr()
+ if err != nil {
+ return map[string]any{}
+ }
+ return m
+}
+
+// wireMapErr is the single source of truth for a ChunkDoc's wire representation,
+// shared by ToMap and MarshalJSON. It marshals the alias (a distinct type
+// without a MarshalJSON method) to avoid infinite recursion, then applies the
+// canonical normalization: Extra values are decoded, "text" is always emitted
+// (even for media-only chunks, so chunk identity/image upload share one wire
+// contract), the stale content_with_weight key is dropped, and structured
+// positions are decoded to their canonical Go form.
+func (d ChunkDoc) wireMapErr() (map[string]any, error) {
type alias ChunkDoc
- base, err := json.Marshal(alias(d))
+ b, err := json.Marshal(alias(d))
if err != nil {
return nil, err
}
- var raw map[string]json.RawMessage
- if err := json.Unmarshal(base, &raw); err != nil {
+ var out map[string]any
+ if err := json.Unmarshal(b, &out); err != nil {
return nil, err
}
- for k, v := range d.Extra {
- if _, exists := raw[k]; !exists {
- raw[k] = v
+ for k, raw := range d.Extra {
+ // ck_type is a retired parser key (#21259); a stray Extra entry must
+ // not resurrect it through the wire form.
+ if k == "ck_type" {
+ continue
+ }
+ // Fill-missing-only: typed fields already emitted by the alias marshal
+ // (id, doc_type_kwd, table, ...) take precedence over a colliding Extra
+ // key. UnmarshalJSON copies the incoming "id" into Extra (its deletion
+ // whitelist omits "id"), so an unconditional assignment would let a stale
+ // Extra copy shadow the real typed value through the GetChunks bridge.
+ if _, exists := out[k]; exists {
+ continue
}
+ out[k] = decodeExtraValue(raw)
}
- return json.Marshal(raw)
+ out["text"] = d.Text
+ delete(out, "content_with_weight")
+ if len(d.PDFPositions) > 0 {
+ out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)
+ }
+ if len(d.Positions) > 0 {
+ out["positions"] = decodeStructuredValue(d.Positions)
+ }
+ return out, nil
}
// ChunkDocFromMap decodes a free-form map into a typed ChunkDoc.
@@ -284,29 +333,7 @@ func chunkDocFromDocItem(it parser.DocItem) (ChunkDoc, error) {
// ToMap converts a typed chunk back to the generic map form used by
// runtime.Component boundaries.
func (d ChunkDoc) ToMap() map[string]any {
- b, err := json.Marshal(d)
- if err != nil {
- return map[string]any{}
- }
- var out map[string]any
- if err := json.Unmarshal(b, &out); err != nil {
- return map[string]any{}
- }
- for k, raw := range d.Extra {
- out[k] = decodeExtraValue(raw)
- }
- // Every pre-index chunk has one canonical text field. Keep the key even
- // for media-only chunks so chunk identity and image upload use the same
- // wire contract; contextual text remains explicit retrieval metadata.
- out["text"] = d.Text
- delete(out, "content_with_weight")
- if len(d.PDFPositions) > 0 {
- out["_pdf_positions"] = decodeStructuredValue(d.PDFPositions)
- }
- if len(d.Positions) > 0 {
- out["positions"] = decodeStructuredValue(d.Positions)
- }
- return out
+ return d.wireMap()
}
// ChunkDocsToMaps converts a typed chunk slice to the generic map form.
diff --git a/internal/ingestion/component/schema/envelope.go b/internal/ingestion/component/schema/envelope.go
new file mode 100644
index 0000000000..75ad651423
--- /dev/null
+++ b/internal/ingestion/component/schema/envelope.go
@@ -0,0 +1,79 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+// Canonical envelope keys. The canvas Component.Invoke contract keeps the
+// envelope as map[string]any, but each well-known key carries a single typed
+// payload. The "chunks" key (chunker/tokenizer output) has a typed accessor,
+// GetChunks, so downstream components read []ChunkDoc instead of bare type
+// assertions, eliminating the silent type-mismatch skips that the untyped map
+// form allowed. The "json" key (parser output) remains a plain wire key.
+//
+// These live in the schema package because it owns ChunkDoc and is imported by
+// every component subpackage (component, chunker, knowledge_compiler) without
+// introducing an import cycle.
+const (
+ KeyParseItems = "json"
+ KeyChunks = "chunks"
+)
+
+// GetChunks reads the canonical "chunks" envelope key. It accepts either a
+// typed []ChunkDoc or the legacy []map[string]any / []any form via
+// ChunkDocsFromAny, so migration can proceed producer-by-producer without a
+// flag day. A missing key, nil input, or a non-chunk carrier returns
+// (nil, false). For the []map[string]any form a malformed entry rejects the
+// whole carrier (nil, false) rather than being skipped; []any non-map entries
+// are skipped. It never panics.
+func GetChunks(in map[string]any) ([]ChunkDoc, bool) {
+ if in == nil {
+ return nil, false
+ }
+ v, ok := in[KeyChunks]
+ if !ok {
+ return nil, false
+ }
+ docs, ok, err := ChunkDocsFromAny(v)
+ if err != nil || !ok {
+ return nil, false
+ }
+ return docs, true
+}
+
+// ChunkCarrierToMaps returns a "chunks" carrier value as []map[string]any,
+// bridging every form losslessly: []map[string]any is returned verbatim (no
+// ChunkDoc round-trip, so non-ChunkDoc fields like content_with_weight
+// survive), []ChunkDoc is converted via ChunkDocsToMaps, and []any is
+// unwrapped to its map elements. Any other value, or a []any carrying a
+// non-map element, returns (nil, false).
+func ChunkCarrierToMaps(v any) ([]map[string]any, bool) {
+ switch t := v.(type) {
+ case []map[string]any:
+ return t, true
+ case []ChunkDoc:
+ return ChunkDocsToMaps(t), true
+ case []any:
+ out := make([]map[string]any, 0, len(t))
+ for _, it := range t {
+ m, ok := it.(map[string]any)
+ if !ok {
+ return nil, false
+ }
+ out = append(out, m)
+ }
+ return out, true
+ default:
+ return nil, false
+ }
+}
diff --git a/internal/ingestion/component/schema/envelope_parity_test.go b/internal/ingestion/component/schema/envelope_parity_test.go
new file mode 100644
index 0000000000..3f1f161072
--- /dev/null
+++ b/internal/ingestion/component/schema/envelope_parity_test.go
@@ -0,0 +1,142 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// TestChunksSerializationParity is the R1 regression gate: the []ChunkDoc wire
+// form and the []map[string]any wire form (ChunkDocsToMaps) must serialize
+// byte-semantically identically. Any
+// code that json.Marshal's the envelope (e.g. the agent canvas marshalling
+// node outputs) must produce JSON byte-semantically identical to the historical
+// []map[string]any wire form produced by ChunkDocsToMaps. If this drifts,
+// external consumers of the serialized envelope observe a shape change.
+func TestChunksSerializationParity(t *testing.T) {
+ cases := map[string]ChunkDoc{
+ "simple text": {
+ Text: "hello world",
+ DocType: "text",
+ ID: "id-1",
+ },
+ "empty text media-only": {
+ Text: "",
+ Image: "img/1.png",
+ ImgID: "imgid-1",
+ DocType: "image",
+ },
+ "with extra": {
+ Text: "body",
+ Extra: map[string]json.RawMessage{
+ "custom_key": json.RawMessage(`"v"`),
+ "page_number": json.RawMessage(`3`),
+ },
+ },
+ "with positions": {
+ Text: "positioned",
+ PDFPositions: json.RawMessage(`[[1,2,3,4]]`),
+ Positions: json.RawMessage(`[[10,20,30,40]]`),
+ },
+ "with table data": {
+ Text: "table chunk",
+ TableData: &entity.TableData{},
+ },
+ "token nums pointer": {
+ Text: "counted",
+ TKNums: intptr(42),
+ },
+ }
+ for name, d := range cases {
+ t.Run(name, func(t *testing.T) {
+ typedJSON, err := json.Marshal([]ChunkDoc{d})
+ if err != nil {
+ t.Fatalf("marshal []ChunkDoc: %v", err)
+ }
+ mapJSON, err := json.Marshal(ChunkDocsToMaps([]ChunkDoc{d}))
+ if err != nil {
+ t.Fatalf("marshal ChunkDocsToMaps: %v", err)
+ }
+ assertSemanticJSONEqual(t, typedJSON, mapJSON)
+ })
+ }
+}
+
+// assertSemanticJSONEqual compares two JSON documents for structural equality,
+// ignoring object key order (which is not semantically meaningful to consumers).
+func assertSemanticJSONEqual(t *testing.T, a, b []byte) {
+ t.Helper()
+ var av, bv any
+ if err := json.Unmarshal(a, &av); err != nil {
+ t.Fatalf("unmarshal A (%s): %v", a, err)
+ }
+ if err := json.Unmarshal(b, &bv); err != nil {
+ t.Fatalf("unmarshal B (%s): %v", b, err)
+ }
+ if !reflect.DeepEqual(av, bv) {
+ t.Fatalf("serialization parity mismatch:\n typed []ChunkDoc = %s\n map wire form = %s", a, b)
+ }
+}
+
+// TestChunkDocMarshalJSON_PropagatesError locks that a marshal failure (e.g.
+// invalid JSON bytes in the RawMessage position fields) is surfaced as an
+// error, not silently swallowed into an empty object. The pre-refactor
+// MarshalJSON propagated the error; the wireMap refactor must preserve that,
+// otherwise a corrupt chunk serializes as {} with no signal.
+func TestChunkDocMarshalJSON_PropagatesError(t *testing.T) {
+ cases := map[string]ChunkDoc{
+ "invalid PDFPositions": {Text: "x", PDFPositions: json.RawMessage("{not json")},
+ "invalid Positions": {Text: "x", Positions: json.RawMessage("[[1,2")},
+ }
+ for name, d := range cases {
+ t.Run(name, func(t *testing.T) {
+ if _, err := json.Marshal(d); err == nil {
+ t.Fatalf("expected marshal error for %s, got nil (error swallowed)", name)
+ }
+ })
+ }
+}
+
+// TestChunkDoc_RetiredCKTypeNotLeaked locks #21259's ck_type retirement
+// contract through the refactored wireMap path: a ck_type that ends up in
+// ChunkDoc.Extra (e.g. set programmatically) must not be re-emitted by ToMap /
+// ChunkDocsToMaps, while legitimate Extra keys survive.
+func TestChunkDoc_RetiredCKTypeNotLeaked(t *testing.T) {
+ d := ChunkDoc{
+ Text: "body",
+ Extra: map[string]json.RawMessage{
+ "ck_type": json.RawMessage(`"text"`), // retired parser key
+ "custom": json.RawMessage(`"keep"`),
+ },
+ }
+ m := d.ToMap()
+ if _, leaked := m["ck_type"]; leaked {
+ t.Fatalf("retired ck_type leaked from Extra through ToMap: %+v", m)
+ }
+ if m["custom"] != "keep" {
+ t.Fatalf("legitimate Extra key dropped: %+v", m)
+ }
+ // Same guarantee through the map wire form used by the envelope bridge.
+ maps := ChunkDocsToMaps([]ChunkDoc{d})
+ if _, leaked := maps[0]["ck_type"]; leaked {
+ t.Fatalf("retired ck_type leaked through ChunkDocsToMaps: %+v", maps[0])
+ }
+}
+
+func intptr(n int) *int { return &n }
diff --git a/internal/ingestion/component/schema/envelope_test.go b/internal/ingestion/component/schema/envelope_test.go
new file mode 100644
index 0000000000..4a0adbd76c
--- /dev/null
+++ b/internal/ingestion/component/schema/envelope_test.go
@@ -0,0 +1,133 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+ "testing"
+)
+
+func TestPutGetChunks_RoundTrip(t *testing.T) {
+ docs := []ChunkDoc{{Text: "a", ID: "1"}, {Text: "b", ID: "2"}}
+ out := map[string]any{KeyChunks: docs}
+
+ got, ok := GetChunks(out)
+ if !ok {
+ t.Fatal("GetChunks ok=false, want true")
+ }
+ if len(got) != 2 {
+ t.Fatalf("len=%d, want 2", len(got))
+ }
+ if got[0].Text != "a" || got[1].Text != "b" {
+ t.Fatalf("content mismatch: %+v", got)
+ }
+}
+
+func TestGetChunks_LegacyMapBridge(t *testing.T) {
+ legacy := []map[string]any{
+ {"text": "a", "id": "1"},
+ {"text": "b", "id": "2"},
+ }
+ out := map[string]any{KeyChunks: legacy}
+
+ got, ok := GetChunks(out)
+ if !ok {
+ t.Fatal("GetChunks ok=false for legacy []map[string]any, want true")
+ }
+ if len(got) != 2 {
+ t.Fatalf("len=%d, want 2", len(got))
+ }
+ if got[0].Text != "a" || got[1].Text != "b" {
+ t.Fatalf("content mismatch: %+v", got)
+ }
+ if got[0].ID != "1" {
+ t.Fatalf("id field lost through bridge: %+v", got[0])
+ }
+}
+
+func TestGetChunks_EmptyAndNil(t *testing.T) {
+ // nil input
+ if _, ok := GetChunks(nil); ok {
+ t.Fatal("nil input should return ok=false")
+ }
+ // absent key
+ if _, ok := GetChunks(map[string]any{}); ok {
+ t.Fatal("absent key should return ok=false")
+ }
+ // typed empty slice -> ok=true, len 0
+ out := map[string]any{KeyChunks: []ChunkDoc{}}
+ got, ok := GetChunks(out)
+ if !ok {
+ t.Fatal("empty typed []ChunkDoc should return ok=true")
+ }
+ if len(got) != 0 {
+ t.Fatalf("empty typed len=%d, want 0", len(got))
+ }
+ // legacy empty slice -> ok=true, len 0
+ out2 := map[string]any{KeyChunks: []map[string]any{}}
+ got2, ok2 := GetChunks(out2)
+ if !ok2 {
+ t.Fatal("empty legacy []map[string]any should return ok=true")
+ }
+ if len(got2) != 0 {
+ t.Fatalf("empty legacy len=%d, want 0", len(got2))
+ }
+}
+
+func TestGetChunks_Garbage(t *testing.T) {
+ // A slice of non-map values must not panic and must not yield partial
+ // chunks. ChunkDocsFromAny skips non-map entries, so we accept either
+ // (empty, true) or (nil, false) as safe.
+ out := map[string]any{KeyChunks: []any{"not-a-map", 42}}
+ got, ok := GetChunks(out)
+ if ok && len(got) != 0 {
+ t.Fatalf("garbage produced %d chunks, want 0", len(got))
+ }
+ // A scalar value for the key is not a chunk carrier.
+ out2 := map[string]any{KeyChunks: 42}
+ if _, ok2 := GetChunks(out2); ok2 {
+ t.Fatal("scalar chunks value should return ok=false")
+ }
+}
+
+func TestChunkCarrierToMaps(t *testing.T) {
+ // []map[string]any is returned verbatim — no ChunkDoc round-trip, so
+ // non-ChunkDoc fields (content_with_weight) survive.
+ maps := []map[string]any{{"id": "c1", "content_with_weight": "X", "text": "t"}}
+ got, ok := ChunkCarrierToMaps(maps)
+ if !ok || len(got) != 1 {
+ t.Fatalf("[]map carrier: ok=%v len=%d", ok, len(got))
+ }
+ if got[0]["content_with_weight"] != "X" {
+ t.Fatalf("[]map carrier lost content_with_weight: %+v", got[0])
+ }
+ // []ChunkDoc is converted to the map wire form.
+ got2, ok2 := ChunkCarrierToMaps([]ChunkDoc{{Text: "a", ID: "1"}})
+ if !ok2 || len(got2) != 1 || got2[0]["text"] != "a" {
+ t.Fatalf("[]ChunkDoc carrier: ok=%v got=%+v", ok2, got2)
+ }
+ // []any of maps is unwrapped.
+ got3, ok3 := ChunkCarrierToMaps([]any{map[string]any{"id": "z"}})
+ if !ok3 || len(got3) != 1 || got3[0]["id"] != "z" {
+ t.Fatalf("[]any carrier: ok=%v got=%+v", ok3, got3)
+ }
+ // []any containing a non-map element is rejected wholesale (matches toChunkMaps).
+ if _, ok4 := ChunkCarrierToMaps([]any{"not-a-map"}); ok4 {
+ t.Fatal("[]any with non-map element should be ok=false")
+ }
+ // A scalar / unrelated value is not a chunk carrier.
+ if _, ok5 := ChunkCarrierToMaps(42); ok5 {
+ t.Fatal("scalar value should be ok=false")
+ }
+}
diff --git a/internal/ingestion/component/schema/wiremap_extra_precedence_test.go b/internal/ingestion/component/schema/wiremap_extra_precedence_test.go
new file mode 100644
index 0000000000..087412f8d9
--- /dev/null
+++ b/internal/ingestion/component/schema/wiremap_extra_precedence_test.go
@@ -0,0 +1,124 @@
+// Copyright 2025 Infiniflow, Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+ "encoding/json"
+ "testing"
+)
+
+// R1: a typed field (id) must not be clobbered by a colliding Extra key.
+// Regression for yuzhichang's P1 review on PR #21273: wireMapErr previously
+// overwrote typed fields unconditionally, so a ChunkDoc whose Extra carried a
+// colliding "id" (which ChunkDocFromMap/UnmarshalJSON populate from legacy
+// []map[string]any input) emitted the wrong id into indexing.
+func TestWireMap_PreservesTypedIDOverExtra(t *testing.T) {
+ d := ChunkDoc{
+ ID: "real-id",
+ Text: "body",
+ Extra: map[string]json.RawMessage{"id": json.RawMessage(`"stale-id"`)},
+ }
+ m := d.ToMap()
+ if got := m["id"]; got != "real-id" {
+ t.Fatalf("ToMap id = %v, want %q (typed field must win over Extra)", got, "real-id")
+ }
+}
+
+// R3 (MarshalJSON path): MarshalJSON delegates to wireMapErr, so the same
+// precedence must hold on the JSON layer.
+func TestMarshalJSON_PreservesTypedIDOverExtra(t *testing.T) {
+ d := ChunkDoc{
+ ID: "real-id",
+ Text: "body",
+ Extra: map[string]json.RawMessage{"id": json.RawMessage(`"stale-id"`)},
+ }
+ b, err := json.Marshal(d)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ var out map[string]any
+ if err := json.Unmarshal(b, &out); err != nil {
+ t.Fatalf("unmarshal result: %v", err)
+ }
+ if got := out["id"]; got != "real-id" {
+ t.Fatalf("marshaled id = %v, want %q", got, "real-id")
+ }
+}
+
+// R2: fill-missing-only must still ADD Extra keys that are not typed fields.
+func TestWireMap_AddsNonTypedExtraKeys(t *testing.T) {
+ d := ChunkDoc{
+ ID: "real-id",
+ Text: "body",
+ Extra: map[string]json.RawMessage{"custom_key": json.RawMessage(`"v"`)},
+ }
+ m := d.ToMap()
+ if got := m["custom_key"]; got != "v" {
+ t.Fatalf("custom Extra key = %v, want %q (non-typed Extra keys must survive)", got, "v")
+ }
+ if got := m["id"]; got != "real-id" {
+ t.Fatalf("id = %v, want %q", got, "real-id")
+ }
+}
+
+// R4/R5 corner: empty and nil Extra must not panic and must still emit typed
+// fields.
+func TestWireMap_EmptyAndNilExtra(t *testing.T) {
+ d1 := ChunkDoc{ID: "a", Text: "t", Extra: map[string]json.RawMessage{}}
+ if got := d1.ToMap()["id"]; got != "a" {
+ t.Fatalf("empty Extra id = %v, want a", got)
+ }
+ d2 := ChunkDoc{ID: "b", Text: "t"}
+ if got := d2.ToMap()["id"]; got != "b" {
+ t.Fatalf("nil Extra id = %v, want b", got)
+ }
+}
+
+// R4: ck_type in Extra must still be skipped (unrelated to the precedence fix,
+// locked so the change does not resurrect the retired key).
+func TestWireMap_SkipsRetiredCKType(t *testing.T) {
+ d := ChunkDoc{
+ ID: "a",
+ Text: "t",
+ Extra: map[string]json.RawMessage{"ck_type": json.RawMessage(`"text"`)},
+ }
+ m := d.ToMap()
+ if _, ok := m["ck_type"]; ok {
+ t.Fatalf("ck_type leaked: %+v", m)
+ }
+}
+
+// R-bridge (reviewer's exact scenario): a legacy []map[string]any chunk whose
+// id is later mutated must keep the mutated id through the GetChunks -> ToMap
+// bridge, not the stale Extra copy that UnmarshalJSON populated. UnmarshalJSON's
+// deletion whitelist omits "id", so decoding a legacy map copies id into both
+// ChunkDoc.ID and ChunkDoc.Extra["id"]; a subsequent re-id step that updates
+// only the typed field must not be shadowed by the stale Extra copy.
+func TestGetChunksBridge_PreservesMutatedID(t *testing.T) {
+ legacy := []map[string]any{{"id": "original-id", "text": "body"}}
+ out := map[string]any{KeyChunks: legacy}
+
+ typed, ok := GetChunks(out)
+ if !ok || len(typed) != 1 {
+ t.Fatalf("GetChunks ok=%v len=%d, want ok=true len=1", ok, len(typed))
+ }
+ // Simulate a re-id step that updates the typed field but leaves Extra stale.
+ typed[0].ID = "reassigned-id"
+
+ maps := ChunkDocsToMaps(typed)
+ if got := maps[0]["id"]; got != "reassigned-id" {
+ t.Fatalf("bridged id = %v, want %q (mutated typed id must survive the Extra copy)", got, "reassigned-id")
+ }
+}
diff --git a/internal/ingestion/component/slice3_integration_test.go b/internal/ingestion/component/slice3_integration_test.go
index 0a6d546c45..820485314d 100644
--- a/internal/ingestion/component/slice3_integration_test.go
+++ b/internal/ingestion/component/slice3_integration_test.go
@@ -22,6 +22,7 @@ package component
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -44,7 +45,7 @@ func TestTokenizer_DropsContentWithWeightOnlyChunks(t *testing.T) {
if err != nil {
t.Fatalf("Tokenizer.Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 0 {
t.Fatalf("content_with_weight-only chunks must be dropped, got %d chunk(s)", len(chunks))
}
@@ -70,7 +71,7 @@ func TestTokenizer_DoesNotChangeChunkText(t *testing.T) {
if err != nil {
t.Fatalf("Tokenizer.Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) == 0 {
t.Fatal("no chunks emitted")
}
diff --git a/internal/ingestion/component/tokenizer_nonpersist_test.go b/internal/ingestion/component/tokenizer_nonpersist_test.go
index 7eee084374..4ec5efd757 100644
--- a/internal/ingestion/component/tokenizer_nonpersist_test.go
+++ b/internal/ingestion/component/tokenizer_nonpersist_test.go
@@ -24,6 +24,7 @@ package component
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -77,7 +78,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {
if got := debugOut["embedding_token_consumption"]; got != nil {
t.Errorf("embedding_token_consumption = %v under debug; must be omitted when embedding is skipped", got)
}
- got, ok := debugOut["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(debugOut["chunks"])
if !ok || len(got) != 2 {
t.Fatalf("chunks malformed under debug: %v", debugOut["chunks"])
}
@@ -96,7 +97,7 @@ func TestTokenizerComponent_DebugSkipsEmbedding(t *testing.T) {
if got := kbOut["embedding_token_consumption"]; got == nil {
t.Error("embedding_token_consumption missing with kb_id set; embedding accounting must run")
}
- kbChunks, ok := kbOut["chunks"].([]map[string]any)
+ kbChunks, ok := schema.ChunkCarrierToMaps(kbOut["chunks"])
if !ok || len(kbChunks) != 2 {
t.Fatalf("chunks malformed with kb_id: %v", kbOut["chunks"])
}
diff --git a/internal/ingestion/component/tokenizer_table_tag_test.go b/internal/ingestion/component/tokenizer_table_tag_test.go
index f9201c4dda..5652ab7983 100644
--- a/internal/ingestion/component/tokenizer_table_tag_test.go
+++ b/internal/ingestion/component/tokenizer_table_tag_test.go
@@ -20,6 +20,7 @@ package component
import (
"context"
+ "ragflow/internal/ingestion/component/schema"
"testing"
)
@@ -50,7 +51,7 @@ func TestTokenizerComponent_Embedding_SkipsOnlyEmptyTableChunk(t *testing.T) {
if len(stub.callInputs) != 2 || len(stub.callInputs[1]) != 1 || stub.callInputs[1][0] != "real text" {
t.Fatalf("content call inputs = %v, want [real text]", stub.callInputs)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("output chunks = %d, want 2 (a skipped chunk stays in the payload)", len(chunks))
}
diff --git a/internal/ingestion/component/tokenizer_test.go b/internal/ingestion/component/tokenizer_test.go
index bc4861bf89..01dd954a3d 100644
--- a/internal/ingestion/component/tokenizer_test.go
+++ b/internal/ingestion/component/tokenizer_test.go
@@ -76,7 +76,7 @@ func TestTokenizerComponent_Invoke_HappyPath(t *testing.T) {
t.Fatalf("Invoke: %v", err)
}
- gotChunks, ok := out["chunks"].([]map[string]any)
+ gotChunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok {
t.Fatalf("chunks type = %T, want []map[string]any", out["chunks"])
}
@@ -142,7 +142,7 @@ func TestTokenizerComponent_Invoke_Unicode(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- gotChunks, _ := out["chunks"].([]map[string]any)
+ gotChunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(gotChunks) != len(inputs) {
t.Fatalf("chunks len = %d, want %d", len(gotChunks), len(inputs))
}
@@ -172,7 +172,7 @@ func TestTokenizerComponent_Invoke_TextPayload(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
@@ -198,7 +198,7 @@ func TestTokenizerComponent_Invoke_JSONPayload(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) != 2 {
t.Fatalf("chunks len = %d, want 2", len(got))
}
@@ -253,7 +253,7 @@ func TestTokenizerComponent_Invoke_FullTextOnly(t *testing.T) {
if out["embedding_token_consumption"] != nil {
t.Errorf("embedding_token_consumption should be absent, got %v", out["embedding_token_consumption"])
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) == 0 || got[0]["content_ltks"] == nil {
t.Errorf("content_ltks missing: %v", got)
}
@@ -282,7 +282,7 @@ func TestTokenizerComponent_Invoke_KeywordSplitCommaOnly(t *testing.T) {
if err != nil {
t.Fatalf("Invoke(%q): %v", keywords, err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
@@ -325,7 +325,7 @@ func TestTokenizerComponent_Invoke_FullTextAndEmbedding(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
@@ -472,7 +472,7 @@ func TestTokenizerComponent_Smoke_EndToEnd(t *testing.T) {
t.Errorf("elapsed %v exceeds 5s ceiling", elapsed)
}
- got, ok := out["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(got) != 1 {
t.Fatalf("chunks output malformed: %v", out["chunks"])
}
@@ -513,7 +513,7 @@ func TestTokenizerComponent_Embedding_MergesTitleAndContentVectors(t *testing.T)
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
want0 := []float64{1.9, 3.8}
want1 := []float64{3.7, 5.6}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want0) {
@@ -549,7 +549,7 @@ func TestTokenizerComponent_Embedding_UsesFilenameWeight(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
want := []float64{3.5, 3.5}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {
t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)
@@ -583,7 +583,7 @@ func TestTokenizerComponent_Embedding_EmptyNameWarnsAndUsesContentVector(t *test
if got := logs.FilterMessageSnippet("empty name provided from upstream").Len(); got != 1 {
t.Fatalf("empty-name warnings = %d, want 1 (entries: %v)", got, logs.All())
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
want := []float64{2, 4}
if !floatSliceClose(got[0]["q_2_vec"].([]float64), want) {
t.Fatalf("q_2_vec = %v, want %v", got[0]["q_2_vec"], want)
@@ -665,7 +665,7 @@ func TestTokenizerComponent_Embedding_SkipsEmptyCleanedTextsButReturnsZeroWhenAl
if got := out["embedding_token_consumption"]; got != 0 {
t.Fatalf("embedding_token_consumption = %v, want 0", got)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
for i, ck := range got {
if _, ok := ck["q_2_vec"]; ok {
t.Fatalf("chunk[%d] should not have vector: %v", i, ck)
@@ -762,8 +762,16 @@ func TestTokenizerComponent_InstanceResolversDoNotLeakAcrossComponents(t *testin
if err != nil {
t.Fatalf("Invoke B: %v", err)
}
- vecA := outA["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)
- vecB := outB["chunks"].([]map[string]any)[0]["q_2_vec"].([]float64)
+ chunksA, okA := schema.ChunkCarrierToMaps(outA["chunks"])
+ if !okA || len(chunksA) == 0 {
+ t.Fatalf("A: no chunks emitted: %v", outA["chunks"])
+ }
+ vecA, _ := chunksA[0]["q_2_vec"].([]float64)
+ chunksB, okB := schema.ChunkCarrierToMaps(outB["chunks"])
+ if !okB || len(chunksB) == 0 {
+ t.Fatalf("B: no chunks emitted: %v", outB["chunks"])
+ }
+ vecB, _ := chunksB[0]["q_2_vec"].([]float64)
if reflect.DeepEqual(vecA, vecB) {
t.Fatalf("instance resolvers leaked: vecA=%v vecB=%v", vecA, vecB)
}
diff --git a/internal/ingestion/component/tokenizer_unit_test.go b/internal/ingestion/component/tokenizer_unit_test.go
index 2db6eeb52b..233e241753 100644
--- a/internal/ingestion/component/tokenizer_unit_test.go
+++ b/internal/ingestion/component/tokenizer_unit_test.go
@@ -174,7 +174,10 @@ func TestTokenizerComponent_Invoke_EmptyChunks(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
+ if !ok {
+ t.Fatalf("chunks carrier missing or invalid: %v", out["chunks"])
+ }
if len(chunks) != 0 {
t.Errorf("chunks len = %d, want 0", len(chunks))
}
@@ -201,7 +204,10 @@ func TestTokenizerComponent_Invoke_NilChunks(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks, _ := out["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(out["chunks"])
+ if !ok {
+ t.Fatalf("chunks carrier missing or invalid: %v", out["chunks"])
+ }
if len(chunks) != 0 {
t.Errorf("chunks len = %d, want 0", len(chunks))
}
@@ -225,7 +231,7 @@ func TestTokenizerComponent_Invoke_EmbeddingOnly(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, _ := out["chunks"].([]map[string]any)
+ got, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(got) != 1 {
t.Fatalf("chunks len = %d, want 1", len(got))
}
@@ -264,7 +270,7 @@ func TestTokenizerComponent_FullTextIncludesMediaContext(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if got := chunks[0]["content_ltks"]; got != "above table body below" {
t.Fatalf("content_ltks = %q, want contextual text", got)
}
@@ -555,7 +561,7 @@ func TestChunkOrderInt_EmbeddingOnly(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("want 2 chunks, got %d", len(chunks))
}
@@ -598,7 +604,7 @@ func TestChunksFromTokenizerUpstream_FiltersPhantomChunks(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("want 1 chunk (5 phantoms filtered), got %d: %#v", len(chunks), chunks)
}
@@ -628,7 +634,7 @@ func TestTokenizerComponent_KeepsContextOnlyMediaChunk(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 1 {
t.Fatalf("chunks = %#v, want context-bearing media chunk retained", chunks)
}
@@ -689,7 +695,7 @@ func TestChunkOrderInt_AllPathsUnconditional(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != tc.wantChunks {
t.Fatalf("want %d chunks, got %d: %#v", tc.wantChunks, len(chunks), chunks)
}
@@ -742,7 +748,7 @@ func TestChunkOrderInt_A5FilterKeepsSequenceContiguous(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- chunks := out["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(out["chunks"])
if len(chunks) != 2 {
t.Fatalf("want 2 surviving chunks, got %d: %#v", len(chunks), chunks)
}
@@ -790,7 +796,7 @@ func TestTokenizerComponent_ImportantKwd_CommaOnly(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, ok := out["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(got) != 1 {
t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])
}
@@ -843,7 +849,7 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, ok := out["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(got) != 1 {
t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])
}
@@ -874,7 +880,7 @@ func TestTokenizerComponent_ImportantKwd_PreservesEmptyElements(t *testing.T) {
if err != nil {
t.Fatalf("Invoke(empty keywords): %v", err)
}
- gotEmpty, ok := outEmpty["chunks"].([]map[string]any)
+ gotEmpty, ok := schema.ChunkCarrierToMaps(outEmpty["chunks"])
if !ok || len(gotEmpty) != 1 {
t.Fatalf("empty chunks = %v, want 1 chunk", outEmpty["chunks"])
}
@@ -958,7 +964,7 @@ func TestTokenizerComponent_ImportantKwd_Bounded(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- got, ok := out["chunks"].([]map[string]any)
+ got, ok := schema.ChunkCarrierToMaps(out["chunks"])
if !ok || len(got) != 1 {
t.Fatalf("chunks = %v, want 1 chunk", out["chunks"])
}
diff --git a/internal/ingestion/pipeline/image_ocr_test.go b/internal/ingestion/pipeline/image_ocr_test.go
index 351cab397e..42856485a1 100644
--- a/internal/ingestion/pipeline/image_ocr_test.go
+++ b/internal/ingestion/pipeline/image_ocr_test.go
@@ -36,6 +36,7 @@ import (
"ragflow/internal/entity"
"ragflow/internal/ingestion/component"
"ragflow/internal/ingestion/component/chunker"
+ "ragflow/internal/ingestion/component/schema"
)
type imageOCREmbedder struct{ texts []string }
@@ -193,7 +194,7 @@ func TestImageOCRSavedConfigToSearchableChunk(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- chunks := indexable["chunks"].([]map[string]any)
+ chunks, _ := schema.ChunkCarrierToMaps(indexable["chunks"])
if len(chunks) != 1 || chunks[0]["text"] != text {
t.Fatalf("searchable OCR text was lost: %+v", chunks)
}
diff --git a/internal/ingestion/pipeline/real_storage_integration_test.go b/internal/ingestion/pipeline/real_storage_integration_test.go
index 0881c549e2..3d5744d0d8 100644
--- a/internal/ingestion/pipeline/real_storage_integration_test.go
+++ b/internal/ingestion/pipeline/real_storage_integration_test.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/entity"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
"ragflow/internal/server"
"ragflow/internal/server/config"
@@ -118,7 +119,7 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -187,7 +188,7 @@ func TestPipelineRun_TemplateGeneral_RealMySQLMinIO_OutputShape(t *testing.T) {
if got := chunkerState["output_format"]; got != "chunks" {
t.Fatalf("chunker output_format = %v, want chunks", got)
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
diff --git a/internal/ingestion/pipeline/template_integration_test.go b/internal/ingestion/pipeline/template_integration_test.go
index 31cd18920d..77f34d9798 100644
--- a/internal/ingestion/pipeline/template_integration_test.go
+++ b/internal/ingestion/pipeline/template_integration_test.go
@@ -40,6 +40,7 @@ import (
doctype "ragflow/internal/deepdoc/parser/type"
componentpkg "ragflow/internal/ingestion/component"
_ "ragflow/internal/ingestion/component/chunker"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/ingestion/testutil"
"ragflow/internal/parser/parser"
"ragflow/internal/storage"
@@ -103,7 +104,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -141,7 +142,7 @@ func TestPipelineRun_TemplateGeneral_RealComponents(t *testing.T) {
if got := chunkerState["output_format"]; got != "chunks" {
t.Fatalf("chunker output_format = %v, want chunks", got)
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
@@ -230,7 +231,7 @@ func TestPipelineRun_TemplateOne_RealComponents(t *testing.T) {
if got := chunkerState["output_format"]; got != "chunks" {
t.Fatalf("chunker output_format = %v, want chunks", got)
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != 1 {
t.Fatalf("chunker chunks = %T/%v, want 1 item", chunkerState["chunks"], chunkerState["chunks"])
}
@@ -277,7 +278,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok || len(chunks) != 1 {
t.Fatalf("chunks = %T/%v, want 1 merged chunk", payload["chunks"], payload["chunks"])
}
@@ -329,7 +330,7 @@ func TestPipelineRun_TemplateOne_RealComponents_PDFDeepdocChunking(t *testing.T)
if !ok {
t.Fatal("missing OneChunker:DryDrinksVisit state")
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != 1 {
t.Fatalf("chunker chunks = %T/%v, want 1 item", chunkerState["chunks"], chunkerState["chunks"])
}
@@ -376,7 +377,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -425,7 +426,7 @@ func TestPipelineRun_TemplateManual_RealComponents(t *testing.T) {
if !ok {
t.Fatal("missing ManualChunker:NineInsectsFind state")
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
@@ -474,7 +475,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -508,7 +509,7 @@ func TestPipelineRun_TemplateLaws_RealComponents(t *testing.T) {
if !ok {
t.Fatal("missing TitleChunker:SpicyKeysKick state")
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
@@ -557,7 +558,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -589,7 +590,7 @@ func TestPipelineRun_TemplatePaper_RealComponents(t *testing.T) {
if !ok {
t.Fatal("missing TitleChunker:SparklySchoolsTravel state")
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
@@ -638,7 +639,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
@@ -675,7 +676,7 @@ func TestPipelineRun_TemplateBook_RealComponents(t *testing.T) {
if !ok {
t.Fatal("missing TitleChunker:GrumpyGarlicsBake state")
}
- chunkerChunks, ok := chunkerState["chunks"].([]map[string]any)
+ chunkerChunks, ok := schema.ChunkCarrierToMaps(chunkerState["chunks"])
if !ok || len(chunkerChunks) != len(wantChunkTexts) {
t.Fatalf("chunker chunks = %T/%v, want %d items", chunkerState["chunks"], chunkerState["chunks"], len(wantChunkTexts))
}
@@ -741,7 +742,7 @@ func TestPipelineRun_AllIngestionTemplates_RealComponentsSmoke(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok || len(chunks) == 0 {
t.Fatalf("chunks = %T/%v, want non-empty []map[string]any", payload["chunks"], payload["chunks"])
}
@@ -1020,7 +1021,7 @@ func assertTokenizerTerminalChunk(t *testing.T, payload map[string]any, name, wa
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}
diff --git a/internal/ingestion/task/indexdoc/normalize.go b/internal/ingestion/task/indexdoc/normalize.go
index 0395d1b553..aee0b78ed7 100644
--- a/internal/ingestion/task/indexdoc/normalize.go
+++ b/internal/ingestion/task/indexdoc/normalize.go
@@ -20,6 +20,7 @@ import (
"fmt"
"ragflow/internal/common"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
)
@@ -33,6 +34,15 @@ func NormalizeChunks(output map[string]any) []map[string]any {
if chunks, ok := output["chunks"].([]map[string]any); ok {
return DeepCopyChunks(chunks)
}
+ // Typed []ChunkDoc carrier (post-migration chunker/tokenizer output):
+ // normalize it to the map wire form. Match the concrete typed slice only —
+ // do NOT use schema.GetChunks here, because it also accepts the legacy
+ // []any carrier and would round-trip it through ChunkDoc, dropping
+ // non-ChunkDoc fields (e.g. content_with_weight). The legacy []any carrier
+ // must stay on the value-preserving toChunkMaps path below.
+ if typed, ok := output["chunks"].([]schema.ChunkDoc); ok {
+ return DeepCopyChunks(schema.ChunkDocsToMaps(typed))
+ }
if chunks, ok := toChunkMaps(output["chunks"]); ok {
return DeepCopyChunks(chunks)
}
diff --git a/internal/ingestion/task/indexdoc/normalize_test.go b/internal/ingestion/task/indexdoc/normalize_test.go
index 8555f2807b..b1ecfcfa5b 100644
--- a/internal/ingestion/task/indexdoc/normalize_test.go
+++ b/internal/ingestion/task/indexdoc/normalize_test.go
@@ -3,6 +3,7 @@ package indexdoc
import (
"testing"
+ "ragflow/internal/ingestion/component/schema"
"ragflow/internal/parser/parser"
)
@@ -26,6 +27,54 @@ func TestNormalizeChunks_ChunksFormat(t *testing.T) {
}
}
+// TestNormalizeChunks_TypedChunkDocCarrier: once a chunker/tokenizer emits the
+// typed []ChunkDoc carrier, NormalizeChunks must bridge it to the map wire form
+// instead of silently dropping every chunk (which would yield zero index
+// writes).
+func TestNormalizeChunks_TypedChunkDocCarrier(t *testing.T) {
+ input := map[string]any{
+ "chunks": []schema.ChunkDoc{
+ {Text: "typed-a", ID: "1"},
+ {Text: "typed-b", ID: "2"},
+ },
+ }
+ result := NormalizeChunks(input)
+ if len(result) != 2 {
+ t.Fatalf("len = %d, want 2 (typed []ChunkDoc carrier)", len(result))
+ }
+ if result[0]["text"] != "typed-a" {
+ t.Errorf("result[0][\"text\"] = %q, want \"typed-a\"", result[0]["text"])
+ }
+}
+
+// TestNormalizeChunks_PreservesAnyCarrierFields locks the legacy []any-of-map
+// carrier path: such a carrier must be preserved verbatim (matching the
+// historical toChunkMaps behavior), NOT round-tripped through ChunkDoc. The
+// ChunkDoc round-trip drops non-ChunkDoc fields (e.g. content_with_weight) and
+// forces an empty "text" key, which the index path would then persist as an
+// empty content_with_weight.
+func TestNormalizeChunks_PreservesAnyCarrierFields(t *testing.T) {
+ input := map[string]any{
+ "chunks": []any{
+ map[string]any{"id": "c1", "content_with_weight": "hello world"},
+ map[string]any{"id": "c2", "text": "body", "content_with_weight": "rich text"},
+ },
+ }
+ result := NormalizeChunks(input)
+ if len(result) != 2 {
+ t.Fatalf("len = %d, want 2", len(result))
+ }
+ if result[0]["content_with_weight"] != "hello world" {
+ t.Errorf("result[0] content_with_weight lost: %+v", result[0])
+ }
+ if _, hasText := result[0]["text"]; hasText {
+ t.Errorf("result[0] must not gain a spurious empty text key: %+v", result[0])
+ }
+ if result[1]["text"] != "body" || result[1]["content_with_weight"] != "rich text" {
+ t.Errorf("result[1] fields lost: %+v", result[1])
+ }
+}
+
func TestNormalizeChunks_JSONFormat(t *testing.T) {
input := map[string]any{
"json": []map[string]any{
diff --git a/internal/ingestion/task/pipeline_run_test.go b/internal/ingestion/task/pipeline_run_test.go
index 41c47ac7ae..1df0e8278b 100644
--- a/internal/ingestion/task/pipeline_run_test.go
+++ b/internal/ingestion/task/pipeline_run_test.go
@@ -23,6 +23,7 @@ import (
"ragflow/internal/dao"
"ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
pipelinepkg "ragflow/internal/ingestion/pipeline"
)
@@ -72,7 +73,7 @@ func TestExtractPipelinePayload_UnwrapsSingleTerminalOutput(t *testing.T) {
if got := payload["output_format"]; got != "chunks" {
t.Fatalf("output_format = %v, want chunks", got)
}
- chunks, ok := payload["chunks"].([]map[string]any)
+ chunks, ok := schema.ChunkCarrierToMaps(payload["chunks"])
if !ok {
t.Fatalf("chunks = %T, want []map[string]any", payload["chunks"])
}