RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): type ParseResult.JSON as []DocItem boundary adapter (#20998)
xugangqiangeabad0c74c73
diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.goindex ef60f8aabc..64a7709391 100644--- a/internal/ingestion/component/chunker/sheet_identity_test.go+++ b/internal/ingestion/component/chunker/sheet_identity_test.go@@ -49,12 +49,12 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {factory func(map[string]any) (runtime.Component, error)}{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {t.Run(source.name+"/"+tc.name, func(t *testing.T) {- batches := [][]map[string]any{source.result.JSON}+ batches := [][]map[string]any{parser.ItemsToMaps(source.result.JSON)}// One preserves positions only for a single source item.if tc.name == "One" {batches = nilfor _, item := range source.result.JSON {- batches = append(batches, []map[string]any{item})+ batches = append(batches, []map[string]any{item.ToMap()})}}for _, items := range batches {@@ -147,7 +147,9 @@ func TestQAChunkerSheetNameOnlyPreservesSourceRows(t *testing.T) {t.Fatalf("parser items=%d, want 1", len(parsed.JSON))}// Keep real parser selections, but exercise the supported name-only identity.- delete(parsed.JSON[0], "sheet_index")+ m := parsed.JSON[0].ToMap()+ delete(m, "sheet_index")+ parsed.JSON[0] = parser.RawItem{M: m}comp, err := NewQAChunker(nil)if err != nil {t.Fatal(err)diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.goindex 1b8cd2fc72..be1ca2a73a 100644--- a/internal/ingestion/component/chunker/token.go+++ b/internal/ingestion/component/chunker/token.go@@ -85,6 +85,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/ingestion/component/globals""ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser""ragflow/internal/parser/tableutil""ragflow/internal/parser/chunk"@@ -247,7 +248,22 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputsfunc decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstream, error) {var out schema.ChunkerFromUpstream- data, err := json.Marshal(stripChunkerRuntimeTimestamps(inputs))+ // Fast path: the parser emits typed []parser.DocItem. Convert it to+ // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which reads the+ // item's typed fields and never calls DocItem.ToMap), then let the JSON+ // marshal below serialize the already-typed ChunkDoc. This keeps the+ // chunker free of the historical map intermediate for the in-memory+ // Parser -> Chunker leg. When inputs["json"] is already []map[string]any+ // (legacy fixtures, round-tripped JSON, or absent), the original path runs.+ inputs = stripChunkerRuntimeTimestamps(inputs)+ if docs, ok := inputs["json"].([]parser.DocItem); ok && len(docs) > 0 {+ conv, _, err := schema.ChunkDocsFromDocItem(docs)+ if err != nil {+ return out, err+ }+ inputs["json"] = conv+ }+ data, err := json.Marshal(inputs)if err != nil {return out, err}diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.goindex d5c0cd2afe..419f3a9180 100644--- a/internal/ingestion/component/media_dispatch.go+++ b/internal/ingestion/component/media_dispatch.go@@ -125,8 +125,10 @@ func maybeDispatchImage(return parsed, true, fmt.Errorf("parser: image parser returned no image item")}parsed.OutputFormat = "json"- imageData, _ := parsed.JSON[0]["image"].(string)- parsed.JSON[0]["text"] = text+ imageData, _ := parsed.JSON[0].ToMap()["image"].(string)+ m := parsed.JSON[0].ToMap()+ m["text"] = text+ parsed.JSON[0] = parser.DocItemFromMap(m)if err != nil {parsed.Warnings = append(parsed.Warnings, fmt.Sprintf("image OCR unavailable: %v", err))} else if useOCR && strings.TrimSpace(text) == "" {@@ -155,7 +157,9 @@ func maybeDispatchImage(description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, imageVision)parsed.Warnings = append(parsed.Warnings, warnings...)if description != "" {- appendItemText(parsed.JSON[0], description)+ m := parsed.JSON[0].ToMap()+ appendItemText(m, description)+ parsed.JSON[0] = parser.DocItemFromMap(m)} else if strings.TrimSpace(text) == "" {// Enhancement was permitted and the model produced nothing usable,// so the item still has no text at all. Without this the chunk is@@ -361,10 +365,7 @@ func maybeDispatchAudio(return parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{- "text": transcription,- "doc_type_kwd": "text",- }},+ JSON: []parser.DocItem{parser.NewTextJSONItem(transcription)},}, true, nil}diff --git a/internal/ingestion/component/media_dispatch_integration_test.go b/internal/ingestion/component/media_dispatch_integration_test.goindex 2e6e53b490..69e808f3f7 100644--- a/internal/ingestion/component/media_dispatch_integration_test.go+++ b/internal/ingestion/component/media_dispatch_integration_test.go@@ -46,12 +46,13 @@ func imageSetupsWithSwitch(ocrEnabled bool) map[string]schema.ParserSetup {func assertNativeOCRTextAndImage(t *testing.T, result parser.ParseResult) {t.Helper()- text, _ := result.JSON[0]["text"].(string)+ maps := parser.ItemsToMaps(result.JSON)+ text, _ := maps[0]["text"].(string)if strings.Join(strings.Fields(strings.ToUpper(text)), "") != pictureOCRExpectedText {t.Fatalf("native OCR text = %q, want HELLO WORLD: warnings = %v", text, result.Warnings)}- if result.JSON[0]["image"] == "" || result.JSON[0]["doc_type_kwd"] != "image" {- t.Fatalf("missing image attachment: %+v", result.JSON[0])+ if maps[0]["image"] == "" || maps[0]["doc_type_kwd"] != "image" {+ t.Fatalf("missing image attachment: %+v", maps[0])}}@@ -101,11 +102,12 @@ func TestMaybeDispatchImageNativeSwitchOffRunsNoOCR(t *testing.T) {if !handled || len(result.JSON) != 1 {t.Fatalf("result = %+v, handled = %v", result, handled)}- if text, _ := result.JSON[0]["text"].(string); text != "" {+ maps := parser.ItemsToMaps(result.JSON)+ if text, _ := maps[0]["text"].(string); text != "" {t.Fatalf("OCR must not run with the switch off, got text %q", text)}- if result.JSON[0]["image"] == "" {- t.Fatalf("image attachment must survive: %+v", result.JSON[0])+ if maps[0]["image"] == "" {+ t.Fatalf("image attachment must survive: %+v", maps[0])}if !strings.Contains(strings.Join(result.Warnings, "\n"), "the Tokenizer will discard this item") {t.Fatalf("expected a no-text-source warning, got %v", result.Warnings)diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.goindex 9e5d844e87..38a3b4f775 100644--- a/internal/ingestion/component/media_dispatch_test.go+++ b/internal/ingestion/component/media_dispatch_test.go@@ -130,7 +130,8 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {if err != nil {t.Fatal(err)}- items, _ := result["json"].([]map[string]any)+ jsonDocs := result["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if len(items) != 1 || items[0]["text"] != tc.want {t.Fatalf("items = %+v, want text %q", items, tc.want)}@@ -200,13 +201,13 @@ func TestMaybeDispatchImage_ParseMethodModelDoesNotOverrideGlobalEnhancement(t *if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- text, _ := res.JSON[0]["text"].(string)+ text, _ := res.JSON[0].ToMap()["text"].(string)if strings.TrimSpace(text) != "" {t.Fatalf("image item text = %q, want empty: enhancement is off and OCR is off", text)}// The image itself is still carried; only the description is withheld, so// turning the switch on later recovers the caption without a re-upload.- if res.JSON[0]["image"] == "" || res.JSON[0]["doc_type_kwd"] != "image" {+ if res.JSON[0].ToMap()["image"] == "" || res.JSON[0].ToMap()["doc_type_kwd"] != "image" {t.Fatalf("image attachment lost: %+v", res.JSON[0])}// #20668 reported this exact configuration producing zero chunks with no@@ -241,7 +242,7 @@ func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {if resolveModelConfigCalled {t.Fatal("resolveModelConfig must not be called for parse_method=ocr without enhancement")}- if len(res.JSON) != 1 || res.JSON[0]["text"] != "OCR text" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "OCR text" {t.Fatalf("res = %+v, want OCR text", res.JSON)}}@@ -350,7 +351,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1 (image result must be a single JSON item)", len(res.JSON))}- if txt, _ := res.JSON[0]["text"].(string); txt == "" {+ if txt, _ := res.JSON[0].ToMap()["text"].(string); txt == "" {t.Fatalf("expected non-empty combined text in JSON[0][\"text\"]")}@@ -505,7 +506,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- item := res.JSON[0]+ item := res.JSON[0].ToMap()if got, _ := item["doc_type_kwd"].(string); got != "image" {t.Errorf("doc_type_kwd = %q, want \"image\"", got)}@@ -608,10 +609,10 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1 (transcription must be carried as a JSON item)", len(res.JSON))}- if got, _ := res.JSON[0]["text"].(string); got != want {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {t.Fatalf("JSON[0].text = %q, want %q", got, want)}- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {+ if got, _ := res.JSON[0].ToMap()["doc_type_kwd"].(string); got != "text" {t.Fatalf("JSON[0].doc_type_kwd = %q, want text", got)}}@@ -651,7 +652,7 @@ func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- if got, _ := res.JSON[0]["text"].(string); got != want {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {t.Fatalf("JSON[0].text = %q, want %q", got, want)}}@@ -804,7 +805,7 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {if tenantResolverCalled {t.Error("tenant default resolver must not be called when audio setup vlm.llm_id is set")}- if len(res.JSON) != 1 || res.JSON[0]["text"] != "transcribed" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "transcribed" {t.Fatalf("unexpected audio result: %#v", res.JSON)}}@@ -866,7 +867,7 @@ func TestMaybeDispatchImageWithoutOCRTextKeepsImage(t *testing.T) {if err != nil {t.Fatal(err)}- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" || len(result.Warnings) == 0 {+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "" || result.JSON[0].ToMap()["image"] == "" || len(result.Warnings) == 0 {t.Fatalf("method %q, available %v: result = %+v, handled = %v", method, available, result, handled)}}@@ -890,7 +891,7 @@ func TestMaybeDispatchImageUnhealthyOCRKeepsImage(t *testing.T) {if err != nil || result.Err != nil {t.Fatalf("unhealthy OCR should degrade without an error: err = %v, result.Err = %v", err, result.Err)}- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" {+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "" || result.JSON[0].ToMap()["image"] == "" {t.Fatalf("unhealthy OCR should retain an image without text: result = %+v, handled = %v", result, handled)}if !strings.Contains(strings.Join(result.Warnings, "\n"), "image OCR unavailable: local OCR analyzer is unavailable") {@@ -960,7 +961,7 @@ func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {if err != nil {t.Fatal(err)}- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "captured" || result.JSON[0]["image"] == "" {+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "captured" || result.JSON[0].ToMap()["image"] == "" {t.Fatalf("VLM result = %+v, handled = %v", result, handled)}})@@ -997,8 +998,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {if err != nil || !handled {t.Fatalf("err = %v, handled = %v", err, handled)}- if res.JSON[0]["text"] != "OCR text" {- t.Fatalf("text = %v, want OCR text", res.JSON[0]["text"])+ if res.JSON[0].ToMap()["text"] != "OCR text" {+ t.Fatalf("text = %v, want OCR text", res.JSON[0].ToMap()["text"])}})t.Run("ocr-off-no-enhancement-keeps-empty", func(t *testing.T) {@@ -1007,7 +1008,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {if err != nil || !handled {t.Fatalf("err = %v, handled = %v", err, handled)}- if res.JSON[0]["text"] != "" || res.JSON[0]["image"] == "" {+ if res.JSON[0].ToMap()["text"] != "" || res.JSON[0].ToMap()["image"] == "" {t.Fatalf("want empty text with image retained, got %#v", res.JSON[0])}if len(res.Warnings) == 0 {@@ -1023,8 +1024,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {if err != nil || !handled {t.Fatalf("err = %v, handled = %v", err, handled)}- if res.JSON[0]["text"] != "captured" {- t.Fatalf("text = %v, want captured (VLM only)", res.JSON[0]["text"])+ if res.JSON[0].ToMap()["text"] != "captured" {+ t.Fatalf("text = %v, want captured (VLM only)", res.JSON[0].ToMap()["text"])}})t.Run("switch-beats-stale-parse-method", func(t *testing.T) {@@ -1035,8 +1036,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {if err != nil {t.Fatal(err)}- if res.JSON[0]["text"] != "" {- t.Fatalf("ocr_enabled:false must override stale parse_method, got text %v", res.JSON[0]["text"])+ if res.JSON[0].ToMap()["text"] != "" {+ t.Fatalf("ocr_enabled:false must override stale parse_method, got text %v", res.JSON[0].ToMap()["text"])}})}@@ -1065,7 +1066,7 @@ func TestMaybeDispatchImageBlankVisionDescriptionWarns(t *testing.T) {if err != nil || !handled {t.Fatalf("err = %v, handled = %v", err, handled)}- if text, _ := res.JSON[0]["text"].(string); strings.TrimSpace(text) != "" {+ if text, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(text) != "" {t.Fatalf("text = %q, want empty", text)}if !strings.Contains(strings.Join(res.Warnings, "\n"), "the Tokenizer will discard this item") {diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.goindex 579873cc79..88c8fd0307 100644--- a/internal/ingestion/component/parser.go+++ b/internal/ingestion/component/parser.go@@ -591,7 +591,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st// forwards only this explicit output to the next node, so shared// fields must live in Globals.globals.PublishGlobals(ctx, out)- items, _ := out["json"].([]map[string]any)+ items, _ := out["json"].([]parser.DocItem)logParserOutput(dispatched, items)// Progress (_created_time / _elapsed_time stamping, start/done// callbacks) is owned by the canvas framework (realComponentBody),@@ -599,7 +599,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[streturn out, nil}-func logParserOutput(dispatched parser.ParseResult, items []map[string]any) {+func logParserOutput(dispatched parser.ParseResult, items []parser.DocItem) {common.Debug("parser stage output",zap.String("component", "Parser"),zap.String("normalized_from", resolveParserNormalizationSource(dispatched)),diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex 1d082e6818..c9fcc66968 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -320,6 +320,10 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name}if dispatched.Err == nil && dispatched.OutputFormat != "" {out["output_format"] = "json"+ // Emit the parser's typed []DocItem directly. The chunker consumes+ // it via schema.ChunkDocsFromDocItem (no ToMap round-trip); other+ // consumers that still need the map form convert at their own+ // boundary via parser.ItemsToMaps.out["json"] = normalizeParserJSON(ctx, name, dispatched)if dispatched.File != nil {out["file"] = dispatched.File@@ -331,7 +335,7 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, nameif len(rawPages) == 0 {rawPages = [][]byte{nil}}- fallbackItems := make([]map[string]any, 0, len(rawPages))+ fallbackItems := make([]parser.DocItem, 0, len(rawPages))for _, pageBytes := range rawPages {txt := string(pageBytes)fallbackItems = append(fallbackItems, parser.NewTextJSONItem(txt))@@ -341,7 +345,11 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, namereturn out}-func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []map[string]any {+// normalizeParserJSON returns the parser output as the typed []DocItem. It no+// longer round-trips through map[string]any (the historical ItemsToMaps call),+// so the downstream chunker receives structured items and converts them via+// schema.ChunkDocsFromDocItem without ever calling DocItem.ToMap.+func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []parser.DocItem {if len(dispatched.JSON) > 0 {return dispatched.JSON}@@ -349,9 +357,9 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parserreturn parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)}if dispatched.Text != "" {- return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}+ return []parser.DocItem{parser.NewTextJSONItem(dispatched.Text)}}- return []map[string]any{}+ return []parser.DocItem{}}func warnParserNormalizationFallback(filename, source string, err error) {diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.goindex d6bf465d11..90697b8fb4 100644--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go@@ -11,6 +11,7 @@ import ("testing"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/parser/parser""gorm.io/gorm")@@ -87,7 +88,8 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {t.Fatalf("vision call count = %d, want %d", got, pageCount)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) == 0 {t.Fatalf("json payload missing or empty: %T", out["json"])}diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 470b5a5384..9b7adc63d4 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -82,7 +82,8 @@ func requireJSONText(t *testing.T, out map[string]any, want string) {if got := out["output_format"]; got != "json" {t.Fatalf("output_format = %v, want json", got)}- items, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if !ok || len(items) == 0 {t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])}@@ -124,7 +125,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {requireJSONText(t, out, tt.wantText)if tt.name == "markdown" {- items, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json = %T, want []map[string]any", out["json"])}@@ -169,7 +171,7 @@ func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {out := buildParserOutputs(t.Context(), parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{},+ JSON: parser.MapsToItems([]map[string]any{}),Markdown: "# Recovered title",}, "sample.md", nil, "")@@ -184,7 +186,7 @@ func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {"name": "report.pdf","outline": wantOutline,},- JSON: []map[string]any{{"text": "body", "doc_type_kwd": "text"}},+ JSON: parser.MapsToItems([]map[string]any{{"text": "body", "doc_type_kwd": "text"}}),}, "report.pdf", nil, "")file, ok := out["file"].(map[string]any)if !ok {@@ -235,7 +237,7 @@ func TestLogParserOutputReportsNormalizationSource(t *testing.T) {logParserOutput(parser.ParseResult{OutputFormat: "markdown",Markdown: "# Title\n\nBody",- }, []map[string]any{{"text": "Title"}, {"text": "Body"}})+ }, []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")})entries := logs.FilterMessage("parser stage output").All()if len(entries) != 1 {@@ -292,7 +294,8 @@ func TestDispatch_JSONOutput(t *testing.T) {if got, want := out["output_format"], "json"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json payload missing or wrong type: %T", out["json"])}@@ -321,7 +324,9 @@ func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {if got := out["output_format"]; got != "json" {t.Fatalf("output_format = %v, want json", got)}- if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)+ if !ok || len(items) == 0 {t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])}}@@ -346,7 +351,8 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {if got, want := out["output_format"], "json"; got != want {t.Errorf("output_format = %v, want %v (raw-text mode)", got, want)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) == 0 {t.Fatalf("json items missing or empty: %T", out["json"])}@@ -478,7 +484,9 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {if got := out["output_format"]; got != "json" {t.Fatalf("output_format = %v, want json", got)}- if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)+ if !ok || len(items) == 0 {t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])}}@@ -510,7 +518,8 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) == 0 {t.Fatalf("json payload missing or empty: %T", out["json"])}@@ -603,7 +612,8 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {"tenant_id": "tenant-1",})if err == nil {- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) != 2 {t.Fatalf("json payload = %#v, want 2 items", out["json"])}@@ -670,7 +680,8 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) != 2 {t.Fatalf("json payload = %#v, want 2 items", out["json"])}@@ -727,7 +738,8 @@ func TestDispatch_PDFVisionJSON_HonorsPageRanges(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json payload = %T, want []map[string]any", out["json"])}@@ -862,7 +874,8 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {if got, want := out["output_format"], "json"; got != want {t.Fatalf("output_format = %v, want %v", got, want)}- items, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if !ok || len(items) == 0 {t.Fatalf("json payload = %#v, want non-empty structured items", out["json"])}diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.goindex 75b0cbdd5d..fb7065d7ac 100644--- a/internal/ingestion/component/parser_test.go+++ b/internal/ingestion/component/parser_test.go@@ -24,6 +24,7 @@ import ("github.com/xuri/excelize/v2""ragflow/internal/agent/runtime""ragflow/internal/entity"+ "ragflow/internal/parser/parser")func TestReportParserWarningsAsProgressMessages(t *testing.T) {@@ -266,7 +267,8 @@ func TestParserComponent_Invoke_TextInput(t *testing.T) {if got, want := out["output_format"], "json"; got != want {t.Errorf("output_format = %v, want %v", got, want)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) != 1 {t.Fatalf("json: got %T (len %d), want 1 item", out["json"], len(jsonItems))}@@ -297,7 +299,8 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- items, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json = %T, want []map[string]any", out["json"])}@@ -317,7 +320,8 @@ func TestParserComponent_Invoke_PageRangeFilter(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json: got %T, want []map[string]any", out["json"])}@@ -491,7 +495,8 @@ func TestParserComponent_Invoke_ResolvesBinaryFromDocID(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) != 2 {t.Fatalf("json = %T/%v, want 2 items", out["json"], out["json"])}@@ -518,7 +523,8 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok || len(jsonItems) != 1 {t.Fatalf("json = %T/%v, want 1 item", out["json"], out["json"])}@@ -557,7 +563,8 @@ func TestParserComponent_Invoke_AcceptsBytes(t *testing.T) {if err != nil {t.Fatalf("Invoke: %v", err)}- jsonItems, ok := out["json"].([]map[string]any)+ jsonDocs, ok := out["json"].([]parser.DocItem)+ jsonItems := parser.ItemsToMaps(jsonDocs)if !ok {t.Fatalf("json: got %T", out["json"])}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex bad7e288cd..f11cd1b5a3 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -246,11 +246,11 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, breturn buildMarkdownOCRDispatchResult(md), nil}-func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []map[string]any {+func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []parser.DocItem {mp, err := parser.NewMarkdownParser(parser.GoMarkdown)if err != nil {warnParserNormalizationFallback(filename, "markdown", err)- return []map[string]any{parser.NewTextJSONItem(mdText)}+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}}// OCR Markdown is an untrusted backend response. Keep normalization// deterministic and local: embedded data URIs remain available, while@@ -259,11 +259,11 @@ func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []mares := mp.ParseWithResult(ctx, filename, []byte(mdText))if res.Err != nil {warnParserNormalizationFallback(filename, "markdown", res.Err)- return []map[string]any{parser.NewTextJSONItem(mdText)}+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}}if len(res.JSON) == 0 {warnParserNormalizationFallback(filename, "markdown", fmt.Errorf("parser returned no JSON items"))- return []map[string]any{parser.NewTextJSONItem(mdText)}+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}}return res.JSON}@@ -1050,7 +1050,7 @@ func mineruJSONParseResult(filename string, items []map[string]any) parser.Parse"outline": []map[string]any{},"parse_method": "mineru",},- JSON: items,+ JSON: parser.MapsToItems(items),}}@@ -1148,7 +1148,7 @@ func dispatchPDFVision(return parser.ParseResult{OutputFormat: "json",File: fileMeta,- JSON: items,+ JSON: parser.MapsToItems(items),}, nil}diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.gonew file mode 100644index 0000000000..42d8b5e27a--- /dev/null+++ b/internal/ingestion/component/schema/chunk_docitem_test.go@@ -0,0 +1,229 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package schema++import (+ "reflect"+ "testing"++ "ragflow/internal/entity"+ "ragflow/internal/parser/parser"+)++// ChunkDocsFromDocItem must produce the same []ChunkDoc as the historical+// map path. The historical contract is ChunkDocsFromAny over the []map form+// the parser produced before emitting DocItems directly: ItemsToMaps+// projects each DocItem via its canonical ToMap, then ChunkDocFromMap decodes+// it. ChunkDocsFromDocItem must converge to that exact result, otherwise+// consuming DocItem directly would silently change chunking behavior.+//+// The baseline is deliberately the map path (NOT ChunkDocsFromAny(items),+// which dispatches []DocItem straight back to ChunkDocsFromDocItem and would+// make the comparison tautological — the bug this test guards against).+func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {+ cases := []struct {+ name string+ items []parser.DocItem+ }{+ {+ name: "text with extra passthrough",+ items: []parser.DocItem{+ parser.TextItem{Text: "hello", CKType: "heading", Extra: map[string]any{"layout": "1", "page_number": 3}},+ },+ },+ {+ name: "image with extra passthrough",+ items: []parser.DocItem{+ parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", CKType: "image", Extra: map[string]any{"page_number": 2}},+ },+ },+ {+ name: "table with sheet and positions",+ items: []parser.DocItem{+ parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: [][]float64{{1, 2}}, Extra: map[string]any{"sheet_index": 0}},+ },+ },+ {+ name: "raw item passthrough",+ items: []parser.DocItem{+ parser.RawItem{M: map[string]any{"text": "x", "doc_type_kwd": "text", "page_number": 5}},+ },+ },+ {+ name: "mixed typed and raw",+ items: []parser.DocItem{+ parser.TextItem{Text: "a"},+ parser.RawItem{M: map[string]any{"text": "b", "doc_type_kwd": "text"}},+ parser.ImageItem{Image: "i", CKType: "image"},+ },+ },+ // A canonical key present in Extra must NOT shadow the typed field when+ // the typed field is empty. ToMap keeps the empty canonical field and+ // blocks Extra via mergeExtra; the typed conversion must match exactly.+ // Regression guard for the divergent docItemWireMapFromFields builder,+ // which omitted empty canonical keys and let Extra override them.+ {+ name: "empty text must not be shadowed by extra text",+ items: []parser.DocItem{+ parser.TextItem{Text: "", Extra: map[string]any{"text": "stale"}},+ },+ },+ {+ name: "empty image text must not be shadowed by extra text",+ items: []parser.DocItem{+ parser.ImageItem{Text: "", Image: "i", CKType: "image", Extra: map[string]any{"text": "stale"}},+ },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got, found, err := ChunkDocsFromDocItem(tc.items)+ if err != nil {+ t.Fatalf("ChunkDocsFromDocItem err: %v", err)+ }+ if !found {+ t.Fatalf("expected found=true")+ }+ // Historical map-path baseline: ItemsToMaps -> ChunkDocsFromAny.+ maps := parser.ItemsToMaps(tc.items)+ want, wfound, werr := ChunkDocsFromAny(maps)+ if werr != nil || !wfound {+ t.Fatalf("map-path baseline unexpected: err=%v found=%v", werr, wfound)+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("ChunkDocsFromDocItem diverges from map path\n got=%#v\nwant=%#v", got, want)+ }+ })+ }+}++// The dispatch boundary in ChunkDocsFromAny must route []parser.DocItem to+// ChunkDocsFromDocItem and report found=true, while any non-slice/unknown+// shape falls through to the default case and reports found=false. This is+// the only place a "non-DocItem" input can be rejected: ChunkDocsFromDocItem+// itself is statically typed to []parser.DocItem, so the guard lives here.+func TestChunkDocsFromAny_DispatchRejectsNonDocItem(t *testing.T) {+ if _, found, _ := ChunkDocsFromAny([]parser.DocItem{parser.TextItem{Text: "a"}}); !found {+ t.Fatalf("expected found=true for []parser.DocItem (routed to ChunkDocsFromDocItem)")+ }+ if _, found, _ := ChunkDocsFromAny("not a slice"); found {+ t.Fatalf("expected found=false for non-slice input")+ }+ // []map[string]any keeps its own map path (found=true) — it is NOT a+ // DocItem and must not be silently routed to ChunkDocsFromDocItem.+ if _, found, _ := ChunkDocsFromAny([]map[string]any{{"text": "x"}}); !found {+ t.Fatalf("expected found=true for []map[string]any via map path")+ }+}++// Empty slice is a valid DocItem input: found=true, no docs.+func TestChunkDocsFromDocItem_EmptySlice(t *testing.T) {+ got, found, err := ChunkDocsFromDocItem([]parser.DocItem{})+ if err != nil {+ t.Fatalf("err: %v", err)+ }+ if !found {+ t.Fatalf("expected found=true for empty slice")+ }+ if len(got) != 0 {+ t.Fatalf("expected empty result, got %d", len(got))+ }+}++// A nil element inside the slice must be skipped (never dereferenced),+// matching ItemsToMaps behavior. Regression guard against a nil-interface+// method call panic in the DocItem conversion path.+func TestChunkDocsFromDocItem_SkipsNilElement(t *testing.T) {+ items := []parser.DocItem{+ parser.TextItem{Text: "a"},+ nil,+ parser.ImageItem{Image: "i", CKType: "image"},+ }+ got, found, err := ChunkDocsFromDocItem(items)+ if err != nil {+ t.Fatalf("err: %v", err)+ }+ if !found {+ t.Fatalf("expected found=true")+ }+ if len(got) != 2 {+ t.Fatalf("expected 2 docs after skipping nil, got %d", len(got))+ }+ if got[0].Text != "a" || got[1].Image != "i" {+ t.Fatalf("unexpected docs after nil skip: %#v", got)+ }+}++// Producers emit []map[string]any and wrap with parser.MapsToItems, which now+// detects doc_type_kwd and builds typed DocItems. The resulting []ChunkDoc the+// chunker sees must be identical to the historical path that decoded the raw+// []map directly. This pins the producer-side typed emission to the exact+// wire contract, so the migration cannot change chunking behavior.+func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {+ cases := []struct {+ name string+ maps []map[string]any+ }{+ {+ name: "text with extra passthrough",+ maps: []map[string]any{+ {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "ck_type": "heading", "layout": "1", "page_number": 3},+ },+ },+ {+ name: "image with extra passthrough",+ maps: []map[string]any{+ {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "ck_type": "image", "page_number": 2},+ },+ },+ {+ name: "table with sheet and positions",+ maps: []map[string]any{+ {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "ck_type": parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},+ },+ },+ {+ name: "opaque map without doc_type_kwd stays raw",+ maps: []map[string]any{+ {"foo": "bar", "nested": map[string]any{"a": 1}},+ },+ },+ {+ name: "mixed typed and opaque",+ maps: []map[string]any{+ {"text": "a", parser.DocTypeKey: parser.DocTypeText, "ck_type": "text"},+ {"foo": "bar"},+ },+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got, found, err := ChunkDocsFromDocItem(parser.DocItemsFromMaps(tc.maps))+ if err != nil {+ t.Fatalf("ChunkDocsFromDocItem err: %v", err)+ }+ if !found {+ t.Fatalf("expected found=true")+ }+ want, wfound, werr := ChunkDocsFromAny(tc.maps)+ if werr != nil || !wfound {+ t.Fatalf("map-path baseline unexpected: err=%v found=%v", werr, wfound)+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("DocItemsFromMaps path diverges from map path\n got=%#v\nwant=%#v", got, want)+ }+ })+ }+}diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex 5aa15ed2df..8ca5ae09f7 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -20,6 +20,7 @@ import ("encoding/json""ragflow/internal/entity"+ "ragflow/internal/parser/parser")// PayloadFormat is the discriminator shared by parser/chunker/tokenizer@@ -233,11 +234,50 @@ func ChunkDocsFromAny(in any) ([]ChunkDoc, bool, error) {out = append(out, doc)}return out, true, nil+ case []parser.DocItem:+ return ChunkDocsFromDocItem(t)default:return nil, false, nil}}+// ChunkDocsFromDocItem converts a parser-produced []DocItem directly into a+// typed chunk slice, without routing through DocItem.ToMap. It reads the+// typed fields of each item and feeds the canonical wire map to ChunkDocFromMap,+// so the resulting []ChunkDoc is identical to what the historical+// ChunkDocsFromAny([]DocItem) path produced (which round-tripped via ToMap).+// This is the consumer-side entry the chunker uses once parsers emit DocItems+// directly; other callers that still pass []map[string]any / []any / []ChunkDoc+// keep using ChunkDocsFromAny.+func ChunkDocsFromDocItem(items []parser.DocItem) ([]ChunkDoc, bool, error) {+ if items == nil {+ return nil, false, nil+ }+ out := make([]ChunkDoc, 0, len(items))+ for _, it := range items {+ if it == nil {+ continue+ }+ doc, err := chunkDocFromDocItem(it)+ if err != nil {+ return nil, true, err+ }+ out = append(out, doc)+ }+ return out, true, nil+}++// chunkDocFromDocItem converts a parser DocItem directly into a typed chunk by+// projecting it through its canonical ToMap wire shape and decoding that via+// ChunkDocFromMap. Using ToMap — the parser package's single canonical+// projection, with mergeExtra semantics — instead of a second, divergent wire+// map builder keeps the conversion identical to the historical []map path and+// prevents an Extra key from shadowing an empty canonical field (e.g. a stray+// Extra["text"] must never override TextItem.Text == "").+func chunkDocFromDocItem(it parser.DocItem) (ChunkDoc, error) {+ return ChunkDocFromMap(it.ToMap())+}+// ToMap converts a typed chunk back to the generic map form used by// runtime.Component boundaries.func (d ChunkDoc) ToMap() map[string]any {diff --git a/internal/ingestion/component/vision_config_test.go b/internal/ingestion/component/vision_config_test.goindex 7eacd066c9..505d3110b2 100644--- a/internal/ingestion/component/vision_config_test.go+++ b/internal/ingestion/component/vision_config_test.go@@ -10,6 +10,7 @@ import (deepdoctype "ragflow/internal/deepdoc/parser/type""ragflow/internal/entity"modelModule "ragflow/internal/entity/models"+ "ragflow/internal/parser/parser")func TestParserGlobalVisionModel(t *testing.T) {@@ -69,7 +70,8 @@ func TestParserGlobalVisionModel(t *testing.T) {t.Fatalf("resolved %v, want [%s]", resolved, tc.want)}if fileType == "image" {- items := result["json"].([]map[string]any)+ jsonDocs := result["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if !strings.Contains(items[0]["text"].(string), "OCR text") {t.Fatalf("OCR text lost: %v", items)}@@ -124,7 +126,8 @@ func TestParserGlobalVisionDoesNotOverrideASR(t *testing.T) {if err != nil {t.Fatal(err)}- items := result["json"].([]map[string]any)+ jsonDocs := result["json"].([]parser.DocItem)+ items := parser.ItemsToMaps(jsonDocs)if calls != 1 || len(items) != 1 || items[0]["text"] != "transcribed" {t.Fatalf("ASR calls = %d, items = %v", calls, items)}diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.goindex 1d2c0602bb..3d37ce56c3 100644--- a/internal/ingestion/component/vision_enhancement.go+++ b/internal/ingestion/component/vision_enhancement.go@@ -153,14 +153,15 @@ func maybeDispatchVisionEnhancement(// Collect visual resources, including Markdown images whose type is text// because flatten_media_to_text is enabled.var items []int- for i, item := range dispatched.JSON {- if img, _ := item["image"].(string); img != "" {+ for i, rawItem := range dispatched.JSON {+ m := rawItem.ToMap()+ if img, _ := m["image"].(string); img != "" {items = append(items, i)continue}- kd, _ := item["doc_type_kwd"].(string)+ kd, _ := m["doc_type_kwd"].(string)if kd == "image" || kd == "table" {- if _, ok := parser.ExtractPDFPositions(item); ok {+ if _, ok := parser.ExtractPDFPositions(m); ok {items = append(items, i)}}@@ -246,7 +247,7 @@ func maybeDispatchVisionEnhancement(}var resource *visionImagefunc() {- item := dispatched.JSON[itemIdx]+ item := dispatched.JSON[itemIdx].ToMap()if err := cropper.Prepare(ctx, item); err != nil {if ctx.Err() == nil {if payload, _ := item["image"].(string); isUsableVisionImage(payload) {@@ -380,7 +381,9 @@ func maybeDispatchVisionEnhancement(if desc == "" {continue}- appendItemText(dispatched.JSON[itemIdx], desc)+ m := dispatched.JSON[itemIdx].ToMap()+ appendItemText(m, desc)+ dispatched.JSON[itemIdx] = parser.DocItemFromMap(m)modified = true}diff --git a/internal/ingestion/component/vision_enhancement_progress_test.go b/internal/ingestion/component/vision_enhancement_progress_test.goindex 124623b267..efaaeed9f1 100644--- a/internal/ingestion/component/vision_enhancement_progress_test.go+++ b/internal/ingestion/component/vision_enhancement_progress_test.go@@ -24,12 +24,12 @@ func TestVisionEnhancement_ReportsPerFigureFractions(t *testing.T) {png := visionTestPNGBase64(t)dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "1", "image": png, "doc_type_kwd": "image"},{"text": "2", "image": png, "doc_type_kwd": "image"},{"text": "3", "image": png, "doc_type_kwd": "image"},{"text": "4", "image": png, "doc_type_kwd": "image"},- },+ }),}win := progress.Window{Base: 0.5, Span: 0.5}@@ -79,10 +79,10 @@ func TestVisionEnhancement_SettledFractionsCountFailedFigures(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "valid", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},{"text": "invalid", "image": "not-valid-base64", "doc_type_kwd": "image"},- },+ }),}win := progress.Window{Base: 0.2, Span: 0.8}@@ -126,9 +126,9 @@ func TestVisionEnhancement_ZeroWindowReportsNothing(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "1", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},- },+ }),}// Zero window (default when unpassed or inactive)@@ -161,10 +161,10 @@ func TestVisionEnhancement_CancelledContextReportsNothing(t *testing.T) {png := visionTestPNGBase64(t)dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "1", "image": png, "doc_type_kwd": "image"},{"text": "2", "image": png, "doc_type_kwd": "image"},- },+ }),}win := progress.Window{Base: 0.5, Span: 0.5}diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.goindex 191b48b5f8..367e19bdba 100644--- a/internal/ingestion/component/vision_enhancement_test.go+++ b/internal/ingestion/component/vision_enhancement_test.go@@ -186,7 +186,8 @@ func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T)if resolverCalls != 0 || len(invoker.images) != 0 {t.Fatalf("default-off enhancement used model: %d resolves, %d VLM calls", resolverCalls, len(invoker.images))}- withoutItems, ok := withoutVision["json"].([]map[string]any)+ jsonDocs, ok := withoutVision["json"].([]parser.DocItem)+ withoutItems := parser.ItemsToMaps(jsonDocs)if !ok || len(withoutItems) == 0 {t.Fatalf("default JSON = %T/%v", withoutVision["json"], withoutVision["json"])}@@ -202,7 +203,8 @@ func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T)if resolverCalls != 1 || len(invoker.images) != 1 {t.Fatalf("enabled enhancement used model %d/%d times, want 1/1", resolverCalls, len(invoker.images))}- withItems, ok := withVision["json"].([]map[string]any)+ jsonDocs, ok = withVision["json"].([]parser.DocItem)+ withItems := parser.ItemsToMaps(jsonDocs)if !ok || len(withItems) == 0 {t.Fatalf("enabled JSON = %T/%v", withVision["json"], withVision["json"])}@@ -219,11 +221,11 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{+ JSON: parser.MapsToItems([]map[string]any{{"text": "Existing caption","image": visionTestPNGBase64(t),"doc_type_kwd": "image",- }},+ }}),}res, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,@@ -236,7 +238,7 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {t.Fatal("handled = false, want true")}want := "Existing caption\na diagram of a pipeline"- if got := res.JSON[0]["text"]; got != want {+ if got := res.JSON[0].ToMap()["text"]; got != want {t.Errorf("enhanced text = %q, want %q", got, want)}if len(invoker.images) != 1 {@@ -257,10 +259,10 @@ func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{+ JSON: parser.MapsToItems([]map[string]any{{"image": cropper.payload,"doc_type_kwd": "image",- }},+ }}),}if _, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,@@ -280,10 +282,10 @@ func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{+ JSON: parser.MapsToItems([]map[string]any{{"image": visionTestPNGBase64(t),"doc_type_kwd": "image",- }},+ }}),}result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,map[string]any{"tenant_id": "t1"},@@ -308,10 +310,10 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{+ JSON: parser.MapsToItems([]map[string]any{{"image": visionTestPNGBase64(t),"doc_type_kwd": "image",- }},+ }}),}result, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,@@ -328,7 +330,7 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {if !strings.HasPrefix(invoker.images[0], "data:image/png;base64,") {t.Errorf("VLM image URL = %q, want inline PNG data URI", invoker.images[0])}- if got, want := result.JSON[0]["text"], "a diagram of a pipeline"; got != want {+ if got, want := result.JSON[0].ToMap()["text"], "a diagram of a pipeline"; got != want {t.Errorf("enhanced text = %q, want %q", got, want)}}@@ -341,11 +343,11 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.Tdispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{+ JSON: parser.MapsToItems([]map[string]any{{"text": "Existing table","image": visionTestPNGBase64(t),"doc_type_kwd": "table",- }},+ }}),}res, handled, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypePDF, dispatched,@@ -357,7 +359,7 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.Tif !handled {t.Fatal("handled = false, want VLM enhancement")}- if got, want := res.JSON[0]["text"], "Existing table\na diagram of a pipeline"; got != want {+ if got, want := res.JSON[0].ToMap()["text"], "Existing table\na diagram of a pipeline"; got != want {t.Errorf("enhanced text = %q, want %q", got, want)}})@@ -370,7 +372,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {imagePayload := visionTestPNGBase64(t)dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},{"text": "cell alt",@@ -381,7 +383,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {"column_index": 1,"media_order": 1,},- },+ }),}result, handled, err := maybeDispatchVisionEnhancement(@@ -393,11 +395,11 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {if !handled {t.Fatal("handled = false, want table and cell image VLM processing")}- if got, want := result.JSON[0]["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {+ if got, want := result.JSON[0].ToMap()["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {t.Errorf("table text = %q, want %q", got, want)}cellWant := "cell alt\na diagram of a pipeline"- if got := result.JSON[1]["text"]; got != cellWant {+ if got := result.JSON[1].ToMap()["text"]; got != cellWant {t.Errorf("cell image text = %q, want %q", got, cellWant)}if len(invoker.images) != 2 {@@ -410,7 +412,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}},+ JSON: parser.MapsToItems([]map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}}),}result, handled, err := maybeDispatchVisionEnhancement(@@ -422,7 +424,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {if !handled {t.Fatal("handled = false, want VLM enhancement")}- if got, want := result.JSON[0]["text"], "existing\na diagram of a pipeline"; got != want {+ if got, want := result.JSON[0].ToMap()["text"], "existing\na diagram of a pipeline"; got != want {t.Errorf("image text = %q, want %q", got, want)}if len(invoker.images) != 1 {@@ -452,12 +454,12 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},{"text": "existing table", "image": "dGFibGUtaW1hZ2U=", "doc_type_kwd": "table"},{"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -476,18 +478,18 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {if len(res.JSON) != 4 {t.Fatalf("JSON len = %d, want 4", len(res.JSON))}- if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {+ if got := res.JSON[0].ToMap()["text"].(string); got != "Intro paragraph" {t.Errorf("text item text = %q, want unchanged", got)}// image item: VLM description cleaned of ```markdown and appended- if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {+ if got, _ := res.JSON[1].ToMap()["text"].(string); got != "a diagram of a pipeline" {t.Errorf("image item text = %q, want 'a diagram of a pipeline'", got)}// table item with image: VLM description appended with single \n- if got, _ := res.JSON[2]["text"].(string); got != "existing table\na diagram of a pipeline" {+ if got, _ := res.JSON[2].ToMap()["text"].(string); got != "existing table\na diagram of a pipeline" {t.Errorf("table item text = %q, want 'existing table\\na diagram of a pipeline'", got)}- if got, _ := res.JSON[3]["text"].(string); got != "<table></table>" {+ if got, _ := res.JSON[3].ToMap()["text"].(string); got != "<table></table>" {t.Errorf("table item without image text = %q, want unchanged", got)}if capturedLanguage != "Japanese" {@@ -539,9 +541,9 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},- },+ }),}_, handled, err := maybeDispatchVisionEnhancement(@@ -607,9 +609,9 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -625,7 +627,7 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {if !handled {t.Fatal("handled = false, want true for a visual payload in FileTypeOTHER")}- if got, want := res.JSON[0]["text"], "caption\na diagram of a pipeline"; got != want {+ if got, want := res.JSON[0].ToMap()["text"], "caption\na diagram of a pipeline"; got != want {t.Errorf("text = %q, want %q", got, want)}if len(invoker.images) != 1 {@@ -636,9 +638,9 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}// No tenant_id@@ -652,8 +654,8 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {if err != nil || handled {t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)}- if res.JSON[0]["text"] != "" {- t.Errorf("text = %q, want untouched", res.JSON[0]["text"])+ if res.JSON[0].ToMap()["text"] != "" {+ t.Errorf("text = %q, want untouched", res.JSON[0].ToMap()["text"])}}@@ -664,9 +666,9 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {dispatched := parser.ParseResult{Err: parseErr,OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -702,9 +704,9 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -721,7 +723,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {t.Error("handled = true with cancelled context, want false")}// text field should be untouched since invoker returned no answer.- if got, _ := res.JSON[0]["text"].(string); got != "" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {t.Errorf("text = %q, want empty (no successful VLM response)", got)}}@@ -741,12 +743,12 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{// non-string image field — filtered by target collector{"text": "para", "image": 12345, "doc_type_kwd": "image"},// nil image — also filtered{"text": "para2", "image": nil, "doc_type_kwd": "image"},- },+ }),}_, handled, err := maybeDispatchVisionEnhancement(@@ -789,7 +791,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {}dispatched := parser.ParseResult{OutputFormat: "json",- JSON: items,+ JSON: parser.MapsToItems(items),}res, handled, err := maybeDispatchVisionEnhancement(@@ -806,7 +808,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {t.Error("handled = false, want true")}// All n items should have received a description appended.- for i, item := range res.JSON {+ for i, item := range parser.ItemsToMaps(res.JSON) {got, _ := item["text"].(string)if got == fmt.Sprintf("item%d", i) {t.Errorf("item[%d] text unchanged %q, expected description appended", i, got)@@ -828,9 +830,9 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -846,7 +848,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {if !handled {t.Error("handled = false, want true")}- if got, _ := res.JSON[0]["text"].(string); got != plain {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != plain {t.Errorf("plain text response = %q, want %q (must not be truncated)", got, plain)}}@@ -867,9 +869,9 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -885,7 +887,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {if handled {t.Error("handled = true, want false when prompt builder fails")}- if got, _ := res.JSON[0]["text"].(string); got != "" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {t.Errorf("text = %q, want empty", got)}}@@ -906,9 +908,9 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -924,7 +926,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {if handled {t.Error("handled = true, want false when model resolve fails")}- if got, _ := res.JSON[0]["text"].(string); got != "" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {t.Errorf("text = %q, want empty", got)}}@@ -957,7 +959,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T}dispatched := parser.ParseResult{OutputFormat: "json",- JSON: items,+ JSON: parser.MapsToItems(items),}ctx, cancel := context.WithCancel(t.Context())@@ -977,7 +979,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.Tt.Error("handled = true with cancelled context, want false")}// No item should be modified when dispatch is cancelled before scheduling.- for i, item := range res.JSON {+ for i, item := range parser.ItemsToMaps(res.JSON) {if got, _ := item["text"].(string); got != fmt.Sprintf("item%d", i) {t.Errorf("item[%d] text = %q, want untouched after cancellation", i, got)}@@ -1181,9 +1183,9 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},- },+ }),}_, handled, err := maybeDispatchVisionEnhancement(@@ -1223,9 +1225,9 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {dispatched := parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},- },+ }),}res, handled, err := maybeDispatchVisionEnhancement(@@ -1244,7 +1246,7 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {if invokerCalled {t.Error("invoker must not be called for invalid image data")}- if got, _ := res.JSON[0]["text"].(string); got != "keep" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "keep" {t.Errorf("text = %q, want unchanged %q", got, "keep")}}diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.goindex 874e9d86b2..23eda6c731 100644--- a/internal/ingestion/component/vision_source_admission_test.go+++ b/internal/ingestion/component/vision_source_admission_test.go@@ -76,7 +76,7 @@ func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {}}run := func(inputs map[string]any) []string {- dispatched := parser.ParseResult{OutputFormat: "json", JSON: []map[string]any{newItem()}}+ dispatched := parser.ParseResult{OutputFormat: "json", JSON: parser.MapsToItems([]map[string]any{newItem()})}res, _, err := maybeDispatchVisionEnhancement(context.Background(), dao.DB, utility.FileTypePDF, dispatched,inputs, visionSettings{}, progress.Window{},diff --git a/internal/ingestion/task/indexdoc/normalize.go b/internal/ingestion/task/indexdoc/normalize.goindex cc678cf679..0395d1b553 100644--- a/internal/ingestion/task/indexdoc/normalize.go+++ b/internal/ingestion/task/indexdoc/normalize.go@@ -20,6 +20,7 @@ import ("fmt""ragflow/internal/common"+ "ragflow/internal/parser/parser")// NormalizeChunks converts pipeline output into a uniform []map[string]any slice.@@ -38,6 +39,9 @@ func NormalizeChunks(output map[string]any) []map[string]any {if json, ok := output["json"].([]map[string]any); ok {return DeepCopyChunks(json)}+ if docs, ok := output["json"].([]parser.DocItem); ok {+ return DeepCopyChunks(parser.ItemsToMaps(docs))+ }if json, ok := toChunkMaps(output["json"]); ok {return DeepCopyChunks(json)}diff --git a/internal/ingestion/task/indexdoc/normalize_test.go b/internal/ingestion/task/indexdoc/normalize_test.goindex e27e9ab0cd..8555f2807b 100644--- a/internal/ingestion/task/indexdoc/normalize_test.go+++ b/internal/ingestion/task/indexdoc/normalize_test.go@@ -2,6 +2,8 @@ package indexdocimport ("testing"++ "ragflow/internal/parser/parser")// =============================================================================@@ -39,6 +41,25 @@ func TestNormalizeChunks_JSONFormat(t *testing.T) {}}+// An empty (non-nil) []parser.DocItem under "json" is a present result and+// must take precedence over markdown, matching the historical+// []map[string]any empty-slice behavior (which returned no chunks, not the+// markdown text). Regression guard for the len(docs) > 0 guard that skipped+// the typed json branch for an empty slice.+func TestNormalizeChunks_EmptyDocItemSlicePrecedence(t *testing.T) {+ input := map[string]any{+ "json": []parser.DocItem{},+ "markdown": "# heading",+ }+ result := NormalizeChunks(input)+ if result == nil {+ t.Fatalf("expected empty (non-nil) slice, got nil")+ }+ if len(result) != 0 {+ t.Fatalf("len = %d, want 0 (json present, takes precedence over markdown)", len(result))+ }+}+func TestNormalizeChunks_JSONFormatFromGenericSlice(t *testing.T) {input := map[string]any{"json": []any{diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.goindex 1445a2279f..e1b3b06e7a 100644--- a/internal/parser/parser/charset_test.go+++ b/internal/parser/parser/charset_test.go@@ -166,7 +166,7 @@ func TestCSVParser_GBK(t *testing.T) {t.Fatalf("CSVParser returned no JSON items")}var rowText strings.Builder- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text := tableItemHTML(t, item)rowText.WriteString(text)}@@ -217,7 +217,7 @@ func TestEPUBParser_GB2312(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("want non-empty JSON items from EPUB")}- text, _ := res.JSON[0]["text"].(string)+ text, _ := res.JSON[0].ToMap()["text"].(string)if strings.ContainsRune(text, '\ufffd') {t.Errorf("EPUB text contains replacement runes: %s", text)}diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.goindex 153df9012e..88274d3e75 100644--- a/internal/parser/parser/csv_parser.go+++ b/internal/parser/parser/csv_parser.go@@ -155,7 +155,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data ["format": "csv","sheets": 1,},- JSON: items,+ JSON: MapsToItems(items),}}diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.goindex 5f9487875d..24fa1be83f 100644--- a/internal/parser/parser/csv_parser_test.go+++ b/internal/parser/parser/csv_parser_test.go@@ -26,7 +26,7 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}- item := res.JSON[0]+ item := res.JSON[0].ToMap()if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])}@@ -69,7 +69,7 @@ func TestCSVParser_UTF8BOM(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %#v, want one table item", res.JSON)}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {t.Errorf("header row = %q", text)}@@ -91,7 +91,7 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult failed: %v", res.Err)}- if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["ck_type"] != "table" {t.Fatalf("items = %#v, want one table item", res.JSON)}if !reflect.DeepEqual(res.JSON, plainRes.JSON) {@@ -117,10 +117,10 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table" {- t.Fatalf("ck_type = %v, want table", res.JSON[0]["ck_type"])+ if res.JSON[0].ToMap()["ck_type"] != "table" {+ t.Fatalf("ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])}- if positions := res.JSON[0]["positions"].([][]float64); len(positions) != 2 {+ if positions := res.JSON[0].ToMap()["positions"].([][]float64); len(positions) != 2 {t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))}}@@ -156,7 +156,7 @@ func TestCSVParser_PreservesEveryDataRow(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))}- positions := res.JSON[0]["positions"].([][]float64)+ positions := res.JSON[0].ToMap()["positions"].([][]float64)if len(positions) != 5 {t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus four rows)", len(positions))}@@ -177,12 +177,12 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||!strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {t.Fatalf("row widths not preserved in markup: %q", text)}- positions := res.JSON[0]["positions"].([][]float64)+ positions := res.JSON[0].ToMap()["positions"].([][]float64)// Per-row column ranges: Q1 row spans cols 1-2, Q2 row spans 1-3.if len(positions) != 3 || positions[1][4] != 2 || positions[2][4] != 3 {t.Fatalf("position matrix lost per-row widths: %v", positions)@@ -201,7 +201,7 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||!strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {t.Errorf("separator %q: row not split into three columns: %q", sep, text)@@ -272,7 +272,7 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {t.Errorf("empty tab-separated field not kept: %q", text)}@@ -292,7 +292,7 @@ func TestCSVParser_PreservesInternalJoiners(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item", len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<td>"+emoji+"</td>") {t.Errorf("emoji ZWJ sequence broken: %q", text)}diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.gonew file mode 100644index 0000000000..3eb9b23d75--- /dev/null+++ b/internal/parser/parser/doc_item.go@@ -0,0 +1,326 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "encoding/json"++ "ragflow/internal/entity"+)++// DocItem is one structured piece of a parser's output, replacing the previous+// loosely-typed `map[string]any` payload. The parser package emits DocItems;+// the ingestion component and downstream consumers still read the wire form+// produced by ToMap, so the external JSON shape (key names, pipeline-only+// keys, the doc_type_kwd discriminator) is unchanged by this type.+//+// The design mirrors the existing typed chunk AST (schema.ChunkDoc) but lives+// in the parser package so the parser layer does not import ingestion. Ad-hoc+// pipeline keys (layout, page_number, _pdf_positions, sheet_index, media_order,+// parent_table_id, ...) that are not worth modelling as named fields are passed+// through verbatim via the Extra map, keeping ToMap a 1:1 projection of the+// historical map shape. A later pass may promote the well-known Extra keys to+// named fields; that is out of scope for the boundary-adapter change.+type DocItem interface {+ // DocType returns the item's doc_type_kwd discriminator (text/table/image).+ DocType() string+ // ToMap projects the item onto the historical wire shape. Callers must not+ // mutate the returned map's shared Extra contents.+ ToMap() map[string]any+}++// TextItem is a body/heading/list/code span. CKType optionally carries the+// legacy ck_type discriminator (heading/list/code/...); it is omitted from+// ToMap when empty. Extra carries layout/page_number/level/... passthrough.+type TextItem struct {+ Text string+ CKType string+ Extra map[string]any+}++func (t TextItem) DocType() string { return DocTypeText }++func (t TextItem) ToMap() map[string]any {+ m := map[string]any{+ "text": t.Text,+ DocTypeKey: DocTypeText,+ }+ if t.CKType != "" {+ m["ck_type"] = t.CKType+ }+ return mergeExtra(m, t.Extra)+}++// TableItem carries a parsed TableData. CKType is the legacy ck_type+// discriminator (always "table" today, mirroring NewTableDataJSONItem); it is+// emitted for pipeline compatibility and will be dropped in a later pass.+type TableItem struct {+ Table *entity.TableData+ Sheet string+ Positions [][]float64+ CKType string+ Extra map[string]any+}++func (t TableItem) DocType() string { return DocTypeTable }++func (t TableItem) ToMap() map[string]any {+ m := map[string]any{+ "table": t.Table,+ DocTypeKey: DocTypeTable,+ "ck_type": firstNonEmptyStr(t.CKType, DocTypeTable),+ }+ if t.Sheet != "" {+ m["sheet"] = t.Sheet+ }+ if len(t.Positions) > 0 {+ m["positions"] = t.Positions+ }+ return mergeExtra(m, t.Extra)+}++// ImageItem is an embedded image. Text is the alt text, Image the data+// URL/base64 payload (possibly empty when omitted). CKType carries the legacy+// ck_type discriminator ("image" for xlsx/pptx/html/markdown, empty for docx);+// it is omitted from ToMap when empty.+type ImageItem struct {+ Text string+ Image string+ Sheet string+ Positions [][]float64+ CKType string+ Extra map[string]any+}++func (i ImageItem) DocType() string { return DocTypeImage }++func (i ImageItem) ToMap() map[string]any {+ m := map[string]any{+ "text": i.Text,+ "image": i.Image,+ DocTypeKey: DocTypeImage,+ }+ if i.CKType != "" {+ m["ck_type"] = i.CKType+ }+ if i.Sheet != "" {+ m["sheet"] = i.Sheet+ }+ if len(i.Positions) > 0 {+ m["positions"] = i.Positions+ }+ return mergeExtra(m, i.Extra)+}++// RawItem is a passthrough for items the parser constructs as an opaque map+// (the email content item, or any parser still emitting map literals during+// the migration). ToMap returns the map unchanged, so the wire shape is+// preserved exactly. DocType is empty because the discriminator lives in the+// map under DocTypeKey.+type RawItem struct {+ M map[string]any+}++func (RawItem) DocType() string { return "" }++func (r RawItem) ToMap() map[string]any { return r.M }++// MarshalJSON projects each item onto its wire shape so that downstream+// consumers that round-trip the payload through encoding/json (the chunker+// decodes inputs["json"] via json.Marshal -> json.Unmarshal into []ChunkDoc)+// observe the historical map shape unchanged.+func (t TextItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()) }+func (t TableItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()) }+func (i ImageItem) MarshalJSON() ([]byte, error) { return json.Marshal(i.ToMap()) }+func (r RawItem) MarshalJSON() ([]byte, error) { return json.Marshal(r.M) }++// FromMap wraps an opaque payload map as a DocItem. Used by boundary readers+// that receive a []map[string]any and must hand it to DocItem-typed APIs.+func FromMap(m map[string]any) DocItem { return RawItem{M: m} }++// Raw wraps an opaque payload map as a DocItem. Parsers that still build items+// as map literals use it at the assignment site (e.g. `JSON: MapsToItems(items)`+// or `res.JSON = append(res.JSON, Raw(m))`), so their internal logic stays+// unchanged while the field gains the DocItem type.+func Raw(m map[string]any) DocItem { return RawItem{M: m} }++// MapsToItems wraps a slice of payload maps as typed DocItems (one+// TextItem / TableItem / ImageItem / RawItem each, detected via doc_type_kwd).+// Parsers that accumulate results as []map[string]any use it at the assignment+// site (`JSON: MapsToItems(items)`) so their internal logic is unchanged while+// the field gains the DocItem type and the concrete TextItem/TableItem/+// ImageItem constructors become the emission path. Opaque maps without a known+// discriminator stay RawItem, preserving the wire shape exactly.+func MapsToItems(maps []map[string]any) []DocItem {+ return DocItemsFromMaps(maps)+}++// DocItemFromMap detects the item's doc_type_kwd and builds the matching typed+// DocItem, extracting the canonical wire fields into the struct and leaving+// every other key in Extra. For an opaque map without a known discriminator it+// falls back to RawItem, preserving the map exactly. This is the producer-side+// counterpart to ItemsToMaps: parsers can emit typed DocItems without altering+// the wire shape downstream consumers (ToMap) rely on.+func DocItemFromMap(m map[string]any) DocItem {+ if m == nil {+ return RawItem{}+ }+ switch docType, _ := m[DocTypeKey].(string); docType {+ case DocTypeText:+ // Only build a TextItem when the map actually carries a text field.+ // A text-typed map without a text key (e.g. an email metadata item+ // whose body is gated by fields) must round-trip through ToMap+ // unchanged; TextItem.ToMap always writes "text", so forcing it here+ // would invent a text key the original map lacked.+ if _, ok := m["text"]; ok {+ return textItemFromMap(m)+ }+ return RawItem{M: m}+ case DocTypeImage:+ if _, ok := m["image"]; ok {+ return imageItemFromMap(m)+ }+ return RawItem{M: m}+ case DocTypeTable:+ if _, ok := m["table"]; ok {+ return tableItemFromMap(m)+ }+ return RawItem{M: m}+ default:+ return RawItem{M: m}+ }+}++func textItemFromMap(m map[string]any) DocItem {+ return TextItem{+ Text: stringField(m, "text"),+ CKType: stringField(m, "ck_type"),+ Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),+ }+}++func imageItemFromMap(m map[string]any) DocItem {+ return ImageItem{+ Text: stringField(m, "text"),+ Image: stringField(m, "image"),+ CKType: stringField(m, "ck_type"),+ Sheet: stringField(m, "sheet"),+ Positions: positionsField(m),+ // "positions" is intentionally NOT stripped: when the map carries+ // positions as [][]any (e.g. PDF crop regions from normalizePDFPositions)+ // the typed field cannot capture them, and stripping would drop the key+ // from ToMap. Leaving it in Extra preserves the wire shape exactly.+ Extra: extraFromMap(m, "text", "image", DocTypeKey, "ck_type", "sheet"),+ }+}++func tableItemFromMap(m map[string]any) DocItem {+ td, _ := m["table"].(*entity.TableData)+ return TableItem{+ Table: td,+ Sheet: stringField(m, "sheet"),+ Positions: positionsField(m),+ CKType: stringField(m, "ck_type"),+ // Same as imageItemFromMap: keep "positions" in Extra so non-[][]float64+ // position matrices survive the typed conversion unchanged.+ Extra: extraFromMap(m, "table", DocTypeKey, "ck_type", "sheet"),+ }+}++func stringField(m map[string]any, key string) string {+ s, _ := m[key].(string)+ return s+}++func positionsField(m map[string]any) [][]float64 {+ p, _ := m["positions"].([][]float64)+ return p+}++// extraFromMap returns every key of m that is not a canonical field for the+// item type, so it lands in the item's Extra passthrough (matching mergeExtra+// semantics downstream). A nil result means "no extra keys".+func extraFromMap(m map[string]any, canonical ...string) map[string]any {+ skip := make(map[string]struct{}, len(canonical))+ for _, c := range canonical {+ skip[c] = struct{}{}+ }+ extra := make(map[string]any)+ for k, v := range m {+ if _, ok := skip[k]; ok {+ continue+ }+ extra[k] = v+ }+ if len(extra) == 0 {+ return nil+ }+ return extra+}++// DocItemsFromMaps maps a []map[string]any to typed DocItems, preserving order,+// via DocItemFromMap. Parsers that accumulate results as []map[string]any use+// it at the assignment site (`JSON: DocItemsFromMaps(items)`) so their internal+// logic is unchanged while the field gains the DocItem type and the concrete+// constructors become the emission path.+func DocItemsFromMaps(maps []map[string]any) []DocItem {+ if maps == nil {+ return nil+ }+ out := make([]DocItem, 0, len(maps))+ for _, m := range maps {+ out = append(out, DocItemFromMap(m))+ }+ return out+}++// ItemsToMaps projects a slice of DocItems onto the historical wire form. The+// ingestion boundary calls this once so downstream consumers keep reading+// []map[string]any unchanged.+func ItemsToMaps(items []DocItem) []map[string]any {+ if items == nil {+ return nil+ }+ out := make([]map[string]any, 0, len(items))+ for _, it := range items {+ if it == nil {+ continue+ }+ out = append(out, it.ToMap())+ }+ return out+}++// mergeExtra copies Extra entries onto m without clobbering canonical keys+// that were already set. Canonical typed fields always win; ad-hoc pipeline+// keys (layout, page_number, level, ...) are added only when not already+// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.+func mergeExtra(m, extra map[string]any) map[string]any {+ for k, v := range extra {+ if _, ok := m[k]; !ok {+ m[k] = v+ }+ }+ return m+}++func firstNonEmptyStr(values ...string) string {+ for _, v := range values {+ if v != "" {+ return v+ }+ }+ return ""+}diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.gonew file mode 100644index 0000000000..dfbb5db632--- /dev/null+++ b/internal/parser/parser/doc_item_test.go@@ -0,0 +1,339 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "reflect"+ "testing"++ "ragflow/internal/entity"+)++// mustMap fails the test if the two maps differ.+func mustMap(t *testing.T, want, got map[string]any) {+ t.Helper()+ if !reflect.DeepEqual(want, got) {+ t.Fatalf("map mismatch\nwant: %#v\ngot: %#v", want, got)+ }+}++func TestTextItem_ToMap(t *testing.T) {+ t.Run("minimal", func(t *testing.T) {+ mustMap(t, map[string]any{"text": "", DocTypeKey: DocTypeText},+ TextItem{}.ToMap())+ })+ t.Run("with text and no ck_type", func(t *testing.T) {+ mustMap(t, map[string]any{"text": "body", DocTypeKey: DocTypeText},+ TextItem{Text: "body"}.ToMap())+ })+ t.Run("ck_type emitted only when non-empty", func(t *testing.T) {+ mustMap(t, map[string]any{"text": "h", DocTypeKey: DocTypeText, "ck_type": "heading"},+ TextItem{Text: "h", CKType: "heading"}.ToMap())+ // empty CKType must NOT appear as a key.+ if _, ok := (TextItem{Text: "h"}).ToMap()["ck_type"]; ok {+ t.Error("ck_type present despite empty CKType")+ }+ })+ t.Run("extra passthrough", func(t *testing.T) {+ mustMap(t, map[string]any{+ "text": "x",+ DocTypeKey: DocTypeText,+ "layout": "text",+ "page_number": float64(3),+ "level": float64(2),+ }, TextItem{Text: "x", Extra: map[string]any{+ "layout": "text", "page_number": float64(3), "level": float64(2),+ }}.ToMap())+ })+}++func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {+ // An attacker/stray Extra key must never shadow the typed fields.+ got := TextItem{+ Text: "real",+ CKType: "heading",+ Extra: map[string]any{+ "text": "evil",+ DocTypeKey: "evil",+ "ck_type": "evil",+ "layout": "kept", // not a canonical key -> passes through+ },+ }.ToMap()+ mustMap(t, map[string]any{+ "text": "real",+ DocTypeKey: DocTypeText,+ "ck_type": "heading",+ "layout": "kept",+ }, got)+}++func TestTableItem_ToMap(t *testing.T) {+ t.Run("nil table still emits shape", func(t *testing.T) {+ mustMap(t, map[string]any{+ "table": (*entity.TableData)(nil),+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ }, TableItem{}.ToMap())+ })+ t.Run("ck_type falls back to table when empty", func(t *testing.T) {+ mustMap(t, map[string]any{+ "table": (*entity.TableData)(nil),+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ }, TableItem{CKType: ""}.ToMap())+ })+ t.Run("ck_type explicit wins", func(t *testing.T) {+ mustMap(t, map[string]any{+ "table": (*entity.TableData)(nil),+ DocTypeKey: DocTypeTable,+ "ck_type": "table",+ "sheet": "Sheet1",+ "positions": [][]float64{{0, 1, 1, 2, 2}},+ }, TableItem{+ CKType: "table",+ Sheet: "Sheet1",+ Positions: [][]float64{{0, 1, 1, 2, 2}},+ }.ToMap())+ })+ t.Run("empty sheet/positions omitted", func(t *testing.T) {+ got := TableItem{Table: &entity.TableData{}, Sheet: "", Positions: nil}.ToMap()+ if _, ok := got["sheet"]; ok {+ t.Error("sheet present despite empty Sheet")+ }+ if _, ok := got["positions"]; ok {+ t.Error("positions present despite nil Positions")+ }+ })+}++func TestImageItem_ToMap(t *testing.T) {+ t.Run("docx-like: no ck_type, empty image", func(t *testing.T) {+ // docx image items historically set doc_type_kwd:"image" with no+ // ck_type and an empty image when over budget.+ mustMap(t, map[string]any{+ "text": "",+ "image": "",+ DocTypeKey: DocTypeImage,+ }, ImageItem{}.ToMap())+ if _, ok := (ImageItem{}).ToMap()["ck_type"]; ok {+ t.Error("ck_type present despite empty CKType")+ }+ })+ t.Run("xlsx-like: ck_type image + sheet + positions", func(t *testing.T) {+ mustMap(t, map[string]any{+ "text": "alt",+ "image": "data:image/png;base64,xxx",+ DocTypeKey: DocTypeImage,+ "ck_type": "image",+ "sheet": "Sheet2",+ "positions": [][]float64{{1, 0, 0, 3, 4}},+ }, ImageItem{+ Text: "alt",+ Image: "data:image/png;base64,xxx",+ CKType: "image",+ Sheet: "Sheet2",+ Positions: [][]float64{{1, 0, 0, 3, 4}},+ }.ToMap())+ })+ t.Run("extra passthrough (media_order/parent_table_id)", func(t *testing.T) {+ mustMap(t, map[string]any{+ "text": "",+ "image": "",+ DocTypeKey: DocTypeImage,+ "media_order": float64(1),+ "parent_table_id": "t0",+ "row_index": float64(0),+ }, ImageItem{Extra: map[string]any{+ "media_order": float64(1), "parent_table_id": "t0", "row_index": float64(0),+ }}.ToMap())+ })+}++func TestRawItem_Passthrough(t *testing.T) {+ t.Run("exact shape preserved", func(t *testing.T) {+ in := map[string]any{+ "text": "hello",+ DocTypeKey: DocTypeText,+ "ck_type": "heading",+ "layout": "title",+ "arbitrary": []int{1, 2, 3},+ "nested": map[string]any{"k": "v"},+ }+ got := RawItem{M: in}.ToMap()+ mustMap(t, in, got)+ })+ t.Run("DocType is empty (lives in map)", func(t *testing.T) {+ if (RawItem{M: map[string]any{DocTypeKey: DocTypeText}}).DocType() != "" {+ t.Error("RawItem.DocType should be empty; discriminator is in the map")+ }+ })+ t.Run("nil map does not panic", func(t *testing.T) {+ var ri RawItem+ if ri.ToMap() != nil {+ t.Error("ToMap of nil RawItem.M should be nil")+ }+ })+}++func TestItemsToMaps(t *testing.T) {+ t.Run("order preserved, nil skipped", func(t *testing.T) {+ items := []DocItem{+ TextItem{Text: "a"},+ nil,+ RawItem{M: map[string]any{DocTypeKey: DocTypeText, "text": "b"}},+ TableItem{Table: &entity.TableData{}},+ }+ got := ItemsToMaps(items)+ if len(got) != 3 {+ t.Fatalf("expected 3 items (nil skipped), got %d: %#v", len(got), got)+ }+ if got[0]["text"] != "a" || got[1]["text"] != "b" {+ t.Errorf("order not preserved: %#v", got)+ }+ if got[2][DocTypeKey] != DocTypeTable {+ t.Errorf("third item should be table: %#v", got[2])+ }+ })+ t.Run("nil slice", func(t *testing.T) {+ if ItemsToMaps(nil) != nil {+ t.Error("ItemsToMaps(nil) should be nil")+ }+ })+}++func TestConstructorWireShapeRegression(t *testing.T) {+ // Guards that the two canonical constructors still emit the exact wire form+ // parsers (and the chunker) depend on.+ t.Run("NewTextJSONItem", func(t *testing.T) {+ mustMap(t, map[string]any{"text": "hi", DocTypeKey: DocTypeText},+ NewTextJSONItem("hi").ToMap())+ })+ t.Run("NewTableDataJSONItem", func(t *testing.T) {+ td := &entity.TableData{}+ mustMap(t, map[string]any{+ "table": td,+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ "sheet": "S",+ "positions": [][]float64{{0, 0, 0, 1, 1}},+ }, NewTableDataJSONItem(td, "S", [][]float64{{0, 0, 0, 1, 1}}).ToMap())+ })+}++func TestToMapIndependentPerCall(t *testing.T) {+ // Mutating the returned map must not affect a later call or shared Extra.+ it := TextItem{Text: "x", Extra: map[string]any{"k": "v"}}+ first := it.ToMap()+ first["text"] = "mutated"+ second := it.ToMap()+ if second["text"] != "x" {+ t.Errorf("ToMap result was not independent: %#v", second)+ }+ if first["k"] != "v" || second["k"] != "v" {+ t.Errorf("Extra passthrough broke: %#v %#v", first, second)+ }+}++// DocItemFromMap detects doc_type_kwd and builds the matching typed item,+// extracting canonical fields into the struct and leaving the rest in Extra.+// For an opaque map without a known discriminator it falls back to RawItem,+// preserving the map exactly. The produced item's ToMap must equal the input+// map (the wire shape is unchanged), so producers can emit typed items+// without altering downstream chunking.+func TestDocItemFromMap_WireShapePreserved(t *testing.T) {+ textMap := map[string]any{+ "text": "hello",+ DocTypeKey: DocTypeText,+ "ck_type": "heading",+ "layout": "1",+ "page_number": 3,+ }+ imageMap := map[string]any{+ "text": "alt",+ "image": "data:image/png;base64,xxx",+ DocTypeKey: DocTypeImage,+ "ck_type": "image",+ "page_number": 2,+ }+ tableMap := map[string]any{+ "table": &entity.TableData{},+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ "sheet": "s",+ "positions": [][]float64{{1, 2}},+ "sheet_index": 0,+ }+ opaqueMap := map[string]any{"foo": "bar", "nested": map[string]any{"a": 1}}++ // PDF crop positions arrive as [][]any (normalizePDFPositions), not+ // [][]float64; the typed conversion must preserve them verbatim. Regression+ // guard for the positions-dropping bug caught during review.+ imageMapAny := map[string]any{+ "text": "alt",+ "image": "data:image/png;base64,xxx",+ DocTypeKey: DocTypeImage,+ "ck_type": "image",+ "positions": [][]any{{1, 2, 3, 4}},+ }+ tableMapAny := map[string]any{+ "table": &entity.TableData{},+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ "positions": [][]any{{1, 2, 3, 4}},+ }++ for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny} {+ got := DocItemFromMap(m).ToMap()+ if !reflect.DeepEqual(got, m) {+ t.Fatalf("DocItemFromMap round-trip mismatch for %q\n got=%#v\nwant=%#v", m[DocTypeKey], got, m)+ }+ }++ // The typed detection must actually yield the concrete type, not RawItem.+ if _, ok := DocItemFromMap(textMap).(TextItem); !ok {+ t.Error("text map should produce a TextItem")+ }+ if _, ok := DocItemFromMap(imageMap).(ImageItem); !ok {+ t.Error("image map should produce an ImageItem")+ }+ if _, ok := DocItemFromMap(tableMap).(TableItem); !ok {+ t.Error("table map should produce a TableItem")+ }+ if _, ok := DocItemFromMap(opaqueMap).(RawItem); !ok {+ t.Error("opaque map should fall back to RawItem")+ }+}++// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way+// DocItemFromMap does, preserving order and skipping nothing.+func TestDocItemsFromMaps_OrderAndType(t *testing.T) {+ maps := []map[string]any{+ {"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},+ {"foo": "bar"},+ }+ items := DocItemsFromMaps(maps)+ if len(items) != 2 {+ t.Fatalf("expected 2 items, got %d", len(items))+ }+ if _, ok := items[0].(TextItem); !ok {+ t.Errorf("first should be TextItem: %#v", items[0])+ }+ if _, ok := items[1].(RawItem); !ok {+ t.Errorf("second (opaque) should be RawItem: %#v", items[1])+ }+ if items[0].ToMap()["text"] != "a" {+ t.Errorf("first item text lost: %#v", items[0].ToMap())+ }+}diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.goindex 295511eac4..252a883975 100644--- a/internal/parser/parser/doc_parser.go+++ b/internal/parser/parser/doc_parser.go@@ -91,7 +91,7 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [}res.OutputFormat = "json"- res.JSON = []map[string]any{NewTextJSONItem(text)}+ res.JSON = []DocItem{NewTextJSONItem(text)}return res}diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.goindex 6f2ae1c0b2..d48e7b3f9b 100644--- a/internal/parser/parser/doc_parser_synth_test.go+++ b/internal/parser/parser/doc_parser_synth_test.go@@ -162,7 +162,7 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)}}- if got, _ := pr.JSON[0]["text"].(string); got != pr.Text {+ if got, _ := pr.JSON[0].ToMap()["text"].(string); got != pr.Text {t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)}diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.goindex f691f653d4..21ef9ffaf4 100644--- a/internal/parser/parser/docx_parser.go+++ b/internal/parser/parser/docx_parser.go@@ -151,7 +151,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{OutputFormat: "json",File: fileMeta,- JSON: sections,+ JSON: MapsToItems(sections),Warnings: budget.warnings(),}}diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.goindex 6132e1eff7..1141a0ded0 100644--- a/internal/parser/parser/docx_parser_cgo_test.go+++ b/internal/parser/parser/docx_parser_cgo_test.go@@ -24,7 +24,7 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("JSON items is empty; expected parsed content")}- for i, item := range res.JSON {+ for i, item := range ItemsToMaps(res.JSON) {if _, ok := item["text"]; !ok {t.Errorf("item[%d] missing 'text' field", i)}@@ -161,7 +161,7 @@ func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {if jsonRes.Err != nil {t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)}- if got := joinItemTexts(jsonRes.JSON); !strings.Contains(got, tc.want) {+ if got := joinItemTexts(ItemsToMaps(jsonRes.JSON)); !strings.Contains(got, tc.want) {t.Errorf("json text = %q, want it to contain %q", got, tc.want)}diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.goindex ccfad86dbb..e34d3c60a8 100644--- a/internal/parser/parser/email_parser.go+++ b/internal/parser/parser/email_parser.go@@ -199,13 +199,13 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []bycontent["doc_type_kwd"] = "text"items := []map[string]any{content}if headerText := searchableEmailHeaders(content); headerText != "" {- items = append(items, NewTextJSONItem(headerText))+ items = append(items, NewTextJSONItem(headerText).ToMap())}items = append(items, extraItems...)return ParseResult{OutputFormat: "json",File: map[string]any{"name": filename},- JSON: items,+ JSON: MapsToItems(items),}}@@ -845,7 +845,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[svar texts []stringif res.OutputFormat == "json" && len(res.JSON) > 0 {for _, it := range res.JSON {- if t, ok := it["text"].(string); ok {+ if t, ok := it.ToMap()["text"].(string); ok {if t = strings.TrimSpace(t); t != "" {texts = append(texts, t)}diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.goindex 1ffd9012a5..aafde00560 100644--- a/internal/parser/parser/email_parser_test.go+++ b/internal/parser/parser/email_parser_test.go@@ -68,7 +68,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {if len(result.JSON) != 2 {t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))}- item := result.JSON[0]+ item := result.JSON[0].ToMap()if v, ok := item["from"].(string); !ok || v != "[email protected]" {t.Errorf("from: got %q", v)@@ -93,7 +93,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {t.Errorf("doc_type_kwd: got %q", v)}- header := result.JSON[1]+ header := result.JSON[1].ToMap()headerText, ok := header["text"].(string)if !ok {t.Fatalf("header item text missing: %#v", header)@@ -235,7 +235,7 @@ func TestEmailParser_EmlTextChunkText(t *testing.T) {if jres.Err != nil {t.Fatalf("unexpected error: %v", jres.Err)}- item := jres.JSON[0]+ item := jres.JSON[0].ToMap()if _, ok := item["metadata"].(map[string]any); !ok {t.Errorf("json output must keep metadata: %#v", item)}@@ -271,7 +271,7 @@ func TestEmailParser_MsgSupported(t *testing.T) {if len(result.JSON) != 2 {t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))}- item := result.JSON[0]+ item := result.JSON[0].ToMap()if v, ok := item["from"].(string); !ok || v != "<[email protected]>" {t.Errorf("from: got %q", v)@@ -359,7 +359,7 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()if _, ok := item["metadata"].(map[string]any); !ok {t.Fatalf("metadata must always be present for .msg, got %T", item["metadata"])@@ -405,7 +405,7 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {// concatenated item text so the test survives the json-path Text removal.var sb strings.Builderfor _, it := range result.JSON {- if txt, _ := it["text"].(string); txt != "" {+ if txt, _ := it.ToMap()["text"].(string); txt != "" {sb.WriteString(txt)sb.WriteString("\n")}@@ -463,7 +463,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()// attachments are dropped from the final ParseResult (consumed by// rechunk, then deleted); verify the high-level result is clean...@@ -541,7 +541,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()// Verify body text was extracted from nested multipart/alternativeif v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") {@@ -598,7 +598,7 @@ func TestEmailParser_Multipart(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") {t.Errorf("text: got %q", v)}@@ -636,7 +636,7 @@ func TestEmailParser_MetadataAlwaysPresent(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()// metadata must exist even though it is not in fields (Python contract).meta, ok := item["metadata"].(map[string]any)@@ -696,7 +696,7 @@ func TestEmailParser_TextHTMLAlwaysPresent(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain body") {t.Errorf("text: got %q", v)@@ -760,13 +760,13 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {}// Body must remain on the main item.- if v, ok := result.JSON[0]["text"].(string); !ok || !strings.Contains(v, "Email body text") {- t.Errorf("body missing on main item: %v", result.JSON[0]["text"])+ if v, ok := result.JSON[0].ToMap()["text"].(string); !ok || !strings.Contains(v, "Email body text") {+ t.Errorf("body missing on main item: %v", result.JSON[0].ToMap()["text"])}// Attachment text must appear as a separate retrievable JSON item.found := false- for _, it := range result.JSON {+ for _, it := range ItemsToMaps(result.JSON) {if txt, ok := it["text"].(string); ok && strings.Contains(txt, "quick brown fox") {found = true}@@ -866,7 +866,7 @@ func TestEmailParser_AttachmentEmptyPayloadSkipped(t *testing.T) {}found := false- for _, it := range result.JSON {+ for _, it := range ItemsToMaps(result.JSON) {if txt, ok := it["text"].(string); ok && strings.Contains(txt, "VALID attachment payload") {found = true}@@ -922,7 +922,7 @@ func TestEmailParser_AttachmentUnknownExtSkipped(t *testing.T) {}found := false- for _, it := range result.JSON {+ for _, it := range ItemsToMaps(result.JSON) {if txt, ok := it["text"].(string); ok && strings.Contains(txt, "KNOWN attachment payload") {found = true}@@ -967,7 +967,7 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()// text/text_html must be absent (gated by "body"), matching Python.if _, ok := item["text"]; ok {@@ -1054,7 +1054,7 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {// result.JSON; text populates result.Text).var indexed strings.Builderif format == "json" {- for _, it := range result.JSON {+ for _, it := range ItemsToMaps(result.JSON) {if txt, ok := it["text"].(string); ok {indexed.WriteString(txt)indexed.WriteString("\n")@@ -1077,7 +1077,7 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {// metadata key (the old bug leaked a "metadata:{...}" string as// its text).found := false- for _, it := range result.JSON {+ for _, it := range ItemsToMaps(result.JSON) {if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {found = trueif _, hasMeta := it["metadata"]; hasMeta {@@ -1420,7 +1420,7 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {if result.Err != nil {t.Fatalf("unexpected error: %v", result.Err)}- item := result.JSON[0]+ item := result.JSON[0].ToMap()text, _ := item["text"].(string)if strings.TrimRight(text, "\r\n") != strings.TrimRight(tt.wantText, "\r\n") {diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.goindex d26574d37a..52b496cc89 100644--- a/internal/parser/parser/epub_parser.go+++ b/internal/parser/parser/epub_parser.go@@ -64,7 +64,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data"size": 0,"encoding": "utf-8",},- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},+ JSON: []DocItem{NewTextJSONItem("")},}}@@ -97,7 +97,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": detectedEnc,},- JSON: items,+ JSON: MapsToItems(items),}}diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.goindex 3e22fc8c6c..2d2170cf43 100644--- a/internal/parser/parser/epub_parser_test.go+++ b/internal/parser/parser/epub_parser_test.go@@ -93,7 +93,7 @@ func parseTestEPUB(t *testing.T, chapters []epubTestChapter) ParseResult {func epubTestText(res ParseResult) string {var parts []string- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)parts = append(parts, text)}diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.goindex 604f4ba442..af5a46d250 100644--- a/internal/parser/parser/html_charset_test.go+++ b/internal/parser/parser/html_charset_test.go@@ -62,7 +62,7 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte// itemsText concatenates the text of every item in res, newline separated.func itemsText(res ParseResult) string {var sb strings.Builder- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {sb.WriteString(renderedText(it))sb.WriteString("\n")}@@ -159,7 +159,7 @@ func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var tableText string- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {if it["doc_type_kwd"] == "table" {tableText = renderedText(it)}diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.goindex 024e28a859..c0cc61f7c8 100644--- a/internal/parser/parser/html_media_test.go+++ b/internal/parser/parser/html_media_test.go@@ -21,14 +21,14 @@ func TestHTMLParser_EmitsInlineImagesInDocumentOrder(t *testing.T) {if len(result.JSON) != 3 {t.Fatalf("items = %+v, want text/image/text", result.JSON)}- if result.JSON[0]["text"] != "before" || result.JSON[0]["doc_type_kwd"] != "text" {+ if result.JSON[0].ToMap()["text"] != "before" || result.JSON[0].ToMap()["doc_type_kwd"] != "text" {t.Errorf("before item = %+v", result.JSON[0])}- imageItem := result.JSON[1]+ imageItem := result.JSON[1].ToMap()if imageItem["doc_type_kwd"] != "image" || imageItem["image"] != htmlMediaDataURI || imageItem["text"] != "figure alt" || imageItem["media_order"] != 1 {t.Errorf("image item = %+v", imageItem)}- if result.JSON[2]["text"] != "after" || result.JSON[2]["doc_type_kwd"] != "text" {+ if result.JSON[2].ToMap()["text"] != "after" || result.JSON[2].ToMap()["doc_type_kwd"] != "text" {t.Errorf("after item = %+v", result.JSON[2])}}@@ -42,20 +42,20 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {if len(result.JSON) != 2 {t.Fatalf("items = %+v, want table and one inline image item", result.JSON)}- if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {+ if result.JSON[0].ToMap()["doc_type_kwd"] != "table" || result.JSON[0].ToMap()["source_table_id"] != "html-table-1" {t.Errorf("table item = %+v", result.JSON[0])}- tableText := renderedText(result.JSON[0])+ tableText := renderedText(result.JSON[0].ToMap())if strings.Contains(tableText, "data:image/") {t.Error("table text duplicates the separately emitted inline image payload")}if !strings.Contains(tableText, "A") {t.Errorf("table text lost the cell text: %q", tableText)}- if result.JSON[1]["image"] != htmlMediaDataURI {- t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])+ if result.JSON[1].ToMap()["image"] != htmlMediaDataURI {+ t.Errorf("inline image payload = %v, want data URI", result.JSON[1].ToMap()["image"])}- item := result.JSON[1]+ item := result.JSON[1].ToMap()if item["doc_type_kwd"] != "image" || item["parent_table_id"] != "html-table-1" {t.Errorf("inline image item = %+v", item)}@@ -71,7 +71,7 @@ func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {if result.Err != nil {t.Fatalf("ParseWithResult: %v", result.Err)}- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] == "image" {t.Fatal("oversized data URI emitted as an image item")}diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.goindex a41404d366..d7457a8e92 100644--- a/internal/parser/parser/html_parser.go+++ b/internal/parser/parser/html_parser.go@@ -98,7 +98,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data"name": filename,"encoding": encName,},- JSON: items,+ JSON: MapsToItems(items),Warnings: state.warnings(),}}diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.goindex daf42abfe9..5c0d93a621 100644--- a/internal/parser/parser/html_parser_align_test.go+++ b/internal/parser/parser/html_parser_align_test.go@@ -33,7 +33,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {var tableText stringtableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0- for i, it := range res.JSON {+ for i, it := range ItemsToMaps(res.JSON) {text, _ := it["text"].(string)switch it["doc_type_kwd"] {case "text":@@ -61,7 +61,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {+ if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {t.Errorf("structured table item ck_type = %v, want %v", got, want)}if !strings.Contains(tableText, "<table") ||@@ -116,7 +116,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {var tableText stringtableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0- for i, it := range res.JSON {+ for i, it := range ItemsToMaps(res.JSON) {text, _ := it["text"].(string)switch it["doc_type_kwd"] {case "text":@@ -144,7 +144,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {if tableCount != 1 {t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)}- if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {+ if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {t.Errorf("structured nested table item ck_type = %v, want %v", got, want)}if !strings.Contains(tableText, "<table") ||@@ -202,7 +202,7 @@ func TestHTMLParser_NestedTableWithSurroundingText(t *testing.T) {var tableText stringtableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0- for i, it := range res.JSON {+ for i, it := range ItemsToMaps(res.JSON) {text, _ := it["text"].(string)switch it["doc_type_kwd"] {case "text":@@ -282,7 +282,7 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {var tableItemIdx []inttitleIdx, middleIdx, endIdx := -1, -1, -1- for i, it := range res.JSON {+ for i, it := range ItemsToMaps(res.JSON) {text, _ := it["text"].(string)switch it["doc_type_kwd"] {case "text":@@ -312,8 +312,8 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)}- t1 := renderedText(res.JSON[tableItemIdx[0]])- t2 := renderedText(res.JSON[tableItemIdx[1]])+ t1 := renderedText(res.JSON[tableItemIdx[0]].ToMap())+ t2 := renderedText(res.JSON[tableItemIdx[1]].ToMap())if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {t.Errorf("first table item missing x/y cells: %q", t1)}@@ -356,7 +356,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {// Join every item's text in order to check content + order + fusion.var all string- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {if txt, _ := it["text"].(string); txt != "" {all += txt + "\n"}@@ -390,7 +390,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {// therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,// not a collapse, since downstream chunking reads doc_type_kwd not ck_type.var listText string- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {if it["ck_type"] == "list" {listText, _ = it["text"].(string)break@@ -440,7 +440,7 @@ SELECT * FROM t}var preItem, textareaItem map[string]any- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {txt, _ := it["text"].(string)switch {case strings.Contains(txt, "def foo()"):@@ -525,14 +525,14 @@ func TestHTMLParser_AlignmentGolden(t *testing.T) {// the <table> markup inlined as a "text" item. filterTableDivergence drops// the table content from both sides so the comparison stays on non-table prose.drop := AcceptedDivergences(doc.Meta)- goText := filterTableDivergence(res.JSON, drop)+ goText := filterTableDivergence(ItemsToMaps(res.JSON), drop)pyText := filterTableDivergence(doc.Items, drop)// filterTableDivergence drops the table from the prose comparison;// the table-equivalence guard below (assertTablesEquivalent) checks// the table cell content still matches Python, so a collapse or// dropped column on either side is caught independently.- assertTablesEquivalent(t, res.JSON, doc.Items)+ assertTablesEquivalent(t, ItemsToMaps(res.JSON), doc.Items)if ok, diff := CompareAlignment(goText, pyText, HTMLAlignOptions()); !ok {t.Fatalf("html parser not aligned with Python golden:%s", diff)diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.goindex 6110678611..02eaadeead 100644--- a/internal/parser/parser/html_parser_test.go+++ b/internal/parser/parser/html_parser_test.go@@ -50,7 +50,7 @@ func TestHTMLParser_RemoveHeaderFooter(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)for _, banned := range []string{"Site Navigation", "Copyright 2024", "Ad banner", "Legal info"} {if strings.Contains(text, banned) {@@ -60,7 +60,7 @@ func TestHTMLParser_RemoveHeaderFooter(t *testing.T) {}// Main content must survive.found := false- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if text, _ := item["text"].(string); strings.Contains(text, "Main content here") {found = true}@@ -82,7 +82,7 @@ func TestHTMLParser_RemoveHeaderFooter_DisabledByDefault(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}found := false- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if text, _ := item["text"].(string); strings.Contains(text, "Nav") {found = true}@@ -113,7 +113,7 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {}// "目录" heading and "第一章 概述" (first entry after heading)// should be removed by removeContentsTable.- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if text == "目录" {t.Errorf("TOC heading %q was not removed", text)@@ -124,7 +124,7 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {}// "前言" and "正文开始" should survive.wantTexts := map[string]bool{"前言": false, "正文开始": false}- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if _, ok := wantTexts[text]; ok {wantTexts[text] = truediff --git a/internal/parser/parser/html_parser_unified_test.go b/internal/parser/parser/html_parser_unified_test.goindex 82031e31d2..38599581f9 100644--- a/internal/parser/parser/html_parser_unified_test.go+++ b/internal/parser/parser/html_parser_unified_test.go@@ -62,7 +62,7 @@ func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)}- if got := res.JSON[0]["text"].(string); got != tc.Want {+ if got := res.JSON[0].ToMap()["text"].(string); got != tc.Want {t.Errorf("got %q, want %q", got, tc.Want)}})@@ -90,7 +90,7 @@ func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var texts []string- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {texts = append(texts, item["text"].(string))}want := []string{diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.goindex 22523895e0..491e03bc2b 100644--- a/internal/parser/parser/json_parser.go+++ b/internal/parser/parser/json_parser.go@@ -63,7 +63,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": encName,},- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},+ JSON: []DocItem{NewTextJSONItem("")},}}@@ -78,7 +78,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": encName,},- JSON: items,+ JSON: MapsToItems(items),}}diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.goindex 6abb6b5ae3..08f12d3636 100644--- a/internal/parser/parser/json_parser_test.go+++ b/internal/parser/parser/json_parser_test.go@@ -10,7 +10,7 @@ import (func jsonTexts(res ParseResult) []string {texts := make([]string, 0, len(res.JSON))- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)texts = append(texts, text)}diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.goindex 38843297e6..2a20b9df7d 100644--- a/internal/parser/parser/markdown_parser.go+++ b/internal/parser/parser/markdown_parser.go@@ -132,7 +132,7 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, dFile: map[string]any{"name": filename,},- JSON: items,+ JSON: MapsToItems(items),Warnings: warnings,}}diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.goindex c4bb08dc40..27689d4b3d 100644--- a/internal/parser/parser/markdown_parser_test.go+++ b/internal/parser/parser/markdown_parser_test.go@@ -31,10 +31,10 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {t.Fatal("JSON is empty; want at least one item")}// Verify heading- if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {t.Fatalf("first item text = %q, want %q", got, "# Hello")}- if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {+ if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {t.Fatalf("first item ck_type = %q, want %q", got, "heading")}}@@ -49,10 +49,10 @@ func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {t.Fatalf("first item text = %q, want %q", got, "# Hello")}- if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {+ if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {t.Fatalf("first item ck_type = %q, want %q", got, "heading")}}@@ -78,7 +78,7 @@ func TestMarkdownParser_ListItemsKeepWordBoundaries(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var lists []string- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if ck, _ := item["ck_type"].(string); ck == "list" {text, _ := item["text"].(string)lists = append(lists, text)@@ -140,7 +140,7 @@ func TestMarkdownParser_OrderedListKeepsNumbers(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var lists []string- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if ck, _ := item["ck_type"].(string); ck == "list" {text, _ := item["text"].(string)lists = append(lists, text)@@ -181,7 +181,7 @@ func TestMarkdownParser_TableKeepsCellsBeyondTheHeader(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var joined strings.Builder- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {joined.WriteString(renderedText(item))joined.WriteString("\n")}@@ -222,7 +222,7 @@ func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) {}// Find the image itemfound := false- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if kd, _ := item["doc_type_kwd"].(string); kd == "image" {found = trueif img, ok := item["image"].(string); !ok || img != pixelB64 {@@ -244,7 +244,7 @@ func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if kd, _ := item["doc_type_kwd"].(string); kd == "image" {t.Fatal("unexpected image item in text-only markdown")}@@ -267,7 +267,7 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {t.Fatalf("len(JSON) = %d, want 3 (leading paragraph, table, note)", len(res.JSON))}tableCount, inlineTableCount := 0, 0- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {kd, _ := item["doc_type_kwd"].(string)text, _ := item["text"].(string)switch kd {@@ -320,7 +320,7 @@ func TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var sb strings.Builder- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)sb.WriteString(text)sb.WriteString("\n")@@ -344,7 +344,7 @@ func TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath(t *testing.T) {t.Fatalf("ParseWithResult: %v", resDisplay.Err)}var sbDisplay strings.Builder- for _, item := range resDisplay.JSON {+ for _, item := range ItemsToMaps(resDisplay.JSON) {text, _ := item["text"].(string)sbDisplay.WriteString(text)}@@ -372,7 +372,7 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var sb strings.Builder- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)sb.WriteString(text)sb.WriteString("\n")@@ -451,7 +451,7 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if kd, _ := item["doc_type_kwd"].(string); kd != tc.wantDocType {t.Errorf("doc_type_kwd = %q, want %q (item text=%q)",kd, tc.wantDocType, item["text"])@@ -554,7 +554,7 @@ func TestFindBlockImage(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var imgItems, textItems int- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {switch kd, _ := item["doc_type_kwd"].(string); kd {case "image":imgItems++@@ -574,7 +574,7 @@ func TestFindBlockImage(t *testing.T) {// A document with no image must not produce any image item.noImg := p.ParseWithResult(ctx, "b.md", []byte("# T\n\nNo images here.\n"))- for _, item := range noImg.JSON {+ for _, item := range ItemsToMaps(noImg.JSON) {if kd, _ := item["doc_type_kwd"].(string); kd == "image" {t.Fatal("unexpected image item in text-only markdown")}@@ -623,7 +623,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {var all strings.BuildersawTitle, sawTrailing, sawTableItem, sawInlineTable := false, false, false, falsetableCount := 0- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)all.WriteString(text)all.WriteString("\n")@@ -687,7 +687,7 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {var tableText stringtableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0- for i, item := range res.JSON {+ for i, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)switch kd, _ := item["doc_type_kwd"].(string); kd {case "text":@@ -751,7 +751,7 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if strings.Contains(text, "<div") || strings.Contains(text, "<style") {if ck, ok := item["ck_type"].(string); ok && ck == "table" {@@ -777,7 +777,7 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}// No table item may be emitted — the pipe rows live inside a code block.- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if kd, _ := item["doc_type_kwd"].(string); kd == "table" {t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"])}@@ -785,7 +785,7 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {// The code block item must retain the raw pipe text and must NOT contain// any <table> markup.sawCode := false- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)if strings.Contains(text, "| A | B |") {sawCode = true@@ -814,7 +814,7 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {}sawInlined, sawTableItem := false, falsetableCount := 0- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)switch kd, _ := item["doc_type_kwd"].(string); kd {case "text":@@ -859,7 +859,7 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {var tableItemIdx []intvar titleIdx, middleIdx, endIdx = -1, -1, -1- for i, item := range res.JSON {+ for i, item := range ItemsToMaps(res.JSON) {text, _ := item["text"].(string)switch kd, _ := item["doc_type_kwd"].(string); kd {case "text":@@ -887,8 +887,8 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)}- t1 := renderedText(res.JSON[tableItemIdx[0]])- t2 := renderedText(res.JSON[tableItemIdx[1]])+ t1 := renderedText(res.JSON[tableItemIdx[0]].ToMap())+ t2 := renderedText(res.JSON[tableItemIdx[1]].ToMap())if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {t.Fatalf("first table item missing x/y cells: %q", t1)}@@ -945,14 +945,14 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {// filterTableDivergence additionally drops the inlined <table> markup that// Python keeps as a "text" item, so only non-table prose is compared.ignore := AcceptedDivergences(gd.Meta)- goText := filterTableDivergence(res.JSON, ignore)+ goText := filterTableDivergence(ItemsToMaps(res.JSON), ignore)pyText := filterTableDivergence(gd.Items, ignore)// filterTableDivergence drops the table from the prose comparison;// the table-equivalence guard below (assertTablesEquivalent) checks// the table cell content still matches Python, so a collapse or// dropped column on either side is caught independently.- assertTablesEquivalent(t, res.JSON, gd.Items)+ assertTablesEquivalent(t, ItemsToMaps(res.JSON), gd.Items)if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {t.Fatalf("markdown parser not aligned with Python golden:%s", diff)diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.goindex c7be30b61c..2241e27f08 100644--- a/internal/parser/parser/office_image_fixture_cgo_test.go+++ b/internal/parser/parser/office_image_fixture_cgo_test.go@@ -43,7 +43,7 @@ func TestDOCXParser_RealTableImage(t *testing.T) {t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)}var tableID string- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] == "table" {tableID, _ = item["source_table_id"].(string)}@@ -51,7 +51,7 @@ func TestDOCXParser_RealTableImage(t *testing.T) {if tableID == "" {t.Fatalf("table with image not found: %+v", result.JSON)}- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {continue}@@ -99,7 +99,7 @@ func TestPPTXParser_RealSlideImage(t *testing.T) {if len(result.Warnings) != 0 {t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)}- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] != "image" {continue}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 848ce14f7d..5eca25921f 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -203,9 +203,9 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow intgrid = append(grid, cells)}td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}- item := NewTableDataJSONItem(td, sheet, matrix)- item["sheet_index"] = sheetIndex- return item+ ti := NewTableDataJSONItem(td, sheet, matrix).(TableItem)+ ti.Extra = map[string]any{"sheet_index": sheetIndex}+ return ti.ToMap()}prev := 0diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.goindex 03727deaaf..ed8bf9cfd6 100644--- a/internal/parser/parser/office_table_render_test.go+++ b/internal/parser/parser/office_table_render_test.go@@ -29,7 +29,7 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))}- item := res.JSON[0]+ item := res.JSON[0].ToMap()if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])}@@ -81,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one atomic table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table" || res.JSON[0]["table"] == nil {+ if res.JSON[0].ToMap()["ck_type"] != "table" || res.JSON[0].ToMap()["table"] == nil {t.Fatalf("html4excel item = %#v", res.JSON[0])}if !reflect.DeepEqual(res.JSON, plainRes.JSON) {@@ -105,7 +105,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one HTML table item (parser must not pre-split rows)", len(res.JSON))}- positions := res.JSON[0]["positions"].([][]float64)+ positions := res.JSON[0].ToMap()["positions"].([][]float64)if len(positions) != dataRows+1 {t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus %d rows)", len(positions), dataRows)}@@ -113,7 +113,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<td>258</td>") {t.Fatalf("last data row missing from markup:\n%s", text)}@@ -138,7 +138,7 @@ func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {if len(res.JSON) != 2 {t.Fatalf("items = %d, want one HTML table per sheet", len(res.JSON))}- first, second := res.JSON[0], res.JSON[1]+ first, second := res.JSON[0].ToMap(), res.JSON[1].ToMap()if first["sheet"] != "Sheet1" || first["sheet_index"] != 1 {t.Fatalf("first sheet identity = %#v", first)}@@ -168,7 +168,7 @@ func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("items = %d, want one header-only segment", len(res.JSON))}- item := res.JSON[0]+ item := res.JSON[0].ToMap()wantText := "<table><caption>Sheet1</caption>\n" +"<tr><th>Name</th><th>Amount</th></tr>\n" +"</table>\n"@@ -233,7 +233,7 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- return res.JSON+ return ItemsToMaps(res.JSON)}assertSegments := func(t *testing.T, items []map[string]any, wantFirst, wantSecond []string) {@@ -332,7 +332,7 @@ func renderedText(item map[string]any) string {func spreadsheetText(res ParseResult) string {var out strings.Builder- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {if td, ok := item["table"].(*entity.TableData); ok {out.WriteString(tableutil.RenderTableHTML(td))}@@ -344,7 +344,7 @@ func spreadsheetText(res ParseResult) string {// joined by "; " — the rendered <th> cells of its first <tr> — or "" when no// table item exists.func spreadsheetHeaderText(res ParseResult) string {- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {td, ok := item["table"].(*entity.TableData)if !ok {continue@@ -534,7 +534,7 @@ func TestXLSXParser_MergedDataCellInheritance(t *testing.T) {if !strings.Contains(text, "<tr><td>G1</td><td>y</td></tr>") {t.Fatalf("slave row did not inherit merged master text, got:\n%s", text)}- positions := res.JSON[0]["positions"].([][]float64)+ positions := res.JSON[0].ToMap()["positions"].([][]float64)if len(positions) != 3 {t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex 09ba17d6ff..c09226bcdc 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -61,14 +61,15 @@ type ParseResult struct {// echoed the requested extension.File map[string]any- // JSON is the structured payload when available.- // Shape depends on the parser family: PDF emits- // `[]map[string]any` with `text` + `doc_type_kwd` keys (and- // optional `image` / `layout` / `positions` fields);- // Markdown / text emit normalized- // `{text, doc_type_kwd}` items; image emits an image payload and, after- // ingestion enhancement, its VLM description.- JSON []map[string]any+ // JSON is the structured payload when available, as typed DocItems.+ // Each item projects onto the historical wire form via DocItem.ToMap,+ // so the external key shape (text + doc_type_kwd, optional image /+ // layout / positions, table) is unchanged. Shape depends on the parser+ // family: PDF emits `text` + `doc_type_kwd` items (and optional+ // `image` / `layout` / `positions` fields); Markdown / HTML / text emit+ // normalized `{text, doc_type_kwd}` items; image emits an image payload+ // and, after ingestion enhancement, its VLM description.+ JSON []DocItem// Markdown is a backend response awaiting normalization.Markdown string@@ -101,30 +102,21 @@ const (spreadsheetOutputFormat = "json")-// NewTextJSONItem constructs a canonical text JSON item for parser output.-func NewTextJSONItem(text string) map[string]any {- return map[string]any{- "text": text,- DocTypeKey: DocTypeText,- }+// NewTextJSONItem constructs a canonical text DocItem for parser output.+func NewTextJSONItem(text string) DocItem {+ return TextItem{Text: text}}-// NewTableDataJSONItem constructs a canonical structured table JSON item for+// NewTableDataJSONItem constructs a canonical structured table DocItem for// parser output. It carries the parsed TableData contract (caption, header// rows, cell text already normalized the same way the legacy chunker// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,// so the chunker reads rows directly without re-parsing any markup.-func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) map[string]any {- item := map[string]any{- "table": td,- DocTypeKey: DocTypeTable,- "ck_type": DocTypeTable,- }- if sheet != "" {- item["sheet"] = sheet- }- if len(positions) > 0 {- item["positions"] = positions+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) DocItem {+ return TableItem{+ Table: td,+ Sheet: sheet,+ Positions: positions,+ CKType: DocTypeTable,}- return item}diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 10d6caf0d1..257b8af6e1 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -30,21 +30,22 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {Caption: "Sheet1",}item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})+ m := item.ToMap()- if got, want := item[DocTypeKey], DocTypeTable; got != want {+ if got, want := m[DocTypeKey], DocTypeTable; got != want {t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)}- if got, want := item["ck_type"], DocTypeTable; got != want {+ if got, want := m["ck_type"], DocTypeTable; got != want {t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)}- got, ok := item["table"].(*entity.TableData)+ got, ok := m["table"].(*entity.TableData)if !ok {- t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])+ t.Fatalf("item[%q] = %T, want *entity.TableData", "table", m["table"])}if got.Caption != "Sheet1" || len(got.Grid) != 2 {t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)}- if _, ok := item["positions"].([][]float64); !ok {+ if _, ok := m["positions"].([][]float64); !ok {t.Fatalf("item[%q] missing positions matrix", "positions")}}@@ -72,9 +73,9 @@ func TestParseResult_Contract(t *testing.T) {name: "json family only",in: ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: MapsToItems([]map[string]any{{"text": "hello", "doc_type_kwd": "text"},- },+ }),},wantFmt: "json",},@@ -170,14 +171,14 @@ func TestMarkdownParser_ParseWithResult(t *testing.T) {if len(res.JSON) != 3 {t.Fatalf("JSON len = %d, want 3 (heading, paragraph, list)", len(res.JSON))}- heading := res.JSON[0]+ heading := res.JSON[0].ToMap()if txt, _ := heading["text"].(string); !strings.Contains(txt, "Title") {t.Errorf("JSON[0].text = %q, want contains 'Title'", txt)}if got, _ := heading["ck_type"].(string); got != "heading" {t.Errorf("JSON[0].ck_type = %q, want heading", got)}- for i, it := range res.JSON {+ for i, it := range ItemsToMaps(res.JSON) {if _, ok := it["doc_type_kwd"]; !ok {t.Errorf("JSON[%d] missing doc_type_kwd: %+v", i, it)}diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.goindex d60782848b..7c51365021 100644--- a/internal/parser/parser/parse_with_result_test.go+++ b/internal/parser/parser/parse_with_result_test.go@@ -61,10 +61,10 @@ func TestTextParser_ParseWithResult_PreservesOneTextUnit(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1: %#v", len(res.JSON), res.JSON)}- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)}- got, _ := res.JSON[0]["text"].(string)+ got, _ := res.JSON[0].ToMap()["text"].(string)want := "First sentence.\n\nSecond sentence!\nThird sentence?"if got != want {t.Errorf("JSON[0].text = %q, want %q", got, want)@@ -103,7 +103,7 @@ func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))}- if txt, _ := res.JSON[0]["text"].(string); txt != long {+ if txt, _ := res.JSON[0].ToMap()["text"].(string); txt != long {t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))}}@@ -124,7 +124,7 @@ func TestTextParser_ParseWithResult_GBK(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("want non-empty items")}- got, _ := res.JSON[0]["text"].(string)+ got, _ := res.JSON[0].ToMap()["text"].(string)if got != "测试中文文本内容" {t.Errorf("got %q, want %q", got, "测试中文文本内容")}@@ -154,22 +154,22 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {if len(res.JSON) != 3 {t.Fatalf("JSON len = %d, want 3 (h1, p, ul)", len(res.JSON))}- if got, want := res.JSON[0]["ck_type"], "heading"; got != want {+ if got, want := res.JSON[0].ToMap()["ck_type"], "heading"; got != want {t.Errorf("JSON[0].ck_type = %v, want %v", got, want)}- if got, want := res.JSON[0]["text"], "Title"; got != want {+ if got, want := res.JSON[0].ToMap()["text"], "Title"; got != want {t.Errorf("JSON[0].text = %v, want %v", got, want)}- if got, want := res.JSON[1]["ck_type"], "paragraph"; got != want {+ if got, want := res.JSON[1].ToMap()["ck_type"], "paragraph"; got != want {t.Errorf("JSON[1].ck_type = %v, want %v", got, want)}- if got, want := res.JSON[1]["text"], "First paragraph."; got != want {+ if got, want := res.JSON[1].ToMap()["text"], "First paragraph."; got != want {t.Errorf("JSON[1].text = %v, want %v", got, want)}- if got, want := res.JSON[2]["ck_type"], "list"; got != want {+ if got, want := res.JSON[2].ToMap()["ck_type"], "list"; got != want {t.Errorf("JSON[2].ck_type = %v, want %v", got, want)}- if got, want := res.JSON[2]["text"], "Item one"; got != want {+ if got, want := res.JSON[2].ToMap()["text"], "Item one"; got != want {t.Errorf("JSON[2].text = %v, want %v", got, want)}}@@ -207,13 +207,13 @@ Tail textt.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)}for i, w := range want {- if got := res.JSON[i]["text"]; got != w.text {+ if got := res.JSON[i].ToMap()["text"]; got != w.text {t.Errorf("JSON[%d].text = %v, want %v", i, got, w.text)}- if got := res.JSON[i]["doc_type_kwd"]; got != "text" {+ if got := res.JSON[i].ToMap()["doc_type_kwd"]; got != "text" {t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)}- if got := res.JSON[i]["ck_type"]; got != w.ckType {+ if got := res.JSON[i].ToMap()["ck_type"]; got != w.ckType {t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)}}@@ -235,7 +235,7 @@ Tail text`)t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)}for i, text := range want {- if got := res.JSON[i]["text"]; got != text {+ if got := res.JSON[i].ToMap()["text"]; got != text {t.Errorf("JSON[%d].text = %v, want %v", i, got, text)}}@@ -273,7 +273,7 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {if len(res.JSON) != 3 {t.Errorf("JSON len = %d, want 3 (script+style+noscript skipped)", len(res.JSON))}- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {if txt, _ := it["text"].(string); strings.Contains(txt, "alert") ||strings.Contains(txt, "color") ||strings.Contains(txt, "inline alert") ||@@ -316,7 +316,7 @@ func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}out := make([]string, 0, len(res.JSON))- for _, it := range res.JSON {+ for _, it := range ItemsToMaps(res.JSON) {if txt, _ := it["text"].(string); txt != "" {out = append(out, txt)}@@ -382,7 +382,7 @@ func TestTextParser_AlignmentGolden(t *testing.T) {gd := LoadGoldenDoc(t, tc.golden)ignore := AcceptedDivergences(gd.Meta)- goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)+ goText := FilterOutDocTypes(FilterByDocType(ItemsToMaps(res.JSON), "text"), ignore)pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)delimiter, _ := gd.Meta["delimiter"].(string)diff --git a/internal/parser/parser/pdf_parser_cgo_test.go b/internal/parser/parser/pdf_parser_cgo_test.goindex a9effa4822..b0e3b9a475 100644--- a/internal/parser/parser/pdf_parser_cgo_test.go+++ b/internal/parser/parser/pdf_parser_cgo_test.go@@ -43,7 +43,7 @@ func TestPDFParser_ParseWithResult_CGOEmpty(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- if text, _ := res.JSON[0]["text"].(string); strings.TrimSpace(text) != "" {+ if text, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(text) != "" {t.Fatalf("JSON[0].text = %q, want empty", text)}}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 197d25c358..af5f7576f3 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -411,7 +411,7 @@ func emptyPDFResult(filename string) ParseResult {"page_count": 0,"outline": []map[string]any{},},- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},+ JSON: []DocItem{NewTextJSONItem("")},}}@@ -507,7 +507,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR"page_count": len(processed.PageHeight),"outline": outlinesToFileMeta(processed.Outlines),},- JSON: items,+ JSON: MapsToItems(items),}}@@ -1034,8 +1034,10 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []bytreturn ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}}for i := range res.JSON {- if img, _ := res.JSON[i]["image"].(string); img != "" {- res.JSON[i]["image"] = pdflayout.InlinePNGDataURL(img)+ m := res.JSON[i].ToMap()+ if img, _ := m["image"].(string); img != "" {+ m["image"] = pdflayout.InlinePNGDataURL(img)+ res.JSON[i] = DocItemFromMap(m)}}return resdiff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.goindex 584171bc68..64e43eb63c 100644--- a/internal/parser/parser/pdf_parser_common_test.go+++ b/internal/parser/parser/pdf_parser_common_test.go@@ -64,39 +64,39 @@ func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {if len(res.JSON) != 2 {t.Fatalf("JSON len = %d, want 2", len(res.JSON))}- if got, want := res.JSON[0]["layout"], "title"; got != want {+ if got, want := res.JSON[0].ToMap()["layout"], "title"; got != want {t.Fatalf("JSON[0].layout = %v, want %v", got, want)}- if got, want := res.JSON[0]["page_number"], 1; got != want {+ if got, want := res.JSON[0].ToMap()["page_number"], 1; got != want {t.Fatalf("JSON[0].page_number = %v, want %v", got, want)}- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)}- pdfPositions, ok := res.JSON[0]["_pdf_positions"].([][]any)+ pdfPositions, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)if !ok {- t.Fatalf("JSON[0]._pdf_positions type = %T, want [][]any", res.JSON[0]["_pdf_positions"])+ t.Fatalf("JSON[0]._pdf_positions type = %T, want [][]any", res.JSON[0].ToMap()["_pdf_positions"])}if len(pdfPositions) != 1 || pdfPositions[0][0] != 1 {t.Fatalf("JSON[0]._pdf_positions = %+v, want canonical 1-based positions", pdfPositions)}- if got := res.JSON[0]["positions"]; got == nil {+ if got := res.JSON[0].ToMap()["positions"]; got == nil {t.Fatal("JSON[0].positions missing after normalization")}- if got, want := res.JSON[1]["doc_type_kwd"], "image"; got != want {+ if got, want := res.JSON[1].ToMap()["doc_type_kwd"], "image"; got != want {t.Fatalf("JSON[1].doc_type_kwd = %v, want %v", got, want)}- if got, want := res.JSON[1]["page_number"], 2; got != want {+ if got, want := res.JSON[1].ToMap()["page_number"], 2; got != want {t.Fatalf("JSON[1].page_number = %v, want %v", got, want)}- secondPDFPositions, ok := res.JSON[1]["_pdf_positions"].([][]any)+ secondPDFPositions, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)if !ok {- t.Fatalf("JSON[1]._pdf_positions type = %T, want [][]any", res.JSON[1]["_pdf_positions"])+ t.Fatalf("JSON[1]._pdf_positions type = %T, want [][]any", res.JSON[1].ToMap()["_pdf_positions"])}if len(secondPDFPositions) != 1 || secondPDFPositions[0][0] != 2 {t.Fatalf("JSON[1]._pdf_positions = %+v, want canonical 1-based positions (DeepDoc page 1 → 2)", secondPDFPositions)}- if got, want := res.JSON[1]["image"], "data:image/png;base64,aGVsbG8="; got != want {+ if got, want := res.JSON[1].ToMap()["image"], "data:image/png;base64,aGVsbG8="; got != want {t.Fatalf("JSON[1].image = %v, want %v", got, want)}}@@ -128,7 +128,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {if res.Err != nil {t.Fatalf("pdfParseResultToJSON: %v", res.Err)}- if got := res.JSON[0]["doc_type_kwd"]; got != "image" {+ if got := res.JSON[0].ToMap()["doc_type_kwd"]; got != "image" {t.Fatalf("figure caption with positions doc_type_kwd = %v, want image", got)}@@ -148,7 +148,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {if res2.Err != nil {t.Fatalf("pdfParseResultToJSON: %v", res2.Err)}- if got := res2.JSON[0]["doc_type_kwd"]; got != "text" {+ if got := res2.JSON[0].ToMap()["doc_type_kwd"]; got != "text" {t.Fatalf("figure caption with image but no positions doc_type_kwd = %v, want text", got)}}@@ -211,10 +211,10 @@ func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {res := pdfParseResultToJSON("one-based.pdf", parsed)// DeepDoc page 3 is 0-indexed (the 4th page); normalizePDFPageNumber// converts it to 1-indexed page 4.- if got, want := res.JSON[0]["page_number"], 4; got != want {+ if got, want := res.JSON[0].ToMap()["page_number"], 4; got != want {t.Fatalf("JSON[0].page_number = %v, want %v", got, want)}- if got, want := res.JSON[0]["doc_type_kwd"], "table"; got != want {+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "table"; got != want {t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)}}@@ -227,7 +227,7 @@ func TestPDFParseResultToJSON_EmptySectionsStillEmitPlaceholder(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON len = %d, want 1", len(res.JSON))}- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)}}@@ -261,13 +261,13 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) {// Sections are now sorted by (page, top, left). Header and Footer have// no position data (page=0, top=0), Body has top=30, so the sorted order// is Header/Footer (tied top=0, stable) then Body (top=30).- if got, want := res.JSON[0]["text"], "Header"; got != want {+ if got, want := res.JSON[0].ToMap()["text"], "Header"; got != want {t.Fatalf("JSON[0].text = %v, want %v", got, want)}- if got, want := res.JSON[1]["text"], "Footer"; got != want {+ if got, want := res.JSON[1].ToMap()["text"], "Footer"; got != want {t.Fatalf("JSON[1].text = %v, want %v", got, want)}- if got, want := res.JSON[2]["text"], "Body"; got != want {+ if got, want := res.JSON[2].ToMap()["text"], "Body"; got != want {t.Fatalf("JSON[2].text = %v, want %v", got, want)}}@@ -671,25 +671,25 @@ func TestPDFParseResultToJSON_NoInlineRetainsPositions(t *testing.T) {}// Figure must retain positions but NOT inline an image.- figPos, ok := res.JSON[0]["_pdf_positions"].([][]any)+ figPos, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)if !ok || len(figPos) == 0 {- t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0]["_pdf_positions"])+ t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0].ToMap()["_pdf_positions"])}- if img, _ := res.JSON[0]["image"].(string); img != "" {+ if img, _ := res.JSON[0].ToMap()["image"].(string); img != "" {t.Fatalf("Figure should not inline image under cgo, got %q", img)}// Table likewise retains positions, no inline image.- tblPos, ok := res.JSON[1]["_pdf_positions"].([][]any)+ tblPos, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)if !ok || len(tblPos) == 0 {- t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1]["_pdf_positions"])+ t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1].ToMap()["_pdf_positions"])}- if img, _ := res.JSON[1]["image"].(string); img != "" {+ if img, _ := res.JSON[1].ToMap()["image"].(string); img != "" {t.Fatalf("Table should not inline image under cgo, got %q", img)}// Plain text: no image, positions retained.- if img, _ := res.JSON[2]["image"].(string); img != "" {+ if img, _ := res.JSON[2].ToMap()["image"].(string); img != "" {t.Fatalf("Text should not inline image, got %q", img)}}@@ -719,13 +719,13 @@ func TestPDFParseResultToJSON_FigureCaptionNoInline(t *testing.T) {if res.Err != nil {t.Fatalf("pdfParseResultToJSON: %v", res.Err)}- if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "image"; got != want {t.Fatalf("doc_type_kwd = %v, want %v", got, want)}- if pos, ok := res.JSON[0]["_pdf_positions"].([][]any); !ok || len(pos) == 0 {+ if pos, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any); !ok || len(pos) == 0 {t.Fatal("figure caption should retain _pdf_positions for on-demand crop")}- if image, _ := res.JSON[0]["image"].(string); image != "" {+ if image, _ := res.JSON[0].ToMap()["image"].(string); image != "" {t.Fatalf("figure caption should not be inlined by the parser, got %q", image)}}diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex ac49389613..b08022df2f 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -261,7 +261,7 @@ func doclingTextsToResult(filename string, texts []string, outputFormat string,return ParseResult{OutputFormat: "json",File: fileMeta,- JSON: items,+ JSON: MapsToItems(items),}case "markdown":return ParseResult{diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.goindex 2de37ad80b..83219849a5 100644--- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go+++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go@@ -50,7 +50,7 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {if got := res.File["page_count"]; got == nil {t.Fatal("File.page_count missing")}- if positions, ok := res.JSON[0]["_pdf_positions"].([][]any); ok && len(positions) == 0 {+ if positions, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any); ok && len(positions) == 0 {t.Fatal("JSON[0]._pdf_positions is empty; want normalized positions for fixture text")}}@@ -103,7 +103,7 @@ func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("JSON is empty; want page text items")}- if got, _ := res.JSON[0]["text"].(string); strings.TrimSpace(got) == "" {+ if got, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(got) == "" {t.Fatal("plain_text first page is empty")}}diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.goindex 87304f7090..adaefaf5a6 100644--- a/internal/parser/parser/pdf_parser_mineru_test.go+++ b/internal/parser/parser/pdf_parser_mineru_test.go@@ -135,7 +135,7 @@ func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("JSON is empty; want markdown-normalized items")}- if got := res.JSON[0]["text"]; got == nil {+ if got := res.JSON[0].ToMap()["text"]; got == nil {t.Fatal("JSON[0].text missing")}}diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.goindex 2bdfee283c..172c1d7c53 100644--- a/internal/parser/parser/pdf_parser_paddleocr_test.go+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go@@ -125,7 +125,7 @@ func TestPDFParser_ParseWithResult_PaddleOCRJSONIntegration(t *testing.T) {if len(res.JSON) == 0 {t.Fatal("JSON is empty; want markdown-normalized items")}- if got := res.JSON[0]["text"]; got == nil {+ if got := res.JSON[0].ToMap()["text"]; got == nil {t.Fatal("JSON[0].text missing")}}diff --git a/internal/parser/parser/pdf_parser_pages_test.go b/internal/parser/parser/pdf_parser_pages_test.goindex 88316ae0fb..d12b0e01ef 100644--- a/internal/parser/parser/pdf_parser_pages_test.go+++ b/internal/parser/parser/pdf_parser_pages_test.go@@ -101,7 +101,7 @@ func TestPDFParser_PlainTextHonorsPages(t *testing.T) {t.Fatalf("ParseWithResult: %v", res.Err)}var pageNumbers []any- for _, item := range res.JSON {+ for _, item := range ItemsToMaps(res.JSON) {pageNumbers = append(pageNumbers, item["page_number"])}if want := []any{2, 3}; !reflect.DeepEqual(pageNumbers, want) {diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex a26af71a1d..57d6df306f 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -46,7 +46,7 @@ func pdfItemsToResult(filename string, items []map[string]any, outputFormat strireturn ParseResult{OutputFormat: "json",File: pdfFileMeta(filename, pageCount),- JSON: items,+ JSON: MapsToItems(items),}case "markdown":var b strings.Builderdiff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.goindex cb97a28118..c5f7bc1e0a 100644--- a/internal/parser/parser/picture_parser.go+++ b/internal/parser/parser/picture_parser.go@@ -114,7 +114,7 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da"size": len(data),"doc_type_kwd": DocTypeImage,},- JSON: []map[string]any{item},+ JSON: []DocItem{DocItemFromMap(item)},}}diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.goindex 6190b410ea..5dc809790c 100644--- a/internal/parser/parser/picture_parser_test.go+++ b/internal/parser/parser/picture_parser_test.go@@ -28,12 +28,12 @@ func TestPictureParserKeepsImageWithoutOCR(t *testing.T) {if res.Err != nil {t.Fatal(res.Err)}- if len(res.JSON) != 1 || res.JSON[0]["text"] != "" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "" {t.Fatalf("picture text = %+v, want empty text", res.JSON)}wantImage := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)- if res.JSON[0]["image"] != wantImage {- t.Fatalf("picture image = %v, want %q", res.JSON[0]["image"], wantImage)+ if res.JSON[0].ToMap()["image"] != wantImage {+ t.Fatalf("picture image = %v, want %q", res.JSON[0].ToMap()["image"], wantImage)}if len(res.Warnings) != 0 {t.Fatalf("picture warnings = %v, want none", res.Warnings)diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.goindex 010761deba..5aad089300 100644--- a/internal/parser/parser/ppt_parser_cgo_test.go+++ b/internal/parser/parser/ppt_parser_cgo_test.go@@ -85,7 +85,7 @@ func TestPPTXParser_ParseWithResult_MultiSlide_CGO(t *testing.T) {{"Slide One Title\nFirst slide body about the Alpha topic.", 1},{"Slide Two Title\nSecond slide body about the Beta topic.", 2},} {- it := res.JSON[i]+ it := res.JSON[i].ToMap()if got := it["text"]; got != want.text {t.Errorf("item %d text = %q, want %q", i, got, want.text)}@@ -124,10 +124,10 @@ func TestPPTXParser_ParseWithResult_EmptySlide_CGO(t *testing.T) {if len(res.JSON) != 3 {t.Fatalf("JSON items = %d, want 3 (one per slide): %#v", len(res.JSON), res.JSON)}- if got := res.JSON[1]["text"]; got != "" {+ if got := res.JSON[1].ToMap()["text"]; got != "" {t.Errorf("empty slide text = %q, want empty", got)}- if got := res.JSON[1]["slide_number"]; got != 2 {+ if got := res.JSON[1].ToMap()["slide_number"]; got != 2 {t.Errorf("empty slide slide_number = %v, want 2", got)}}diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.goindex 5a2420df7c..61cc0a4a77 100644--- a/internal/parser/parser/pptx_parser.go+++ b/internal/parser/parser/pptx_parser.go@@ -158,7 +158,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{OutputFormat: "json",File: map[string]any{"name": filename, "format": effFormat},- JSON: itemsFromPlainText(plainText),+ JSON: MapsToItems(itemsFromPlainText(plainText)),}}budget := newEmbeddedMediaBudget()@@ -184,7 +184,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, datareturn ParseResult{OutputFormat: "json",File: map[string]any{"name": filename, "format": effFormat},- JSON: items,+ JSON: MapsToItems(items),Warnings: budget.warnings(),}}diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.goindex 1c1efd33b0..e1f00eff0e 100644--- a/internal/parser/parser/text_parser.go+++ b/internal/parser/parser/text_parser.go@@ -53,7 +53,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data"size": len(data),"encoding": encName,},- JSON: []map[string]any{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},+ JSON: []DocItem{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},}}diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.goindex dfaf702cac..d22e1758c6 100644--- a/internal/parser/parser/xls_parser.go+++ b/internal/parser/parser/xls_parser.go@@ -115,7 +115,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [return ParseResult{OutputFormat: spreadsheetOutputFormat,File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},- JSON: items,+ JSON: MapsToItems(items),Warnings: warnings,}}diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.goindex 98b7b4ee29..95224ff5b2 100644--- a/internal/parser/parser/xls_parser_test.go+++ b/internal/parser/parser/xls_parser_test.go@@ -140,7 +140,7 @@ func TestXLSParser_PreservesFormulaErrors(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("res.JSON len = %d, want 1", len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, "<tr><td>#DIV/0!</td></tr>") {t.Fatalf("table markup = %q, want formula error row", text)}@@ -173,7 +173,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {t.Fatalf("res.JSON len = %d, want one table for each sheet", len(res.JSON))}- first := res.JSON[0]+ first := res.JSON[0].ToMap()text := tableItemHTML(t, first)if !strings.Contains(text, "<caption>Inventory</caption>") ||!strings.Contains(text, "<tr><th>Name</th><th>Count</th></tr>") ||@@ -194,7 +194,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {t.Fatalf("last row position = %v, want %v", got, want)}- second := res.JSON[1]+ second := res.JSON[1].ToMap()secondText := tableItemHTML(t, second)if !strings.Contains(secondText, "<caption>Notes</caption>") ||!strings.Contains(secondText, "<tr><td>Legacy</td><td>BIFF8</td></tr>") {@@ -224,7 +224,7 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {if len(res.JSON) == 0 {t.Fatalf("res.JSON should have structured table items")}- item := res.JSON[0]+ item := res.JSON[0].ToMap()if item["ck_type"] != "table" {t.Errorf("table item ck_type = %v, want table", item["ck_type"])}@@ -271,8 +271,8 @@ func TestXLSXParser_JSONOutput(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)}- if res.JSON[0]["ck_type"] != "table" {- t.Errorf("item ck_type = %v, want table", res.JSON[0]["ck_type"])+ if res.JSON[0].ToMap()["ck_type"] != "table" {+ t.Errorf("item ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])}// Even if configured to legacy "html", format is unified to "json"@@ -326,7 +326,7 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {t.Fatalf("ParseWithResult: %v", result.Err)}var imageItem map[string]any- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] == "image" {imageItem = itembreak@@ -375,7 +375,7 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {t.Fatalf("ParseWithResult: %v", result.Err)}imageCount := 0- for _, item := range result.JSON {+ for _, item := range ItemsToMaps(result.JSON) {if item["doc_type_kwd"] == "image" {imageCount++}diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.goindex f623c3ce98..7768be6cc6 100644--- a/internal/parser/parser/xls_tcadp_test.go+++ b/internal/parser/parser/xls_tcadp_test.go@@ -228,7 +228,7 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table" {+ if res.JSON[0].ToMap()["ck_type"] != "table" {t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])}}diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.goindex e07b2875d7..43ea5bf7a4 100644--- a/internal/parser/parser/xlsx_parse_method_test.go+++ b/internal/parser/parser/xlsx_parse_method_test.go@@ -80,7 +80,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))}- text := tableItemHTML(t, res.JSON[0])+ text := tableItemHTML(t, res.JSON[0].ToMap())if !strings.Contains(text, tc.cellValue) {t.Fatalf("JSON = %#v, want it to contain cell content %q", res.JSON, tc.cellValue)}@@ -113,7 +113,7 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {if len(res.JSON) != 2 {t.Fatalf("JSON item count = %d, want table and image", len(res.JSON))}- image := res.JSON[1]+ image := res.JSON[1].ToMap()if image["text"] != "sheet image" || image["doc_type_kwd"] != "image" {t.Fatalf("unexpected image item: %#v", image)}@@ -145,7 +145,7 @@ func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {if res.Err != nil {t.Fatalf("ParseWithResult: %v", res.Err)}- if len(res.JSON) != 1 || res.JSON[0]["text"] != "C3" {+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "C3" {t.Fatalf("image item = %#v, want anchor-cell text C3", res.JSON)}}@@ -177,7 +177,7 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {if len(res.JSON) != 1 {t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))}- if res.JSON[0]["ck_type"] != "table" {+ if res.JSON[0].ToMap()["ck_type"] != "table" {t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])}}diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.goindex e8cc2ee7ed..b0dc873283 100644--- a/internal/parser/parser/xlsx_parser.go+++ b/internal/parser/parser/xlsx_parser.go@@ -177,7 +177,7 @@ func xlsxParseResult(filename string, items []map[string]any, warnings []string,return ParseResult{OutputFormat: spreadsheetOutputFormat,File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},- JSON: items,+ JSON: MapsToItems(items),Warnings: warnings,}}diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.goindex 60cde67bd4..4a702252cd 100644--- a/internal/service/file/file_content.go+++ b/internal/service/file/file_content.go@@ -235,15 +235,16 @@ func parseResultText(res parser.ParseResult) (string, error) {if len(res.JSON) > 0 {parts := make([]string, 0, len(res.JSON))for _, item := range res.JSON {- if text, ok := item["text"].(string); ok {+ m := item.ToMap()+ if text, ok := m["text"].(string); ok {parts = append(parts, text)continue}- if td, ok := item["table"].(*entity.TableData); ok && td != nil {+ if td, ok := m["table"].(*entity.TableData); ok && td != nil {parts = append(parts, tableutil.RenderTableHTML(td))continue}- raw, err := json.Marshal(item)+ raw, err := json.Marshal(m)if err != nil {return "", err}diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.goindex b0421c0681..b50ea119e9 100644--- a/internal/service/file/file_content_test.go+++ b/internal/service/file/file_content_test.go@@ -60,11 +60,11 @@ func TestParseFileContent_JSONItems(t *testing.T) {func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T) {result, err := parseResultText(parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": "first"},{"text": "second"},{"value": 3},- },+ }),})if err != nil {t.Fatalf("parseResultText: %v", err)@@ -82,9 +82,9 @@ func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {segment := "<table><caption>Data</caption>\n<tr><th>a</th><th>b</th></tr>\n<tr><td>1</td><td>2</td></tr>\n</table>\n"result, err := parseResultText(parser.ParseResult{OutputFormat: "json",- JSON: []map[string]any{+ JSON: parser.MapsToItems([]map[string]any{{"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},- },+ }),})if err != nil {t.Fatalf("parseResultText: %v", err)diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.goindex 85d519411f..c9e12088f3 100644--- a/tools/mdprobe/main.go+++ b/tools/mdprobe/main.go@@ -47,10 +47,11 @@ func main() {fmt.Printf("file=%s bytes=%d output_format=%s items=%d\n", filepath.Base(path), len(data), res.OutputFormat, len(res.JSON))var total intfor i, item := range res.JSON {- text, _ := item["text"].(string)+ m := item.ToMap()+ text, _ := m["text"].(string)total += len(text)- ck := fmt.Sprint(item["ck_type"])- docType := fmt.Sprint(item["doc_type_kwd"])+ ck := fmt.Sprint(m["ck_type"])+ docType := fmt.Sprint(m["doc_type_kwd"])fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, ck, docType, len(text), truncate(text, 90))}fmt.Printf("parser emitted %d chars total (file %d)\n", total, len(data))@@ -59,7 +60,7 @@ func main() {lower := strings.ToLower(needle)found := -1for i, item := range res.JSON {- if text, _ := item["text"].(string); strings.Contains(strings.ToLower(text), lower) {+ if text, _ := item.ToMap()["text"].(string); strings.Contains(strings.ToLower(text), lower) {found = ibreak}