Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): type ParseResult.JSON as []DocItem boundary adapter (#20998)

xugangqiangeabad0c74c73
diff --git a/internal/ingestion/component/chunker/sheet_identity_test.go b/internal/ingestion/component/chunker/sheet_identity_test.go
index ef60f8aabc..64a7709391 100644
--- a/internal/ingestion/component/chunker/sheet_identity_test.go
+++ b/internal/ingestion/component/chunker/sheet_identity_test.go
@@ -49,12 +49,12 @@ func TestSpreadsheetChunkersPreserveSheetIdentityThroughIndexing(t *testing.T) {
factory func(map[string]any) (runtime.Component, error)
}{{"QA", NewQAChunker}, {"Token", NewTokenChunker}, {"One", NewOneChunker}} {
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
- batches := [][]map[string]any{source.result.JSON}
+ batches := [][]map[string]any{parser.ItemsToMaps(source.result.JSON)}
// One preserves positions only for a single source item.
if tc.name == "One" {
batches = nil
for _, item := range source.result.JSON {
- batches = append(batches, []map[string]any{item})
+ batches = append(batches, []map[string]any{item.ToMap()})
}
}
for _, items := range batches {
@@ -147,7 +147,9 @@ func TestQAChunkerSheetNameOnlyPreservesSourceRows(t *testing.T) {
t.Fatalf("parser items=%d, want 1", len(parsed.JSON))
}
// Keep real parser selections, but exercise the supported name-only identity.
- delete(parsed.JSON[0], "sheet_index")
+ m := parsed.JSON[0].ToMap()
+ delete(m, "sheet_index")
+ parsed.JSON[0] = parser.RawItem{M: m}
comp, err := NewQAChunker(nil)
if err != nil {
t.Fatal(err)
diff --git a/internal/ingestion/component/chunker/token.go b/internal/ingestion/component/chunker/token.go
index 1b8cd2fc72..be1ca2a73a 100644
--- a/internal/ingestion/component/chunker/token.go
+++ b/internal/ingestion/component/chunker/token.go
@@ -85,6 +85,7 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/ingestion/component/globals"
"ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
"ragflow/internal/parser/tableutil"
"ragflow/internal/parser/chunk"
@@ -247,7 +248,22 @@ func (c *TokenChunkerComponent) invoke(ctx context.Context, db *gorm.DB, inputs
func decodeChunkerFromUpstream(inputs map[string]any) (schema.ChunkerFromUpstream, error) {
var out schema.ChunkerFromUpstream
- data, err := json.Marshal(stripChunkerRuntimeTimestamps(inputs))
+ // Fast path: the parser emits typed []parser.DocItem. Convert it to
+ // []ChunkDoc directly via schema.ChunkDocsFromDocItem (which reads the
+ // item's typed fields and never calls DocItem.ToMap), then let the JSON
+ // marshal below serialize the already-typed ChunkDoc. This keeps the
+ // chunker free of the historical map intermediate for the in-memory
+ // Parser -> Chunker leg. When inputs["json"] is already []map[string]any
+ // (legacy fixtures, round-tripped JSON, or absent), the original path runs.
+ inputs = stripChunkerRuntimeTimestamps(inputs)
+ if docs, ok := inputs["json"].([]parser.DocItem); ok && len(docs) > 0 {
+ conv, _, err := schema.ChunkDocsFromDocItem(docs)
+ if err != nil {
+ return out, err
+ }
+ inputs["json"] = conv
+ }
+ data, err := json.Marshal(inputs)
if err != nil {
return out, err
}
diff --git a/internal/ingestion/component/media_dispatch.go b/internal/ingestion/component/media_dispatch.go
index d5c0cd2afe..419f3a9180 100644
--- a/internal/ingestion/component/media_dispatch.go
+++ b/internal/ingestion/component/media_dispatch.go
@@ -125,8 +125,10 @@ func maybeDispatchImage(
return parsed, true, fmt.Errorf("parser: image parser returned no image item")
}
parsed.OutputFormat = "json"
- imageData, _ := parsed.JSON[0]["image"].(string)
- parsed.JSON[0]["text"] = text
+ imageData, _ := parsed.JSON[0].ToMap()["image"].(string)
+ m := parsed.JSON[0].ToMap()
+ m["text"] = text
+ parsed.JSON[0] = parser.DocItemFromMap(m)
if err != nil {
parsed.Warnings = append(parsed.Warnings, fmt.Sprintf("image OCR unavailable: %v", err))
} else if useOCR && strings.TrimSpace(text) == "" {
@@ -155,7 +157,9 @@ func maybeDispatchImage(
description, warnings := describeImage(ctx, db, imageData, getStringOr(inputs, "tenant_id", ""), setup, inputs, imageVision)
parsed.Warnings = append(parsed.Warnings, warnings...)
if description != "" {
- appendItemText(parsed.JSON[0], description)
+ m := parsed.JSON[0].ToMap()
+ appendItemText(m, description)
+ parsed.JSON[0] = parser.DocItemFromMap(m)
} else if strings.TrimSpace(text) == "" {
// Enhancement was permitted and the model produced nothing usable,
// so the item still has no text at all. Without this the chunk is
@@ -361,10 +365,7 @@ func maybeDispatchAudio(
return parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
- "text": transcription,
- "doc_type_kwd": "text",
- }},
+ JSON: []parser.DocItem{parser.NewTextJSONItem(transcription)},
}, true, nil
}
diff --git a/internal/ingestion/component/media_dispatch_integration_test.go b/internal/ingestion/component/media_dispatch_integration_test.go
index 2e6e53b490..69e808f3f7 100644
--- a/internal/ingestion/component/media_dispatch_integration_test.go
+++ b/internal/ingestion/component/media_dispatch_integration_test.go
@@ -46,12 +46,13 @@ func imageSetupsWithSwitch(ocrEnabled bool) map[string]schema.ParserSetup {
func assertNativeOCRTextAndImage(t *testing.T, result parser.ParseResult) {
t.Helper()
- text, _ := result.JSON[0]["text"].(string)
+ maps := parser.ItemsToMaps(result.JSON)
+ text, _ := maps[0]["text"].(string)
if strings.Join(strings.Fields(strings.ToUpper(text)), "") != pictureOCRExpectedText {
t.Fatalf("native OCR text = %q, want HELLO WORLD: warnings = %v", text, result.Warnings)
}
- if result.JSON[0]["image"] == "" || result.JSON[0]["doc_type_kwd"] != "image" {
- t.Fatalf("missing image attachment: %+v", result.JSON[0])
+ if maps[0]["image"] == "" || maps[0]["doc_type_kwd"] != "image" {
+ t.Fatalf("missing image attachment: %+v", maps[0])
}
}
@@ -101,11 +102,12 @@ func TestMaybeDispatchImageNativeSwitchOffRunsNoOCR(t *testing.T) {
if !handled || len(result.JSON) != 1 {
t.Fatalf("result = %+v, handled = %v", result, handled)
}
- if text, _ := result.JSON[0]["text"].(string); text != "" {
+ maps := parser.ItemsToMaps(result.JSON)
+ if text, _ := maps[0]["text"].(string); text != "" {
t.Fatalf("OCR must not run with the switch off, got text %q", text)
}
- if result.JSON[0]["image"] == "" {
- t.Fatalf("image attachment must survive: %+v", result.JSON[0])
+ if maps[0]["image"] == "" {
+ t.Fatalf("image attachment must survive: %+v", maps[0])
}
if !strings.Contains(strings.Join(result.Warnings, "\n"), "the Tokenizer will discard this item") {
t.Fatalf("expected a no-text-source warning, got %v", result.Warnings)
diff --git a/internal/ingestion/component/media_dispatch_test.go b/internal/ingestion/component/media_dispatch_test.go
index 9e5d844e87..38a3b4f775 100644
--- a/internal/ingestion/component/media_dispatch_test.go
+++ b/internal/ingestion/component/media_dispatch_test.go
@@ -130,7 +130,8 @@ func TestMaybeDispatchImageOCRIndependentOfVision(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- items, _ := result["json"].([]map[string]any)
+ jsonDocs := result["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if len(items) != 1 || items[0]["text"] != tc.want {
t.Fatalf("items = %+v, want text %q", items, tc.want)
}
@@ -200,13 +201,13 @@ func TestMaybeDispatchImage_ParseMethodModelDoesNotOverrideGlobalEnhancement(t *
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- text, _ := res.JSON[0]["text"].(string)
+ text, _ := res.JSON[0].ToMap()["text"].(string)
if strings.TrimSpace(text) != "" {
t.Fatalf("image item text = %q, want empty: enhancement is off and OCR is off", text)
}
// The image itself is still carried; only the description is withheld, so
// turning the switch on later recovers the caption without a re-upload.
- if res.JSON[0]["image"] == "" || res.JSON[0]["doc_type_kwd"] != "image" {
+ if res.JSON[0].ToMap()["image"] == "" || res.JSON[0].ToMap()["doc_type_kwd"] != "image" {
t.Fatalf("image attachment lost: %+v", res.JSON[0])
}
// #20668 reported this exact configuration producing zero chunks with no
@@ -241,7 +242,7 @@ func TestMaybeDispatchImage_OCRWithoutEnhancementSkipsVision(t *testing.T) {
if resolveModelConfigCalled {
t.Fatal("resolveModelConfig must not be called for parse_method=ocr without enhancement")
}
- if len(res.JSON) != 1 || res.JSON[0]["text"] != "OCR text" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "OCR text" {
t.Fatalf("res = %+v, want OCR text", res.JSON)
}
}
@@ -350,7 +351,7 @@ func TestMaybeDispatchImage_UsesSystemPrompt(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1 (image result must be a single JSON item)", len(res.JSON))
}
- if txt, _ := res.JSON[0]["text"].(string); txt == "" {
+ if txt, _ := res.JSON[0].ToMap()["text"].(string); txt == "" {
t.Fatalf("expected non-empty combined text in JSON[0][\"text\"]")
}
@@ -505,7 +506,7 @@ func TestMaybeDispatchImage_ReturnsJSONWithImage(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- item := res.JSON[0]
+ item := res.JSON[0].ToMap()
if got, _ := item["doc_type_kwd"].(string); got != "image" {
t.Errorf("doc_type_kwd = %q, want \"image\"", got)
}
@@ -608,10 +609,10 @@ func TestMaybeDispatchAudio_JSONCarriesTranscription(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1 (transcription must be carried as a JSON item)", len(res.JSON))
}
- if got, _ := res.JSON[0]["text"].(string); got != want {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {
t.Fatalf("JSON[0].text = %q, want %q", got, want)
}
- if got, _ := res.JSON[0]["doc_type_kwd"].(string); got != "text" {
+ if got, _ := res.JSON[0].ToMap()["doc_type_kwd"].(string); got != "text" {
t.Fatalf("JSON[0].doc_type_kwd = %q, want text", got)
}
}
@@ -651,7 +652,7 @@ func TestMaybeDispatchAudio_IgnoresLegacyOutputFormat(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- if got, _ := res.JSON[0]["text"].(string); got != want {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != want {
t.Fatalf("JSON[0].text = %q, want %q", got, want)
}
}
@@ -804,7 +805,7 @@ func TestMaybeDispatchAudio_UsesConfiguredModel(t *testing.T) {
if tenantResolverCalled {
t.Error("tenant default resolver must not be called when audio setup vlm.llm_id is set")
}
- if len(res.JSON) != 1 || res.JSON[0]["text"] != "transcribed" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "transcribed" {
t.Fatalf("unexpected audio result: %#v", res.JSON)
}
}
@@ -866,7 +867,7 @@ func TestMaybeDispatchImageWithoutOCRTextKeepsImage(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" || len(result.Warnings) == 0 {
+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "" || result.JSON[0].ToMap()["image"] == "" || len(result.Warnings) == 0 {
t.Fatalf("method %q, available %v: result = %+v, handled = %v", method, available, result, handled)
}
}
@@ -890,7 +891,7 @@ func TestMaybeDispatchImageUnhealthyOCRKeepsImage(t *testing.T) {
if err != nil || result.Err != nil {
t.Fatalf("unhealthy OCR should degrade without an error: err = %v, result.Err = %v", err, result.Err)
}
- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "" || result.JSON[0]["image"] == "" {
+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "" || result.JSON[0].ToMap()["image"] == "" {
t.Fatalf("unhealthy OCR should retain an image without text: result = %+v, handled = %v", result, handled)
}
if !strings.Contains(strings.Join(result.Warnings, "\n"), "image OCR unavailable: local OCR analyzer is unavailable") {
@@ -960,7 +961,7 @@ func TestMaybeDispatchImageDecodesRasterOnlyForOCR(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if !handled || len(result.JSON) != 1 || result.JSON[0]["text"] != "captured" || result.JSON[0]["image"] == "" {
+ if !handled || len(result.JSON) != 1 || result.JSON[0].ToMap()["text"] != "captured" || result.JSON[0].ToMap()["image"] == "" {
t.Fatalf("VLM result = %+v, handled = %v", result, handled)
}
})
@@ -997,8 +998,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
}
- if res.JSON[0]["text"] != "OCR text" {
- t.Fatalf("text = %v, want OCR text", res.JSON[0]["text"])
+ if res.JSON[0].ToMap()["text"] != "OCR text" {
+ t.Fatalf("text = %v, want OCR text", res.JSON[0].ToMap()["text"])
}
})
t.Run("ocr-off-no-enhancement-keeps-empty", func(t *testing.T) {
@@ -1007,7 +1008,7 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
}
- if res.JSON[0]["text"] != "" || res.JSON[0]["image"] == "" {
+ if res.JSON[0].ToMap()["text"] != "" || res.JSON[0].ToMap()["image"] == "" {
t.Fatalf("want empty text with image retained, got %#v", res.JSON[0])
}
if len(res.Warnings) == 0 {
@@ -1023,8 +1024,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
}
- if res.JSON[0]["text"] != "captured" {
- t.Fatalf("text = %v, want captured (VLM only)", res.JSON[0]["text"])
+ if res.JSON[0].ToMap()["text"] != "captured" {
+ t.Fatalf("text = %v, want captured (VLM only)", res.JSON[0].ToMap()["text"])
}
})
t.Run("switch-beats-stale-parse-method", func(t *testing.T) {
@@ -1035,8 +1036,8 @@ func TestMaybeDispatchImageOCRSwitch(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if res.JSON[0]["text"] != "" {
- t.Fatalf("ocr_enabled:false must override stale parse_method, got text %v", res.JSON[0]["text"])
+ if res.JSON[0].ToMap()["text"] != "" {
+ t.Fatalf("ocr_enabled:false must override stale parse_method, got text %v", res.JSON[0].ToMap()["text"])
}
})
}
@@ -1065,7 +1066,7 @@ func TestMaybeDispatchImageBlankVisionDescriptionWarns(t *testing.T) {
if err != nil || !handled {
t.Fatalf("err = %v, handled = %v", err, handled)
}
- if text, _ := res.JSON[0]["text"].(string); strings.TrimSpace(text) != "" {
+ if text, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(text) != "" {
t.Fatalf("text = %q, want empty", text)
}
if !strings.Contains(strings.Join(res.Warnings, "\n"), "the Tokenizer will discard this item") {
diff --git a/internal/ingestion/component/parser.go b/internal/ingestion/component/parser.go
index 579873cc79..88c8fd0307 100644
--- a/internal/ingestion/component/parser.go
+++ b/internal/ingestion/component/parser.go
@@ -591,7 +591,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
// forwards only this explicit output to the next node, so shared
// fields must live in Globals.
globals.PublishGlobals(ctx, out)
- items, _ := out["json"].([]map[string]any)
+ items, _ := out["json"].([]parser.DocItem)
logParserOutput(dispatched, items)
// Progress (_created_time / _elapsed_time stamping, start/done
// callbacks) is owned by the canvas framework (realComponentBody),
@@ -599,7 +599,7 @@ func (c *ParserComponent) Invoke(ctx context.Context, db *gorm.DB, inputs map[st
return out, nil
}
-func logParserOutput(dispatched parser.ParseResult, items []map[string]any) {
+func logParserOutput(dispatched parser.ParseResult, items []parser.DocItem) {
common.Debug("parser stage output",
zap.String("component", "Parser"),
zap.String("normalized_from", resolveParserNormalizationSource(dispatched)),
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index 1d082e6818..c9fcc66968 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -320,6 +320,10 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name
}
if dispatched.Err == nil && dispatched.OutputFormat != "" {
out["output_format"] = "json"
+ // Emit the parser's typed []DocItem directly. The chunker consumes
+ // it via schema.ChunkDocsFromDocItem (no ToMap round-trip); other
+ // consumers that still need the map form convert at their own
+ // boundary via parser.ItemsToMaps.
out["json"] = normalizeParserJSON(ctx, name, dispatched)
if dispatched.File != nil {
out["file"] = dispatched.File
@@ -331,7 +335,7 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name
if len(rawPages) == 0 {
rawPages = [][]byte{nil}
}
- fallbackItems := make([]map[string]any, 0, len(rawPages))
+ fallbackItems := make([]parser.DocItem, 0, len(rawPages))
for _, pageBytes := range rawPages {
txt := string(pageBytes)
fallbackItems = append(fallbackItems, parser.NewTextJSONItem(txt))
@@ -341,7 +345,11 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name
return out
}
-func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []map[string]any {
+// normalizeParserJSON returns the parser output as the typed []DocItem. It no
+// longer round-trips through map[string]any (the historical ItemsToMaps call),
+// so the downstream chunker receives structured items and converts them via
+// schema.ChunkDocsFromDocItem without ever calling DocItem.ToMap.
+func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []parser.DocItem {
if len(dispatched.JSON) > 0 {
return dispatched.JSON
}
@@ -349,9 +357,9 @@ func normalizeParserJSON(ctx context.Context, filename string, dispatched parser
return parseMarkdownToJSONItems(ctx, filename, dispatched.Markdown)
}
if dispatched.Text != "" {
- return []map[string]any{parser.NewTextJSONItem(dispatched.Text)}
+ return []parser.DocItem{parser.NewTextJSONItem(dispatched.Text)}
}
- return []map[string]any{}
+ return []parser.DocItem{}
}
func warnParserNormalizationFallback(filename, source string, err error) {
diff --git a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
index d6bf465d11..90697b8fb4 100644
--- a/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
+++ b/internal/ingestion/component/parser_dispatch_pdf_vision_cgo_test.go
@@ -11,6 +11,7 @@ import (
"testing"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/parser/parser"
"gorm.io/gorm"
)
@@ -87,7 +88,8 @@ func TestDispatch_PDFVisionJSON_RealPDFFixture(t *testing.T) {
t.Fatalf("vision call count = %d, want %d", got, pageCount)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) == 0 {
t.Fatalf("json payload missing or empty: %T", out["json"])
}
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 470b5a5384..9b7adc63d4 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -82,7 +82,8 @@ func requireJSONText(t *testing.T, out map[string]any, want string) {
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
- items, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
@@ -124,7 +125,8 @@ func TestBuildParserOutputsNormalizesTextFormatsToJSON(t *testing.T) {
requireJSONText(t, out, tt.wantText)
if tt.name == "markdown" {
- items, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json = %T, want []map[string]any", out["json"])
}
@@ -169,7 +171,7 @@ func TestNormalizeParserJSON_EmptyPayloadYieldsEmpty(t *testing.T) {
func TestBuildParserOutputsFallsBackWhenJSONIsEmpty(t *testing.T) {
out := buildParserOutputs(t.Context(), parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{},
+ JSON: parser.MapsToItems([]map[string]any{}),
Markdown: "# Recovered title",
}, "sample.md", nil, "")
@@ -184,7 +186,7 @@ func TestBuildParserOutputsPreservesPDFOutlineMetadata(t *testing.T) {
"name": "report.pdf",
"outline": wantOutline,
},
- JSON: []map[string]any{{"text": "body", "doc_type_kwd": "text"}},
+ JSON: parser.MapsToItems([]map[string]any{{"text": "body", "doc_type_kwd": "text"}}),
}, "report.pdf", nil, "")
file, ok := out["file"].(map[string]any)
if !ok {
@@ -235,7 +237,7 @@ func TestLogParserOutputReportsNormalizationSource(t *testing.T) {
logParserOutput(parser.ParseResult{
OutputFormat: "markdown",
Markdown: "# Title\n\nBody",
- }, []map[string]any{{"text": "Title"}, {"text": "Body"}})
+ }, []parser.DocItem{parser.NewTextJSONItem("Title"), parser.NewTextJSONItem("Body")})
entries := logs.FilterMessage("parser stage output").All()
if len(entries) != 1 {
@@ -292,7 +294,8 @@ func TestDispatch_JSONOutput(t *testing.T) {
if got, want := out["output_format"], "json"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json payload missing or wrong type: %T", out["json"])
}
@@ -321,7 +324,9 @@ func TestDispatchNormalizesConfiguredOutputFormatToJSON(t *testing.T) {
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
- if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
+ if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
@@ -346,7 +351,8 @@ func TestDispatch_TextPageMode_NoFileType(t *testing.T) {
if got, want := out["output_format"], "json"; got != want {
t.Errorf("output_format = %v, want %v (raw-text mode)", got, want)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) == 0 {
t.Fatalf("json items missing or empty: %T", out["json"])
}
@@ -478,7 +484,9 @@ func TestDispatch_PDFLegacyMarkdownConfigurationEmitsJSON(t *testing.T) {
if got := out["output_format"]; got != "json" {
t.Fatalf("output_format = %v, want json", got)
}
- if items, ok := out["json"].([]map[string]any); !ok || len(items) == 0 {
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
+ if !ok || len(items) == 0 {
t.Fatalf("json = %T/%v, want non-empty structured items", out["json"], out["json"])
}
}
@@ -510,7 +518,8 @@ func TestDispatch_PDFPlainText_UsesConfiguredBackend(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) == 0 {
t.Fatalf("json payload missing or empty: %T", out["json"])
}
@@ -603,7 +612,8 @@ func TestDispatch_PDFVisionJSON_UsesTenantAwareModel(t *testing.T) {
"tenant_id": "tenant-1",
})
if err == nil {
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 2 {
t.Fatalf("json payload = %#v, want 2 items", out["json"])
}
@@ -670,7 +680,8 @@ func TestDispatch_PDFVisionJSON_PreservesEmptyPages(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 2 {
t.Fatalf("json payload = %#v, want 2 items", out["json"])
}
@@ -727,7 +738,8 @@ func TestDispatch_PDFVisionJSON_HonorsPageRanges(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json payload = %T, want []map[string]any", out["json"])
}
@@ -862,7 +874,8 @@ func TestDispatch_PDFMinerUJSON_ParsesMarkdownToStructuredItems(t *testing.T) {
if got, want := out["output_format"], "json"; got != want {
t.Fatalf("output_format = %v, want %v", got, want)
}
- items, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if !ok || len(items) == 0 {
t.Fatalf("json payload = %#v, want non-empty structured items", out["json"])
}
diff --git a/internal/ingestion/component/parser_test.go b/internal/ingestion/component/parser_test.go
index 75b0cbdd5d..fb7065d7ac 100644
--- a/internal/ingestion/component/parser_test.go
+++ b/internal/ingestion/component/parser_test.go
@@ -24,6 +24,7 @@ import (
"github.com/xuri/excelize/v2"
"ragflow/internal/agent/runtime"
"ragflow/internal/entity"
+ "ragflow/internal/parser/parser"
)
func TestReportParserWarningsAsProgressMessages(t *testing.T) {
@@ -266,7 +267,8 @@ func TestParserComponent_Invoke_TextInput(t *testing.T) {
if got, want := out["output_format"], "json"; got != want {
t.Errorf("output_format = %v, want %v", got, want)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 1 {
t.Fatalf("json: got %T (len %d), want 1 item", out["json"], len(jsonItems))
}
@@ -297,7 +299,8 @@ func TestParserComponent_EmptyXLSXDoesNotBecomeRawText(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- items, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json = %T, want []map[string]any", out["json"])
}
@@ -317,7 +320,8 @@ func TestParserComponent_Invoke_PageRangeFilter(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json: got %T, want []map[string]any", out["json"])
}
@@ -491,7 +495,8 @@ func TestParserComponent_Invoke_ResolvesBinaryFromDocID(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 2 {
t.Fatalf("json = %T/%v, want 2 items", out["json"], out["json"])
}
@@ -518,7 +523,8 @@ func TestParserComponent_Invoke_ResolvesBinaryFromBucketPath(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(jsonItems) != 1 {
t.Fatalf("json = %T/%v, want 1 item", out["json"], out["json"])
}
@@ -557,7 +563,8 @@ func TestParserComponent_Invoke_AcceptsBytes(t *testing.T) {
if err != nil {
t.Fatalf("Invoke: %v", err)
}
- jsonItems, ok := out["json"].([]map[string]any)
+ jsonDocs, ok := out["json"].([]parser.DocItem)
+ jsonItems := parser.ItemsToMaps(jsonDocs)
if !ok {
t.Fatalf("json: got %T", out["json"])
}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index bad7e288cd..f11cd1b5a3 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -246,11 +246,11 @@ func dispatchMonkeyOCRv2PDF(ctx context.Context, db *gorm.DB, filename string, b
return buildMarkdownOCRDispatchResult(md), nil
}
-func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []map[string]any {
+func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []parser.DocItem {
mp, err := parser.NewMarkdownParser(parser.GoMarkdown)
if err != nil {
warnParserNormalizationFallback(filename, "markdown", err)
- return []map[string]any{parser.NewTextJSONItem(mdText)}
+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}
}
// OCR Markdown is an untrusted backend response. Keep normalization
// deterministic and local: embedded data URIs remain available, while
@@ -259,11 +259,11 @@ func parseMarkdownToJSONItems(ctx context.Context, filename, mdText string) []ma
res := mp.ParseWithResult(ctx, filename, []byte(mdText))
if res.Err != nil {
warnParserNormalizationFallback(filename, "markdown", res.Err)
- return []map[string]any{parser.NewTextJSONItem(mdText)}
+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}
}
if len(res.JSON) == 0 {
warnParserNormalizationFallback(filename, "markdown", fmt.Errorf("parser returned no JSON items"))
- return []map[string]any{parser.NewTextJSONItem(mdText)}
+ return []parser.DocItem{parser.NewTextJSONItem(mdText)}
}
return res.JSON
}
@@ -1050,7 +1050,7 @@ func mineruJSONParseResult(filename string, items []map[string]any) parser.Parse
"outline": []map[string]any{},
"parse_method": "mineru",
},
- JSON: items,
+ JSON: parser.MapsToItems(items),
}
}
@@ -1148,7 +1148,7 @@ func dispatchPDFVision(
return parser.ParseResult{
OutputFormat: "json",
File: fileMeta,
- JSON: items,
+ JSON: parser.MapsToItems(items),
}, nil
}
diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.go
new file mode 100644
index 0000000000..42d8b5e27a
--- /dev/null
+++ b/internal/ingestion/component/schema/chunk_docitem_test.go
@@ -0,0 +1,229 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+ "reflect"
+ "testing"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/parser/parser"
+)
+
+// ChunkDocsFromDocItem must produce the same []ChunkDoc as the historical
+// map path. The historical contract is ChunkDocsFromAny over the []map form
+// the parser produced before emitting DocItems directly: ItemsToMaps
+// projects each DocItem via its canonical ToMap, then ChunkDocFromMap decodes
+// it. ChunkDocsFromDocItem must converge to that exact result, otherwise
+// consuming DocItem directly would silently change chunking behavior.
+//
+// The baseline is deliberately the map path (NOT ChunkDocsFromAny(items),
+// which dispatches []DocItem straight back to ChunkDocsFromDocItem and would
+// make the comparison tautological — the bug this test guards against).
+func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {
+ cases := []struct {
+ name string
+ items []parser.DocItem
+ }{
+ {
+ name: "text with extra passthrough",
+ items: []parser.DocItem{
+ parser.TextItem{Text: "hello", CKType: "heading", Extra: map[string]any{"layout": "1", "page_number": 3}},
+ },
+ },
+ {
+ name: "image with extra passthrough",
+ items: []parser.DocItem{
+ parser.ImageItem{Text: "alt", Image: "data:image/png;base64,xxx", CKType: "image", Extra: map[string]any{"page_number": 2}},
+ },
+ },
+ {
+ name: "table with sheet and positions",
+ items: []parser.DocItem{
+ parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: [][]float64{{1, 2}}, Extra: map[string]any{"sheet_index": 0}},
+ },
+ },
+ {
+ name: "raw item passthrough",
+ items: []parser.DocItem{
+ parser.RawItem{M: map[string]any{"text": "x", "doc_type_kwd": "text", "page_number": 5}},
+ },
+ },
+ {
+ name: "mixed typed and raw",
+ items: []parser.DocItem{
+ parser.TextItem{Text: "a"},
+ parser.RawItem{M: map[string]any{"text": "b", "doc_type_kwd": "text"}},
+ parser.ImageItem{Image: "i", CKType: "image"},
+ },
+ },
+ // A canonical key present in Extra must NOT shadow the typed field when
+ // the typed field is empty. ToMap keeps the empty canonical field and
+ // blocks Extra via mergeExtra; the typed conversion must match exactly.
+ // Regression guard for the divergent docItemWireMapFromFields builder,
+ // which omitted empty canonical keys and let Extra override them.
+ {
+ name: "empty text must not be shadowed by extra text",
+ items: []parser.DocItem{
+ parser.TextItem{Text: "", Extra: map[string]any{"text": "stale"}},
+ },
+ },
+ {
+ name: "empty image text must not be shadowed by extra text",
+ items: []parser.DocItem{
+ parser.ImageItem{Text: "", Image: "i", CKType: "image", Extra: map[string]any{"text": "stale"}},
+ },
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, found, err := ChunkDocsFromDocItem(tc.items)
+ if err != nil {
+ t.Fatalf("ChunkDocsFromDocItem err: %v", err)
+ }
+ if !found {
+ t.Fatalf("expected found=true")
+ }
+ // Historical map-path baseline: ItemsToMaps -> ChunkDocsFromAny.
+ maps := parser.ItemsToMaps(tc.items)
+ want, wfound, werr := ChunkDocsFromAny(maps)
+ if werr != nil || !wfound {
+ t.Fatalf("map-path baseline unexpected: err=%v found=%v", werr, wfound)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("ChunkDocsFromDocItem diverges from map path\n got=%#v\nwant=%#v", got, want)
+ }
+ })
+ }
+}
+
+// The dispatch boundary in ChunkDocsFromAny must route []parser.DocItem to
+// ChunkDocsFromDocItem and report found=true, while any non-slice/unknown
+// shape falls through to the default case and reports found=false. This is
+// the only place a "non-DocItem" input can be rejected: ChunkDocsFromDocItem
+// itself is statically typed to []parser.DocItem, so the guard lives here.
+func TestChunkDocsFromAny_DispatchRejectsNonDocItem(t *testing.T) {
+ if _, found, _ := ChunkDocsFromAny([]parser.DocItem{parser.TextItem{Text: "a"}}); !found {
+ t.Fatalf("expected found=true for []parser.DocItem (routed to ChunkDocsFromDocItem)")
+ }
+ if _, found, _ := ChunkDocsFromAny("not a slice"); found {
+ t.Fatalf("expected found=false for non-slice input")
+ }
+ // []map[string]any keeps its own map path (found=true) — it is NOT a
+ // DocItem and must not be silently routed to ChunkDocsFromDocItem.
+ if _, found, _ := ChunkDocsFromAny([]map[string]any{{"text": "x"}}); !found {
+ t.Fatalf("expected found=true for []map[string]any via map path")
+ }
+}
+
+// Empty slice is a valid DocItem input: found=true, no docs.
+func TestChunkDocsFromDocItem_EmptySlice(t *testing.T) {
+ got, found, err := ChunkDocsFromDocItem([]parser.DocItem{})
+ if err != nil {
+ t.Fatalf("err: %v", err)
+ }
+ if !found {
+ t.Fatalf("expected found=true for empty slice")
+ }
+ if len(got) != 0 {
+ t.Fatalf("expected empty result, got %d", len(got))
+ }
+}
+
+// A nil element inside the slice must be skipped (never dereferenced),
+// matching ItemsToMaps behavior. Regression guard against a nil-interface
+// method call panic in the DocItem conversion path.
+func TestChunkDocsFromDocItem_SkipsNilElement(t *testing.T) {
+ items := []parser.DocItem{
+ parser.TextItem{Text: "a"},
+ nil,
+ parser.ImageItem{Image: "i", CKType: "image"},
+ }
+ got, found, err := ChunkDocsFromDocItem(items)
+ if err != nil {
+ t.Fatalf("err: %v", err)
+ }
+ if !found {
+ t.Fatalf("expected found=true")
+ }
+ if len(got) != 2 {
+ t.Fatalf("expected 2 docs after skipping nil, got %d", len(got))
+ }
+ if got[0].Text != "a" || got[1].Image != "i" {
+ t.Fatalf("unexpected docs after nil skip: %#v", got)
+ }
+}
+
+// Producers emit []map[string]any and wrap with parser.MapsToItems, which now
+// detects doc_type_kwd and builds typed DocItems. The resulting []ChunkDoc the
+// chunker sees must be identical to the historical path that decoded the raw
+// []map directly. This pins the producer-side typed emission to the exact
+// wire contract, so the migration cannot change chunking behavior.
+func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {
+ cases := []struct {
+ name string
+ maps []map[string]any
+ }{
+ {
+ name: "text with extra passthrough",
+ maps: []map[string]any{
+ {"text": "hello", parser.DocTypeKey: parser.DocTypeText, "ck_type": "heading", "layout": "1", "page_number": 3},
+ },
+ },
+ {
+ name: "image with extra passthrough",
+ maps: []map[string]any{
+ {"text": "alt", "image": "data:image/png;base64,xxx", parser.DocTypeKey: parser.DocTypeImage, "ck_type": "image", "page_number": 2},
+ },
+ },
+ {
+ name: "table with sheet and positions",
+ maps: []map[string]any{
+ {"table": &entity.TableData{}, parser.DocTypeKey: parser.DocTypeTable, "ck_type": parser.DocTypeTable, "sheet": "s", "positions": [][]float64{{1, 2}}, "sheet_index": 0},
+ },
+ },
+ {
+ name: "opaque map without doc_type_kwd stays raw",
+ maps: []map[string]any{
+ {"foo": "bar", "nested": map[string]any{"a": 1}},
+ },
+ },
+ {
+ name: "mixed typed and opaque",
+ maps: []map[string]any{
+ {"text": "a", parser.DocTypeKey: parser.DocTypeText, "ck_type": "text"},
+ {"foo": "bar"},
+ },
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, found, err := ChunkDocsFromDocItem(parser.DocItemsFromMaps(tc.maps))
+ if err != nil {
+ t.Fatalf("ChunkDocsFromDocItem err: %v", err)
+ }
+ if !found {
+ t.Fatalf("expected found=true")
+ }
+ want, wfound, werr := ChunkDocsFromAny(tc.maps)
+ if werr != nil || !wfound {
+ t.Fatalf("map-path baseline unexpected: err=%v found=%v", werr, wfound)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("DocItemsFromMaps path diverges from map path\n got=%#v\nwant=%#v", got, want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index 5aa15ed2df..8ca5ae09f7 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -20,6 +20,7 @@ import (
"encoding/json"
"ragflow/internal/entity"
+ "ragflow/internal/parser/parser"
)
// PayloadFormat is the discriminator shared by parser/chunker/tokenizer
@@ -233,11 +234,50 @@ func ChunkDocsFromAny(in any) ([]ChunkDoc, bool, error) {
out = append(out, doc)
}
return out, true, nil
+ case []parser.DocItem:
+ return ChunkDocsFromDocItem(t)
default:
return nil, false, nil
}
}
+// ChunkDocsFromDocItem converts a parser-produced []DocItem directly into a
+// typed chunk slice, without routing through DocItem.ToMap. It reads the
+// typed fields of each item and feeds the canonical wire map to ChunkDocFromMap,
+// so the resulting []ChunkDoc is identical to what the historical
+// ChunkDocsFromAny([]DocItem) path produced (which round-tripped via ToMap).
+// This is the consumer-side entry the chunker uses once parsers emit DocItems
+// directly; other callers that still pass []map[string]any / []any / []ChunkDoc
+// keep using ChunkDocsFromAny.
+func ChunkDocsFromDocItem(items []parser.DocItem) ([]ChunkDoc, bool, error) {
+ if items == nil {
+ return nil, false, nil
+ }
+ out := make([]ChunkDoc, 0, len(items))
+ for _, it := range items {
+ if it == nil {
+ continue
+ }
+ doc, err := chunkDocFromDocItem(it)
+ if err != nil {
+ return nil, true, err
+ }
+ out = append(out, doc)
+ }
+ return out, true, nil
+}
+
+// chunkDocFromDocItem converts a parser DocItem directly into a typed chunk by
+// projecting it through its canonical ToMap wire shape and decoding that via
+// ChunkDocFromMap. Using ToMap — the parser package's single canonical
+// projection, with mergeExtra semantics — instead of a second, divergent wire
+// map builder keeps the conversion identical to the historical []map path and
+// prevents an Extra key from shadowing an empty canonical field (e.g. a stray
+// Extra["text"] must never override TextItem.Text == "").
+func chunkDocFromDocItem(it parser.DocItem) (ChunkDoc, error) {
+ return ChunkDocFromMap(it.ToMap())
+}
+
// ToMap converts a typed chunk back to the generic map form used by
// runtime.Component boundaries.
func (d ChunkDoc) ToMap() map[string]any {
diff --git a/internal/ingestion/component/vision_config_test.go b/internal/ingestion/component/vision_config_test.go
index 7eacd066c9..505d3110b2 100644
--- a/internal/ingestion/component/vision_config_test.go
+++ b/internal/ingestion/component/vision_config_test.go
@@ -10,6 +10,7 @@ import (
deepdoctype "ragflow/internal/deepdoc/parser/type"
"ragflow/internal/entity"
modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/parser/parser"
)
func TestParserGlobalVisionModel(t *testing.T) {
@@ -69,7 +70,8 @@ func TestParserGlobalVisionModel(t *testing.T) {
t.Fatalf("resolved %v, want [%s]", resolved, tc.want)
}
if fileType == "image" {
- items := result["json"].([]map[string]any)
+ jsonDocs := result["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if !strings.Contains(items[0]["text"].(string), "OCR text") {
t.Fatalf("OCR text lost: %v", items)
}
@@ -124,7 +126,8 @@ func TestParserGlobalVisionDoesNotOverrideASR(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- items := result["json"].([]map[string]any)
+ jsonDocs := result["json"].([]parser.DocItem)
+ items := parser.ItemsToMaps(jsonDocs)
if calls != 1 || len(items) != 1 || items[0]["text"] != "transcribed" {
t.Fatalf("ASR calls = %d, items = %v", calls, items)
}
diff --git a/internal/ingestion/component/vision_enhancement.go b/internal/ingestion/component/vision_enhancement.go
index 1d2c0602bb..3d37ce56c3 100644
--- a/internal/ingestion/component/vision_enhancement.go
+++ b/internal/ingestion/component/vision_enhancement.go
@@ -153,14 +153,15 @@ func maybeDispatchVisionEnhancement(
// Collect visual resources, including Markdown images whose type is text
// because flatten_media_to_text is enabled.
var items []int
- for i, item := range dispatched.JSON {
- if img, _ := item["image"].(string); img != "" {
+ for i, rawItem := range dispatched.JSON {
+ m := rawItem.ToMap()
+ if img, _ := m["image"].(string); img != "" {
items = append(items, i)
continue
}
- kd, _ := item["doc_type_kwd"].(string)
+ kd, _ := m["doc_type_kwd"].(string)
if kd == "image" || kd == "table" {
- if _, ok := parser.ExtractPDFPositions(item); ok {
+ if _, ok := parser.ExtractPDFPositions(m); ok {
items = append(items, i)
}
}
@@ -246,7 +247,7 @@ func maybeDispatchVisionEnhancement(
}
var resource *visionImage
func() {
- item := dispatched.JSON[itemIdx]
+ item := dispatched.JSON[itemIdx].ToMap()
if err := cropper.Prepare(ctx, item); err != nil {
if ctx.Err() == nil {
if payload, _ := item["image"].(string); isUsableVisionImage(payload) {
@@ -380,7 +381,9 @@ func maybeDispatchVisionEnhancement(
if desc == "" {
continue
}
- appendItemText(dispatched.JSON[itemIdx], desc)
+ m := dispatched.JSON[itemIdx].ToMap()
+ appendItemText(m, desc)
+ dispatched.JSON[itemIdx] = parser.DocItemFromMap(m)
modified = true
}
diff --git a/internal/ingestion/component/vision_enhancement_progress_test.go b/internal/ingestion/component/vision_enhancement_progress_test.go
index 124623b267..efaaeed9f1 100644
--- a/internal/ingestion/component/vision_enhancement_progress_test.go
+++ b/internal/ingestion/component/vision_enhancement_progress_test.go
@@ -24,12 +24,12 @@ func TestVisionEnhancement_ReportsPerFigureFractions(t *testing.T) {
png := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": png, "doc_type_kwd": "image"},
{"text": "2", "image": png, "doc_type_kwd": "image"},
{"text": "3", "image": png, "doc_type_kwd": "image"},
{"text": "4", "image": png, "doc_type_kwd": "image"},
- },
+ }),
}
win := progress.Window{Base: 0.5, Span: 0.5}
@@ -79,10 +79,10 @@ func TestVisionEnhancement_SettledFractionsCountFailedFigures(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "valid", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},
{"text": "invalid", "image": "not-valid-base64", "doc_type_kwd": "image"},
- },
+ }),
}
win := progress.Window{Base: 0.2, Span: 0.8}
@@ -126,9 +126,9 @@ func TestVisionEnhancement_ZeroWindowReportsNothing(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"},
- },
+ }),
}
// Zero window (default when unpassed or inactive)
@@ -161,10 +161,10 @@ func TestVisionEnhancement_CancelledContextReportsNothing(t *testing.T) {
png := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "1", "image": png, "doc_type_kwd": "image"},
{"text": "2", "image": png, "doc_type_kwd": "image"},
- },
+ }),
}
win := progress.Window{Base: 0.5, Span: 0.5}
diff --git a/internal/ingestion/component/vision_enhancement_test.go b/internal/ingestion/component/vision_enhancement_test.go
index 191b48b5f8..367e19bdba 100644
--- a/internal/ingestion/component/vision_enhancement_test.go
+++ b/internal/ingestion/component/vision_enhancement_test.go
@@ -186,7 +186,8 @@ func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T)
if resolverCalls != 0 || len(invoker.images) != 0 {
t.Fatalf("default-off enhancement used model: %d resolves, %d VLM calls", resolverCalls, len(invoker.images))
}
- withoutItems, ok := withoutVision["json"].([]map[string]any)
+ jsonDocs, ok := withoutVision["json"].([]parser.DocItem)
+ withoutItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(withoutItems) == 0 {
t.Fatalf("default JSON = %T/%v", withoutVision["json"], withoutVision["json"])
}
@@ -202,7 +203,8 @@ func TestParserComponent_VisionEnhancementSwitchGatesEmbeddedImage(t *testing.T)
if resolverCalls != 1 || len(invoker.images) != 1 {
t.Fatalf("enabled enhancement used model %d/%d times, want 1/1", resolverCalls, len(invoker.images))
}
- withItems, ok := withVision["json"].([]map[string]any)
+ jsonDocs, ok = withVision["json"].([]parser.DocItem)
+ withItems := parser.ItemsToMaps(jsonDocs)
if !ok || len(withItems) == 0 {
t.Fatalf("enabled JSON = %T/%v", withVision["json"], withVision["json"])
}
@@ -219,11 +221,11 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
+ JSON: parser.MapsToItems([]map[string]any{{
"text": "Existing caption",
"image": visionTestPNGBase64(t),
"doc_type_kwd": "image",
- }},
+ }}),
}
res, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypeXLSX, dispatched,
@@ -236,7 +238,7 @@ func TestVisionEnhancement_AppendsVLMToExistingText(t *testing.T) {
t.Fatal("handled = false, want true")
}
want := "Existing caption\na diagram of a pipeline"
- if got := res.JSON[0]["text"]; got != want {
+ if got := res.JSON[0].ToMap()["text"]; got != want {
t.Errorf("enhanced text = %q, want %q", got, want)
}
if len(invoker.images) != 1 {
@@ -257,10 +259,10 @@ func TestVisionEnhancementPreparesPDFSourceBeforeImageAdmission(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
+ JSON: parser.MapsToItems([]map[string]any{{
"image": cropper.payload,
"doc_type_kwd": "image",
- }},
+ }}),
}
if _, _, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
@@ -280,10 +282,10 @@ func TestVisionEnhancement_ReportsVLMInvocationFailure(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
+ JSON: parser.MapsToItems([]map[string]any{{
"image": visionTestPNGBase64(t),
"doc_type_kwd": "image",
- }},
+ }}),
}
result, _, err := maybeDispatchVisionEnhancement(t.Context(), dao.DB, utility.FileTypeDOCX, dispatched,
map[string]any{"tenant_id": "t1"},
@@ -308,10 +310,10 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
+ JSON: parser.MapsToItems([]map[string]any{{
"image": visionTestPNGBase64(t),
"doc_type_kwd": "image",
- }},
+ }}),
}
result, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
@@ -328,7 +330,7 @@ func TestVisionEnhancement_CropFailureFallsBackToInlineImage(t *testing.T) {
if !strings.HasPrefix(invoker.images[0], "data:image/png;base64,") {
t.Errorf("VLM image URL = %q, want inline PNG data URI", invoker.images[0])
}
- if got, want := result.JSON[0]["text"], "a diagram of a pipeline"; got != want {
+ if got, want := result.JSON[0].ToMap()["text"], "a diagram of a pipeline"; got != want {
t.Errorf("enhanced text = %q, want %q", got, want)
}
}
@@ -341,11 +343,11 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{
+ JSON: parser.MapsToItems([]map[string]any{{
"text": "Existing table",
"image": visionTestPNGBase64(t),
"doc_type_kwd": "table",
- }},
+ }}),
}
res, handled, err := maybeDispatchVisionEnhancement(
t.Context(), dao.DB, utility.FileTypePDF, dispatched,
@@ -357,7 +359,7 @@ func TestVisionEnhancement_EnhancesPDFTablesRegardlessOfOCRProvider(t *testing.T
if !handled {
t.Fatal("handled = false, want VLM enhancement")
}
- if got, want := res.JSON[0]["text"], "Existing table\na diagram of a pipeline"; got != want {
+ if got, want := res.JSON[0].ToMap()["text"], "Existing table\na diagram of a pipeline"; got != want {
t.Errorf("enhanced text = %q, want %q", got, want)
}
})
@@ -370,7 +372,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
imagePayload := visionTestPNGBase64(t)
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "<table><tr><td>figure</td></tr></table>", "image": imagePayload, "doc_type_kwd": "table"},
{
"text": "cell alt",
@@ -381,7 +383,7 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
"column_index": 1,
"media_order": 1,
},
- },
+ }),
}
result, handled, err := maybeDispatchVisionEnhancement(
@@ -393,11 +395,11 @@ func TestVisionEnhancement_EnhancesTableAndCellImages(t *testing.T) {
if !handled {
t.Fatal("handled = false, want table and cell image VLM processing")
}
- if got, want := result.JSON[0]["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {
+ if got, want := result.JSON[0].ToMap()["text"], "<table><tr><td>figure</td></tr></table>\na diagram of a pipeline"; got != want {
t.Errorf("table text = %q, want %q", got, want)
}
cellWant := "cell alt\na diagram of a pipeline"
- if got := result.JSON[1]["text"]; got != cellWant {
+ if got := result.JSON[1].ToMap()["text"]; got != cellWant {
t.Errorf("cell image text = %q, want %q", got, cellWant)
}
if len(invoker.images) != 2 {
@@ -410,7 +412,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}},
+ JSON: parser.MapsToItems([]map[string]any{{"text": "existing", "image": visionTestPNGBase64(t), "doc_type_kwd": "image"}}),
}
result, handled, err := maybeDispatchVisionEnhancement(
@@ -422,7 +424,7 @@ func TestVisionEnhancement_EnhancesExternalPDFImage(t *testing.T) {
if !handled {
t.Fatal("handled = false, want VLM enhancement")
}
- if got, want := result.JSON[0]["text"], "existing\na diagram of a pipeline"; got != want {
+ if got, want := result.JSON[0].ToMap()["text"], "existing\na diagram of a pipeline"; got != want {
t.Errorf("image text = %q, want %q", got, want)
}
if len(invoker.images) != 1 {
@@ -452,12 +454,12 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "Intro paragraph", "image": nil, "doc_type_kwd": "text"},
{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
{"text": "existing table", "image": "dGFibGUtaW1hZ2U=", "doc_type_kwd": "table"},
{"text": "<table></table>", "image": nil, "doc_type_kwd": "table"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -476,18 +478,18 @@ func TestVisionEnhancement_EnhancesJSONImagesAndTables(t *testing.T) {
if len(res.JSON) != 4 {
t.Fatalf("JSON len = %d, want 4", len(res.JSON))
}
- if got := res.JSON[0]["text"].(string); got != "Intro paragraph" {
+ if got := res.JSON[0].ToMap()["text"].(string); got != "Intro paragraph" {
t.Errorf("text item text = %q, want unchanged", got)
}
// image item: VLM description cleaned of ```markdown and appended
- if got, _ := res.JSON[1]["text"].(string); got != "a diagram of a pipeline" {
+ if got, _ := res.JSON[1].ToMap()["text"].(string); got != "a diagram of a pipeline" {
t.Errorf("image item text = %q, want 'a diagram of a pipeline'", got)
}
// table item with image: VLM description appended with single \n
- if got, _ := res.JSON[2]["text"].(string); got != "existing table\na diagram of a pipeline" {
+ if got, _ := res.JSON[2].ToMap()["text"].(string); got != "existing table\na diagram of a pipeline" {
t.Errorf("table item text = %q, want 'existing table\\na diagram of a pipeline'", got)
}
- if got, _ := res.JSON[3]["text"].(string); got != "<table></table>" {
+ if got, _ := res.JSON[3].ToMap()["text"].(string); got != "<table></table>" {
t.Errorf("table item without image text = %q, want unchanged", got)
}
if capturedLanguage != "Japanese" {
@@ -539,9 +541,9 @@ func TestVisionEnhancement_LanguagePriority(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8taW1hZ2U=", "doc_type_kwd": "image"},
- },
+ }),
}
_, handled, err := maybeDispatchVisionEnhancement(
@@ -607,9 +609,9 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
swapVisionGlobals(t, fakeResolver, invoker.invoke, fakePrompt)
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "caption", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -625,7 +627,7 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
if !handled {
t.Fatal("handled = false, want true for a visual payload in FileTypeOTHER")
}
- if got, want := res.JSON[0]["text"], "caption\na diagram of a pipeline"; got != want {
+ if got, want := res.JSON[0].ToMap()["text"], "caption\na diagram of a pipeline"; got != want {
t.Errorf("text = %q, want %q", got, want)
}
if len(invoker.images) != 1 {
@@ -636,9 +638,9 @@ func TestVisionEnhancement_UsesVisualPayloadRegardlessOfFileType(t *testing.T) {
func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
// No tenant_id
@@ -652,8 +654,8 @@ func TestVisionEnhancement_EmptyOrNoTenantSkipped(t *testing.T) {
if err != nil || handled {
t.Errorf("handled=%v, err=%v, want false, nil for missing tenant_id", handled, err)
}
- if res.JSON[0]["text"] != "" {
- t.Errorf("text = %q, want untouched", res.JSON[0]["text"])
+ if res.JSON[0].ToMap()["text"] != "" {
+ t.Errorf("text = %q, want untouched", res.JSON[0].ToMap()["text"])
}
}
@@ -664,9 +666,9 @@ func TestVisionEnhancement_DispatchedErrSkipped(t *testing.T) {
dispatched := parser.ParseResult{
Err: parseErr,
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -702,9 +704,9 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -721,7 +723,7 @@ func TestVisionEnhancement_ContextCancellation(t *testing.T) {
t.Error("handled = true with cancelled context, want false")
}
// text field should be untouched since invoker returned no answer.
- if got, _ := res.JSON[0]["text"].(string); got != "" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {
t.Errorf("text = %q, want empty (no successful VLM response)", got)
}
}
@@ -741,12 +743,12 @@ func TestVisionEnhancement_NonStringImageFieldFiltered(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
// non-string image field — filtered by target collector
{"text": "para", "image": 12345, "doc_type_kwd": "image"},
// nil image — also filtered
{"text": "para2", "image": nil, "doc_type_kwd": "image"},
- },
+ }),
}
_, handled, err := maybeDispatchVisionEnhancement(
@@ -789,7 +791,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
}
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: items,
+ JSON: parser.MapsToItems(items),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -806,7 +808,7 @@ func TestVisionEnhancement_MoreThanConcurrencyItems(t *testing.T) {
t.Error("handled = false, want true")
}
// All n items should have received a description appended.
- for i, item := range res.JSON {
+ for i, item := range parser.ItemsToMaps(res.JSON) {
got, _ := item["text"].(string)
if got == fmt.Sprintf("item%d", i) {
t.Errorf("item[%d] text unchanged %q, expected description appended", i, got)
@@ -828,9 +830,9 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -846,7 +848,7 @@ func TestVisionEnhancement_PlainTextResponseNotTruncated(t *testing.T) {
if !handled {
t.Error("handled = false, want true")
}
- if got, _ := res.JSON[0]["text"].(string); got != plain {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != plain {
t.Errorf("plain text response = %q, want %q (must not be truncated)", got, plain)
}
}
@@ -867,9 +869,9 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -885,7 +887,7 @@ func TestVisionEnhancement_PromptBuilderErrorSkipped(t *testing.T) {
if handled {
t.Error("handled = true, want false when prompt builder fails")
}
- if got, _ := res.JSON[0]["text"].(string); got != "" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {
t.Errorf("text = %q, want empty", got)
}
}
@@ -906,9 +908,9 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -924,7 +926,7 @@ func TestVisionEnhancement_ModelResolveFailureSkipped(t *testing.T) {
if handled {
t.Error("handled = true, want false when model resolve fails")
}
- if got, _ := res.JSON[0]["text"].(string); got != "" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "" {
t.Errorf("text = %q, want empty", got)
}
}
@@ -957,7 +959,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
}
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: items,
+ JSON: parser.MapsToItems(items),
}
ctx, cancel := context.WithCancel(t.Context())
@@ -977,7 +979,7 @@ func TestVisionEnhancement_CancellationStopsSchedulingWithManyItems(t *testing.T
t.Error("handled = true with cancelled context, want false")
}
// No item should be modified when dispatch is cancelled before scheduling.
- for i, item := range res.JSON {
+ for i, item := range parser.ItemsToMaps(res.JSON) {
if got, _ := item["text"].(string); got != fmt.Sprintf("item%d", i) {
t.Errorf("item[%d] text = %q, want untouched after cancellation", i, got)
}
@@ -1181,9 +1183,9 @@ func TestVisionEnhancement_GlobalModelPreferredOverPDFParser(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "", "image": "aGVsbG8=", "doc_type_kwd": "image"},
- },
+ }),
}
_, handled, err := maybeDispatchVisionEnhancement(
@@ -1223,9 +1225,9 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
dispatched := parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "keep", "image": "!!!not-base64!!!", "doc_type_kwd": "image"},
- },
+ }),
}
res, handled, err := maybeDispatchVisionEnhancement(
@@ -1244,7 +1246,7 @@ func TestVisionEnhancement_InvalidImageDataSkipped(t *testing.T) {
if invokerCalled {
t.Error("invoker must not be called for invalid image data")
}
- if got, _ := res.JSON[0]["text"].(string); got != "keep" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "keep" {
t.Errorf("text = %q, want unchanged %q", got, "keep")
}
}
diff --git a/internal/ingestion/component/vision_source_admission_test.go b/internal/ingestion/component/vision_source_admission_test.go
index 874e9d86b2..23eda6c731 100644
--- a/internal/ingestion/component/vision_source_admission_test.go
+++ b/internal/ingestion/component/vision_source_admission_test.go
@@ -76,7 +76,7 @@ func TestVisionSourceFetchDoesNotBlockImageAdmission(t *testing.T) {
}
}
run := func(inputs map[string]any) []string {
- dispatched := parser.ParseResult{OutputFormat: "json", JSON: []map[string]any{newItem()}}
+ dispatched := parser.ParseResult{OutputFormat: "json", JSON: parser.MapsToItems([]map[string]any{newItem()})}
res, _, err := maybeDispatchVisionEnhancement(
context.Background(), dao.DB, utility.FileTypePDF, dispatched,
inputs, visionSettings{}, progress.Window{},
diff --git a/internal/ingestion/task/indexdoc/normalize.go b/internal/ingestion/task/indexdoc/normalize.go
index cc678cf679..0395d1b553 100644
--- a/internal/ingestion/task/indexdoc/normalize.go
+++ b/internal/ingestion/task/indexdoc/normalize.go
@@ -20,6 +20,7 @@ import (
"fmt"
"ragflow/internal/common"
+ "ragflow/internal/parser/parser"
)
// NormalizeChunks converts pipeline output into a uniform []map[string]any slice.
@@ -38,6 +39,9 @@ func NormalizeChunks(output map[string]any) []map[string]any {
if json, ok := output["json"].([]map[string]any); ok {
return DeepCopyChunks(json)
}
+ if docs, ok := output["json"].([]parser.DocItem); ok {
+ return DeepCopyChunks(parser.ItemsToMaps(docs))
+ }
if json, ok := toChunkMaps(output["json"]); ok {
return DeepCopyChunks(json)
}
diff --git a/internal/ingestion/task/indexdoc/normalize_test.go b/internal/ingestion/task/indexdoc/normalize_test.go
index e27e9ab0cd..8555f2807b 100644
--- a/internal/ingestion/task/indexdoc/normalize_test.go
+++ b/internal/ingestion/task/indexdoc/normalize_test.go
@@ -2,6 +2,8 @@ package indexdoc
import (
"testing"
+
+ "ragflow/internal/parser/parser"
)
// =============================================================================
@@ -39,6 +41,25 @@ func TestNormalizeChunks_JSONFormat(t *testing.T) {
}
}
+// An empty (non-nil) []parser.DocItem under "json" is a present result and
+// must take precedence over markdown, matching the historical
+// []map[string]any empty-slice behavior (which returned no chunks, not the
+// markdown text). Regression guard for the len(docs) > 0 guard that skipped
+// the typed json branch for an empty slice.
+func TestNormalizeChunks_EmptyDocItemSlicePrecedence(t *testing.T) {
+ input := map[string]any{
+ "json": []parser.DocItem{},
+ "markdown": "# heading",
+ }
+ result := NormalizeChunks(input)
+ if result == nil {
+ t.Fatalf("expected empty (non-nil) slice, got nil")
+ }
+ if len(result) != 0 {
+ t.Fatalf("len = %d, want 0 (json present, takes precedence over markdown)", len(result))
+ }
+}
+
func TestNormalizeChunks_JSONFormatFromGenericSlice(t *testing.T) {
input := map[string]any{
"json": []any{
diff --git a/internal/parser/parser/charset_test.go b/internal/parser/parser/charset_test.go
index 1445a2279f..e1b3b06e7a 100644
--- a/internal/parser/parser/charset_test.go
+++ b/internal/parser/parser/charset_test.go
@@ -166,7 +166,7 @@ func TestCSVParser_GBK(t *testing.T) {
t.Fatalf("CSVParser returned no JSON items")
}
var rowText strings.Builder
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text := tableItemHTML(t, item)
rowText.WriteString(text)
}
@@ -217,7 +217,7 @@ func TestEPUBParser_GB2312(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("want non-empty JSON items from EPUB")
}
- text, _ := res.JSON[0]["text"].(string)
+ text, _ := res.JSON[0].ToMap()["text"].(string)
if strings.ContainsRune(text, '\ufffd') {
t.Errorf("EPUB text contains replacement runes: %s", text)
}
diff --git a/internal/parser/parser/csv_parser.go b/internal/parser/parser/csv_parser.go
index 153df9012e..88274d3e75 100644
--- a/internal/parser/parser/csv_parser.go
+++ b/internal/parser/parser/csv_parser.go
@@ -155,7 +155,7 @@ func (p *CSVParser) ParseWithResult(ctx context.Context, filename string, data [
"format": "csv",
"sheets": 1,
},
- JSON: items,
+ JSON: MapsToItems(items),
}
}
diff --git a/internal/parser/parser/csv_parser_test.go b/internal/parser/parser/csv_parser_test.go
index 5f9487875d..24fa1be83f 100644
--- a/internal/parser/parser/csv_parser_test.go
+++ b/internal/parser/parser/csv_parser_test.go
@@ -26,7 +26,7 @@ func TestCSVParser_EmitsSpreadsheetRows(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
- item := res.JSON[0]
+ item := res.JSON[0].ToMap()
if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
t.Fatalf("item types = %v/%v, want table/table", item["ck_type"], item["doc_type_kwd"])
}
@@ -69,7 +69,7 @@ func TestCSVParser_UTF8BOM(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %#v, want one table item", res.JSON)
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<tr><th>Name</th><th>Age</th></tr>") {
t.Errorf("header row = %q", text)
}
@@ -91,7 +91,7 @@ func TestCSVParserHTML4ExcelIsIgnored(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult failed: %v", res.Err)
}
- if len(res.JSON) != 1 || res.JSON[0]["ck_type"] != "table" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["ck_type"] != "table" {
t.Fatalf("items = %#v, want one table item", res.JSON)
}
if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
@@ -117,10 +117,10 @@ func TestCSVParser_ConfigureOutputFormatJSON(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table" {
- t.Fatalf("ck_type = %v, want table", res.JSON[0]["ck_type"])
+ if res.JSON[0].ToMap()["ck_type"] != "table" {
+ t.Fatalf("ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])
}
- if positions := res.JSON[0]["positions"].([][]float64); len(positions) != 2 {
+ if positions := res.JSON[0].ToMap()["positions"].([][]float64); len(positions) != 2 {
t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
}
}
@@ -156,7 +156,7 @@ func TestCSVParser_PreservesEveryDataRow(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("len(res.JSON) = %d, want one HTML table item", len(res.JSON))
}
- positions := res.JSON[0]["positions"].([][]float64)
+ positions := res.JSON[0].ToMap()["positions"].([][]float64)
if len(positions) != 5 {
t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus four rows)", len(positions))
}
@@ -177,12 +177,12 @@ func TestCSVParserKeepsVariableRowWidths(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<tr><td>Q1</td><td>A1</td></tr>") ||
!strings.Contains(text, "<tr><td>Q2</td><td>A2</td><td>extra</td></tr>") {
t.Fatalf("row widths not preserved in markup: %q", text)
}
- positions := res.JSON[0]["positions"].([][]float64)
+ positions := res.JSON[0].ToMap()["positions"].([][]float64)
// Per-row column ranges: Q1 row spans cols 1-2, Q2 row spans 1-3.
if len(positions) != 3 || positions[1][4] != 2 || positions[2][4] != 3 {
t.Fatalf("position matrix lost per-row widths: %v", positions)
@@ -201,7 +201,7 @@ func TestCSVParser_ReadsTheSeparatorTheFileWasWrittenWith(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("separator %q: items = %d, want one HTML table item", sep, len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<tr><th>Name</th><th>Region</th><th>Units</th></tr>") ||
!strings.Contains(text, "<tr><td>Widget</td><td>EU</td><td>12</td></tr>") {
t.Errorf("separator %q: row not split into three columns: %q", sep, text)
@@ -272,7 +272,7 @@ func TestCSVParser_KeepsAnEmptyTabSeparatedField(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<tr><td>Widget</td><td></td><td>12</td></tr>") {
t.Errorf("empty tab-separated field not kept: %q", text)
}
@@ -292,7 +292,7 @@ func TestCSVParser_PreservesInternalJoiners(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item", len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<td>"+emoji+"</td>") {
t.Errorf("emoji ZWJ sequence broken: %q", text)
}
diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.go
new file mode 100644
index 0000000000..3eb9b23d75
--- /dev/null
+++ b/internal/parser/parser/doc_item.go
@@ -0,0 +1,326 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ "encoding/json"
+
+ "ragflow/internal/entity"
+)
+
+// DocItem is one structured piece of a parser's output, replacing the previous
+// loosely-typed `map[string]any` payload. The parser package emits DocItems;
+// the ingestion component and downstream consumers still read the wire form
+// produced by ToMap, so the external JSON shape (key names, pipeline-only
+// keys, the doc_type_kwd discriminator) is unchanged by this type.
+//
+// The design mirrors the existing typed chunk AST (schema.ChunkDoc) but lives
+// in the parser package so the parser layer does not import ingestion. Ad-hoc
+// pipeline keys (layout, page_number, _pdf_positions, sheet_index, media_order,
+// parent_table_id, ...) that are not worth modelling as named fields are passed
+// through verbatim via the Extra map, keeping ToMap a 1:1 projection of the
+// historical map shape. A later pass may promote the well-known Extra keys to
+// named fields; that is out of scope for the boundary-adapter change.
+type DocItem interface {
+ // DocType returns the item's doc_type_kwd discriminator (text/table/image).
+ DocType() string
+ // ToMap projects the item onto the historical wire shape. Callers must not
+ // mutate the returned map's shared Extra contents.
+ ToMap() map[string]any
+}
+
+// TextItem is a body/heading/list/code span. CKType optionally carries the
+// legacy ck_type discriminator (heading/list/code/...); it is omitted from
+// ToMap when empty. Extra carries layout/page_number/level/... passthrough.
+type TextItem struct {
+ Text string
+ CKType string
+ Extra map[string]any
+}
+
+func (t TextItem) DocType() string { return DocTypeText }
+
+func (t TextItem) ToMap() map[string]any {
+ m := map[string]any{
+ "text": t.Text,
+ DocTypeKey: DocTypeText,
+ }
+ if t.CKType != "" {
+ m["ck_type"] = t.CKType
+ }
+ return mergeExtra(m, t.Extra)
+}
+
+// TableItem carries a parsed TableData. CKType is the legacy ck_type
+// discriminator (always "table" today, mirroring NewTableDataJSONItem); it is
+// emitted for pipeline compatibility and will be dropped in a later pass.
+type TableItem struct {
+ Table *entity.TableData
+ Sheet string
+ Positions [][]float64
+ CKType string
+ Extra map[string]any
+}
+
+func (t TableItem) DocType() string { return DocTypeTable }
+
+func (t TableItem) ToMap() map[string]any {
+ m := map[string]any{
+ "table": t.Table,
+ DocTypeKey: DocTypeTable,
+ "ck_type": firstNonEmptyStr(t.CKType, DocTypeTable),
+ }
+ if t.Sheet != "" {
+ m["sheet"] = t.Sheet
+ }
+ if len(t.Positions) > 0 {
+ m["positions"] = t.Positions
+ }
+ return mergeExtra(m, t.Extra)
+}
+
+// ImageItem is an embedded image. Text is the alt text, Image the data
+// URL/base64 payload (possibly empty when omitted). CKType carries the legacy
+// ck_type discriminator ("image" for xlsx/pptx/html/markdown, empty for docx);
+// it is omitted from ToMap when empty.
+type ImageItem struct {
+ Text string
+ Image string
+ Sheet string
+ Positions [][]float64
+ CKType string
+ Extra map[string]any
+}
+
+func (i ImageItem) DocType() string { return DocTypeImage }
+
+func (i ImageItem) ToMap() map[string]any {
+ m := map[string]any{
+ "text": i.Text,
+ "image": i.Image,
+ DocTypeKey: DocTypeImage,
+ }
+ if i.CKType != "" {
+ m["ck_type"] = i.CKType
+ }
+ if i.Sheet != "" {
+ m["sheet"] = i.Sheet
+ }
+ if len(i.Positions) > 0 {
+ m["positions"] = i.Positions
+ }
+ return mergeExtra(m, i.Extra)
+}
+
+// RawItem is a passthrough for items the parser constructs as an opaque map
+// (the email content item, or any parser still emitting map literals during
+// the migration). ToMap returns the map unchanged, so the wire shape is
+// preserved exactly. DocType is empty because the discriminator lives in the
+// map under DocTypeKey.
+type RawItem struct {
+ M map[string]any
+}
+
+func (RawItem) DocType() string { return "" }
+
+func (r RawItem) ToMap() map[string]any { return r.M }
+
+// MarshalJSON projects each item onto its wire shape so that downstream
+// consumers that round-trip the payload through encoding/json (the chunker
+// decodes inputs["json"] via json.Marshal -> json.Unmarshal into []ChunkDoc)
+// observe the historical map shape unchanged.
+func (t TextItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()) }
+func (t TableItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()) }
+func (i ImageItem) MarshalJSON() ([]byte, error) { return json.Marshal(i.ToMap()) }
+func (r RawItem) MarshalJSON() ([]byte, error) { return json.Marshal(r.M) }
+
+// FromMap wraps an opaque payload map as a DocItem. Used by boundary readers
+// that receive a []map[string]any and must hand it to DocItem-typed APIs.
+func FromMap(m map[string]any) DocItem { return RawItem{M: m} }
+
+// Raw wraps an opaque payload map as a DocItem. Parsers that still build items
+// as map literals use it at the assignment site (e.g. `JSON: MapsToItems(items)`
+// or `res.JSON = append(res.JSON, Raw(m))`), so their internal logic stays
+// unchanged while the field gains the DocItem type.
+func Raw(m map[string]any) DocItem { return RawItem{M: m} }
+
+// MapsToItems wraps a slice of payload maps as typed DocItems (one
+// TextItem / TableItem / ImageItem / RawItem each, detected via doc_type_kwd).
+// Parsers that accumulate results as []map[string]any use it at the assignment
+// site (`JSON: MapsToItems(items)`) so their internal logic is unchanged while
+// the field gains the DocItem type and the concrete TextItem/TableItem/
+// ImageItem constructors become the emission path. Opaque maps without a known
+// discriminator stay RawItem, preserving the wire shape exactly.
+func MapsToItems(maps []map[string]any) []DocItem {
+ return DocItemsFromMaps(maps)
+}
+
+// DocItemFromMap detects the item's doc_type_kwd and builds the matching typed
+// DocItem, extracting the canonical wire fields into the struct and leaving
+// every other key in Extra. For an opaque map without a known discriminator it
+// falls back to RawItem, preserving the map exactly. This is the producer-side
+// counterpart to ItemsToMaps: parsers can emit typed DocItems without altering
+// the wire shape downstream consumers (ToMap) rely on.
+func DocItemFromMap(m map[string]any) DocItem {
+ if m == nil {
+ return RawItem{}
+ }
+ switch docType, _ := m[DocTypeKey].(string); docType {
+ case DocTypeText:
+ // Only build a TextItem when the map actually carries a text field.
+ // A text-typed map without a text key (e.g. an email metadata item
+ // whose body is gated by fields) must round-trip through ToMap
+ // unchanged; TextItem.ToMap always writes "text", so forcing it here
+ // would invent a text key the original map lacked.
+ if _, ok := m["text"]; ok {
+ return textItemFromMap(m)
+ }
+ return RawItem{M: m}
+ case DocTypeImage:
+ if _, ok := m["image"]; ok {
+ return imageItemFromMap(m)
+ }
+ return RawItem{M: m}
+ case DocTypeTable:
+ if _, ok := m["table"]; ok {
+ return tableItemFromMap(m)
+ }
+ return RawItem{M: m}
+ default:
+ return RawItem{M: m}
+ }
+}
+
+func textItemFromMap(m map[string]any) DocItem {
+ return TextItem{
+ Text: stringField(m, "text"),
+ CKType: stringField(m, "ck_type"),
+ Extra: extraFromMap(m, "text", DocTypeKey, "ck_type"),
+ }
+}
+
+func imageItemFromMap(m map[string]any) DocItem {
+ return ImageItem{
+ Text: stringField(m, "text"),
+ Image: stringField(m, "image"),
+ CKType: stringField(m, "ck_type"),
+ Sheet: stringField(m, "sheet"),
+ Positions: positionsField(m),
+ // "positions" is intentionally NOT stripped: when the map carries
+ // positions as [][]any (e.g. PDF crop regions from normalizePDFPositions)
+ // the typed field cannot capture them, and stripping would drop the key
+ // from ToMap. Leaving it in Extra preserves the wire shape exactly.
+ Extra: extraFromMap(m, "text", "image", DocTypeKey, "ck_type", "sheet"),
+ }
+}
+
+func tableItemFromMap(m map[string]any) DocItem {
+ td, _ := m["table"].(*entity.TableData)
+ return TableItem{
+ Table: td,
+ Sheet: stringField(m, "sheet"),
+ Positions: positionsField(m),
+ CKType: stringField(m, "ck_type"),
+ // Same as imageItemFromMap: keep "positions" in Extra so non-[][]float64
+ // position matrices survive the typed conversion unchanged.
+ Extra: extraFromMap(m, "table", DocTypeKey, "ck_type", "sheet"),
+ }
+}
+
+func stringField(m map[string]any, key string) string {
+ s, _ := m[key].(string)
+ return s
+}
+
+func positionsField(m map[string]any) [][]float64 {
+ p, _ := m["positions"].([][]float64)
+ return p
+}
+
+// extraFromMap returns every key of m that is not a canonical field for the
+// item type, so it lands in the item's Extra passthrough (matching mergeExtra
+// semantics downstream). A nil result means "no extra keys".
+func extraFromMap(m map[string]any, canonical ...string) map[string]any {
+ skip := make(map[string]struct{}, len(canonical))
+ for _, c := range canonical {
+ skip[c] = struct{}{}
+ }
+ extra := make(map[string]any)
+ for k, v := range m {
+ if _, ok := skip[k]; ok {
+ continue
+ }
+ extra[k] = v
+ }
+ if len(extra) == 0 {
+ return nil
+ }
+ return extra
+}
+
+// DocItemsFromMaps maps a []map[string]any to typed DocItems, preserving order,
+// via DocItemFromMap. Parsers that accumulate results as []map[string]any use
+// it at the assignment site (`JSON: DocItemsFromMaps(items)`) so their internal
+// logic is unchanged while the field gains the DocItem type and the concrete
+// constructors become the emission path.
+func DocItemsFromMaps(maps []map[string]any) []DocItem {
+ if maps == nil {
+ return nil
+ }
+ out := make([]DocItem, 0, len(maps))
+ for _, m := range maps {
+ out = append(out, DocItemFromMap(m))
+ }
+ return out
+}
+
+// ItemsToMaps projects a slice of DocItems onto the historical wire form. The
+// ingestion boundary calls this once so downstream consumers keep reading
+// []map[string]any unchanged.
+func ItemsToMaps(items []DocItem) []map[string]any {
+ if items == nil {
+ return nil
+ }
+ out := make([]map[string]any, 0, len(items))
+ for _, it := range items {
+ if it == nil {
+ continue
+ }
+ out = append(out, it.ToMap())
+ }
+ return out
+}
+
+// mergeExtra copies Extra entries onto m without clobbering canonical keys
+// that were already set. Canonical typed fields always win; ad-hoc pipeline
+// keys (layout, page_number, level, ...) are added only when not already
+// present, so a stray Extra key can never shadow text/table/image/doc_type_kwd.
+func mergeExtra(m, extra map[string]any) map[string]any {
+ for k, v := range extra {
+ if _, ok := m[k]; !ok {
+ m[k] = v
+ }
+ }
+ return m
+}
+
+func firstNonEmptyStr(values ...string) string {
+ for _, v := range values {
+ if v != "" {
+ return v
+ }
+ }
+ return ""
+}
diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.go
new file mode 100644
index 0000000000..dfbb5db632
--- /dev/null
+++ b/internal/parser/parser/doc_item_test.go
@@ -0,0 +1,339 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ "reflect"
+ "testing"
+
+ "ragflow/internal/entity"
+)
+
+// mustMap fails the test if the two maps differ.
+func mustMap(t *testing.T, want, got map[string]any) {
+ t.Helper()
+ if !reflect.DeepEqual(want, got) {
+ t.Fatalf("map mismatch\nwant: %#v\ngot: %#v", want, got)
+ }
+}
+
+func TestTextItem_ToMap(t *testing.T) {
+ t.Run("minimal", func(t *testing.T) {
+ mustMap(t, map[string]any{"text": "", DocTypeKey: DocTypeText},
+ TextItem{}.ToMap())
+ })
+ t.Run("with text and no ck_type", func(t *testing.T) {
+ mustMap(t, map[string]any{"text": "body", DocTypeKey: DocTypeText},
+ TextItem{Text: "body"}.ToMap())
+ })
+ t.Run("ck_type emitted only when non-empty", func(t *testing.T) {
+ mustMap(t, map[string]any{"text": "h", DocTypeKey: DocTypeText, "ck_type": "heading"},
+ TextItem{Text: "h", CKType: "heading"}.ToMap())
+ // empty CKType must NOT appear as a key.
+ if _, ok := (TextItem{Text: "h"}).ToMap()["ck_type"]; ok {
+ t.Error("ck_type present despite empty CKType")
+ }
+ })
+ t.Run("extra passthrough", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "text": "x",
+ DocTypeKey: DocTypeText,
+ "layout": "text",
+ "page_number": float64(3),
+ "level": float64(2),
+ }, TextItem{Text: "x", Extra: map[string]any{
+ "layout": "text", "page_number": float64(3), "level": float64(2),
+ }}.ToMap())
+ })
+}
+
+func TestTextItem_ExtraDoesNotClobberCanonical(t *testing.T) {
+ // An attacker/stray Extra key must never shadow the typed fields.
+ got := TextItem{
+ Text: "real",
+ CKType: "heading",
+ Extra: map[string]any{
+ "text": "evil",
+ DocTypeKey: "evil",
+ "ck_type": "evil",
+ "layout": "kept", // not a canonical key -> passes through
+ },
+ }.ToMap()
+ mustMap(t, map[string]any{
+ "text": "real",
+ DocTypeKey: DocTypeText,
+ "ck_type": "heading",
+ "layout": "kept",
+ }, got)
+}
+
+func TestTableItem_ToMap(t *testing.T) {
+ t.Run("nil table still emits shape", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "table": (*entity.TableData)(nil),
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ }, TableItem{}.ToMap())
+ })
+ t.Run("ck_type falls back to table when empty", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "table": (*entity.TableData)(nil),
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ }, TableItem{CKType: ""}.ToMap())
+ })
+ t.Run("ck_type explicit wins", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "table": (*entity.TableData)(nil),
+ DocTypeKey: DocTypeTable,
+ "ck_type": "table",
+ "sheet": "Sheet1",
+ "positions": [][]float64{{0, 1, 1, 2, 2}},
+ }, TableItem{
+ CKType: "table",
+ Sheet: "Sheet1",
+ Positions: [][]float64{{0, 1, 1, 2, 2}},
+ }.ToMap())
+ })
+ t.Run("empty sheet/positions omitted", func(t *testing.T) {
+ got := TableItem{Table: &entity.TableData{}, Sheet: "", Positions: nil}.ToMap()
+ if _, ok := got["sheet"]; ok {
+ t.Error("sheet present despite empty Sheet")
+ }
+ if _, ok := got["positions"]; ok {
+ t.Error("positions present despite nil Positions")
+ }
+ })
+}
+
+func TestImageItem_ToMap(t *testing.T) {
+ t.Run("docx-like: no ck_type, empty image", func(t *testing.T) {
+ // docx image items historically set doc_type_kwd:"image" with no
+ // ck_type and an empty image when over budget.
+ mustMap(t, map[string]any{
+ "text": "",
+ "image": "",
+ DocTypeKey: DocTypeImage,
+ }, ImageItem{}.ToMap())
+ if _, ok := (ImageItem{}).ToMap()["ck_type"]; ok {
+ t.Error("ck_type present despite empty CKType")
+ }
+ })
+ t.Run("xlsx-like: ck_type image + sheet + positions", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "text": "alt",
+ "image": "data:image/png;base64,xxx",
+ DocTypeKey: DocTypeImage,
+ "ck_type": "image",
+ "sheet": "Sheet2",
+ "positions": [][]float64{{1, 0, 0, 3, 4}},
+ }, ImageItem{
+ Text: "alt",
+ Image: "data:image/png;base64,xxx",
+ CKType: "image",
+ Sheet: "Sheet2",
+ Positions: [][]float64{{1, 0, 0, 3, 4}},
+ }.ToMap())
+ })
+ t.Run("extra passthrough (media_order/parent_table_id)", func(t *testing.T) {
+ mustMap(t, map[string]any{
+ "text": "",
+ "image": "",
+ DocTypeKey: DocTypeImage,
+ "media_order": float64(1),
+ "parent_table_id": "t0",
+ "row_index": float64(0),
+ }, ImageItem{Extra: map[string]any{
+ "media_order": float64(1), "parent_table_id": "t0", "row_index": float64(0),
+ }}.ToMap())
+ })
+}
+
+func TestRawItem_Passthrough(t *testing.T) {
+ t.Run("exact shape preserved", func(t *testing.T) {
+ in := map[string]any{
+ "text": "hello",
+ DocTypeKey: DocTypeText,
+ "ck_type": "heading",
+ "layout": "title",
+ "arbitrary": []int{1, 2, 3},
+ "nested": map[string]any{"k": "v"},
+ }
+ got := RawItem{M: in}.ToMap()
+ mustMap(t, in, got)
+ })
+ t.Run("DocType is empty (lives in map)", func(t *testing.T) {
+ if (RawItem{M: map[string]any{DocTypeKey: DocTypeText}}).DocType() != "" {
+ t.Error("RawItem.DocType should be empty; discriminator is in the map")
+ }
+ })
+ t.Run("nil map does not panic", func(t *testing.T) {
+ var ri RawItem
+ if ri.ToMap() != nil {
+ t.Error("ToMap of nil RawItem.M should be nil")
+ }
+ })
+}
+
+func TestItemsToMaps(t *testing.T) {
+ t.Run("order preserved, nil skipped", func(t *testing.T) {
+ items := []DocItem{
+ TextItem{Text: "a"},
+ nil,
+ RawItem{M: map[string]any{DocTypeKey: DocTypeText, "text": "b"}},
+ TableItem{Table: &entity.TableData{}},
+ }
+ got := ItemsToMaps(items)
+ if len(got) != 3 {
+ t.Fatalf("expected 3 items (nil skipped), got %d: %#v", len(got), got)
+ }
+ if got[0]["text"] != "a" || got[1]["text"] != "b" {
+ t.Errorf("order not preserved: %#v", got)
+ }
+ if got[2][DocTypeKey] != DocTypeTable {
+ t.Errorf("third item should be table: %#v", got[2])
+ }
+ })
+ t.Run("nil slice", func(t *testing.T) {
+ if ItemsToMaps(nil) != nil {
+ t.Error("ItemsToMaps(nil) should be nil")
+ }
+ })
+}
+
+func TestConstructorWireShapeRegression(t *testing.T) {
+ // Guards that the two canonical constructors still emit the exact wire form
+ // parsers (and the chunker) depend on.
+ t.Run("NewTextJSONItem", func(t *testing.T) {
+ mustMap(t, map[string]any{"text": "hi", DocTypeKey: DocTypeText},
+ NewTextJSONItem("hi").ToMap())
+ })
+ t.Run("NewTableDataJSONItem", func(t *testing.T) {
+ td := &entity.TableData{}
+ mustMap(t, map[string]any{
+ "table": td,
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ "sheet": "S",
+ "positions": [][]float64{{0, 0, 0, 1, 1}},
+ }, NewTableDataJSONItem(td, "S", [][]float64{{0, 0, 0, 1, 1}}).ToMap())
+ })
+}
+
+func TestToMapIndependentPerCall(t *testing.T) {
+ // Mutating the returned map must not affect a later call or shared Extra.
+ it := TextItem{Text: "x", Extra: map[string]any{"k": "v"}}
+ first := it.ToMap()
+ first["text"] = "mutated"
+ second := it.ToMap()
+ if second["text"] != "x" {
+ t.Errorf("ToMap result was not independent: %#v", second)
+ }
+ if first["k"] != "v" || second["k"] != "v" {
+ t.Errorf("Extra passthrough broke: %#v %#v", first, second)
+ }
+}
+
+// DocItemFromMap detects doc_type_kwd and builds the matching typed item,
+// extracting canonical fields into the struct and leaving the rest in Extra.
+// For an opaque map without a known discriminator it falls back to RawItem,
+// preserving the map exactly. The produced item's ToMap must equal the input
+// map (the wire shape is unchanged), so producers can emit typed items
+// without altering downstream chunking.
+func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
+ textMap := map[string]any{
+ "text": "hello",
+ DocTypeKey: DocTypeText,
+ "ck_type": "heading",
+ "layout": "1",
+ "page_number": 3,
+ }
+ imageMap := map[string]any{
+ "text": "alt",
+ "image": "data:image/png;base64,xxx",
+ DocTypeKey: DocTypeImage,
+ "ck_type": "image",
+ "page_number": 2,
+ }
+ tableMap := map[string]any{
+ "table": &entity.TableData{},
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ "sheet": "s",
+ "positions": [][]float64{{1, 2}},
+ "sheet_index": 0,
+ }
+ opaqueMap := map[string]any{"foo": "bar", "nested": map[string]any{"a": 1}}
+
+ // PDF crop positions arrive as [][]any (normalizePDFPositions), not
+ // [][]float64; the typed conversion must preserve them verbatim. Regression
+ // guard for the positions-dropping bug caught during review.
+ imageMapAny := map[string]any{
+ "text": "alt",
+ "image": "data:image/png;base64,xxx",
+ DocTypeKey: DocTypeImage,
+ "ck_type": "image",
+ "positions": [][]any{{1, 2, 3, 4}},
+ }
+ tableMapAny := map[string]any{
+ "table": &entity.TableData{},
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ "positions": [][]any{{1, 2, 3, 4}},
+ }
+
+ for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny} {
+ got := DocItemFromMap(m).ToMap()
+ if !reflect.DeepEqual(got, m) {
+ t.Fatalf("DocItemFromMap round-trip mismatch for %q\n got=%#v\nwant=%#v", m[DocTypeKey], got, m)
+ }
+ }
+
+ // The typed detection must actually yield the concrete type, not RawItem.
+ if _, ok := DocItemFromMap(textMap).(TextItem); !ok {
+ t.Error("text map should produce a TextItem")
+ }
+ if _, ok := DocItemFromMap(imageMap).(ImageItem); !ok {
+ t.Error("image map should produce an ImageItem")
+ }
+ if _, ok := DocItemFromMap(tableMap).(TableItem); !ok {
+ t.Error("table map should produce a TableItem")
+ }
+ if _, ok := DocItemFromMap(opaqueMap).(RawItem); !ok {
+ t.Error("opaque map should fall back to RawItem")
+ }
+}
+
+// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way
+// DocItemFromMap does, preserving order and skipping nothing.
+func TestDocItemsFromMaps_OrderAndType(t *testing.T) {
+ maps := []map[string]any{
+ {"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},
+ {"foo": "bar"},
+ }
+ items := DocItemsFromMaps(maps)
+ if len(items) != 2 {
+ t.Fatalf("expected 2 items, got %d", len(items))
+ }
+ if _, ok := items[0].(TextItem); !ok {
+ t.Errorf("first should be TextItem: %#v", items[0])
+ }
+ if _, ok := items[1].(RawItem); !ok {
+ t.Errorf("second (opaque) should be RawItem: %#v", items[1])
+ }
+ if items[0].ToMap()["text"] != "a" {
+ t.Errorf("first item text lost: %#v", items[0].ToMap())
+ }
+}
diff --git a/internal/parser/parser/doc_parser.go b/internal/parser/parser/doc_parser.go
index 295511eac4..252a883975 100644
--- a/internal/parser/parser/doc_parser.go
+++ b/internal/parser/parser/doc_parser.go
@@ -91,7 +91,7 @@ func (p *DOCParser) ParseWithResult(ctx context.Context, filename string, data [
}
res.OutputFormat = "json"
- res.JSON = []map[string]any{NewTextJSONItem(text)}
+ res.JSON = []DocItem{NewTextJSONItem(text)}
return res
}
diff --git a/internal/parser/parser/doc_parser_synth_test.go b/internal/parser/parser/doc_parser_synth_test.go
index 6f2ae1c0b2..d48e7b3f9b 100644
--- a/internal/parser/parser/doc_parser_synth_test.go
+++ b/internal/parser/parser/doc_parser_synth_test.go
@@ -162,7 +162,7 @@ func TestDOCParser_SyntheticDocRoundTrip(t *testing.T) {
t.Errorf("parsed text missing %q; got:\n%s", want, pr.Text)
}
}
- if got, _ := pr.JSON[0]["text"].(string); got != pr.Text {
+ if got, _ := pr.JSON[0].ToMap()["text"].(string); got != pr.Text {
t.Errorf("JSON text = %q, want complete parser text %q", got, pr.Text)
}
diff --git a/internal/parser/parser/docx_parser.go b/internal/parser/parser/docx_parser.go
index f691f653d4..21ef9ffaf4 100644
--- a/internal/parser/parser/docx_parser.go
+++ b/internal/parser/parser/docx_parser.go
@@ -151,7 +151,7 @@ func (p *DOCXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{
OutputFormat: "json",
File: fileMeta,
- JSON: sections,
+ JSON: MapsToItems(sections),
Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/docx_parser_cgo_test.go b/internal/parser/parser/docx_parser_cgo_test.go
index 6132e1eff7..1141a0ded0 100644
--- a/internal/parser/parser/docx_parser_cgo_test.go
+++ b/internal/parser/parser/docx_parser_cgo_test.go
@@ -24,7 +24,7 @@ func TestDOCXParser_ParseWithResult_JSON(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("JSON items is empty; expected parsed content")
}
- for i, item := range res.JSON {
+ for i, item := range ItemsToMaps(res.JSON) {
if _, ok := item["text"]; !ok {
t.Errorf("item[%d] missing 'text' field", i)
}
@@ -161,7 +161,7 @@ func TestDOCXParser_PreservesXMLCharacterReferences(t *testing.T) {
if jsonRes.Err != nil {
t.Fatalf("ParseWithResult(json): %v", jsonRes.Err)
}
- if got := joinItemTexts(jsonRes.JSON); !strings.Contains(got, tc.want) {
+ if got := joinItemTexts(ItemsToMaps(jsonRes.JSON)); !strings.Contains(got, tc.want) {
t.Errorf("json text = %q, want it to contain %q", got, tc.want)
}
diff --git a/internal/parser/parser/email_parser.go b/internal/parser/parser/email_parser.go
index ccfad86dbb..e34d3c60a8 100644
--- a/internal/parser/parser/email_parser.go
+++ b/internal/parser/parser/email_parser.go
@@ -199,13 +199,13 @@ func (p *EmailParser) parseEmail(ctx context.Context, filename string, data []by
content["doc_type_kwd"] = "text"
items := []map[string]any{content}
if headerText := searchableEmailHeaders(content); headerText != "" {
- items = append(items, NewTextJSONItem(headerText))
+ items = append(items, NewTextJSONItem(headerText).ToMap())
}
items = append(items, extraItems...)
return ParseResult{
OutputFormat: "json",
File: map[string]any{"name": filename},
- JSON: items,
+ JSON: MapsToItems(items),
}
}
@@ -845,7 +845,7 @@ func (p *EmailParser) rechunkEmailAttachments(ctx context.Context, content map[s
var texts []string
if res.OutputFormat == "json" && len(res.JSON) > 0 {
for _, it := range res.JSON {
- if t, ok := it["text"].(string); ok {
+ if t, ok := it.ToMap()["text"].(string); ok {
if t = strings.TrimSpace(t); t != "" {
texts = append(texts, t)
}
diff --git a/internal/parser/parser/email_parser_test.go b/internal/parser/parser/email_parser_test.go
index 1ffd9012a5..aafde00560 100644
--- a/internal/parser/parser/email_parser_test.go
+++ b/internal/parser/parser/email_parser_test.go
@@ -68,7 +68,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {
if len(result.JSON) != 2 {
t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
if v, ok := item["from"].(string); !ok || v != "[email protected]" {
t.Errorf("from: got %q", v)
@@ -93,7 +93,7 @@ func TestEmailParser_EmlJSON(t *testing.T) {
t.Errorf("doc_type_kwd: got %q", v)
}
- header := result.JSON[1]
+ header := result.JSON[1].ToMap()
headerText, ok := header["text"].(string)
if !ok {
t.Fatalf("header item text missing: %#v", header)
@@ -235,7 +235,7 @@ func TestEmailParser_EmlTextChunkText(t *testing.T) {
if jres.Err != nil {
t.Fatalf("unexpected error: %v", jres.Err)
}
- item := jres.JSON[0]
+ item := jres.JSON[0].ToMap()
if _, ok := item["metadata"].(map[string]any); !ok {
t.Errorf("json output must keep metadata: %#v", item)
}
@@ -271,7 +271,7 @@ func TestEmailParser_MsgSupported(t *testing.T) {
if len(result.JSON) != 2 {
t.Fatalf("expected body and header JSON items, got %d", len(result.JSON))
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
if v, ok := item["from"].(string); !ok || v != "<[email protected]>" {
t.Errorf("from: got %q", v)
@@ -359,7 +359,7 @@ func TestEmailParser_MsgMetadataAlwaysPresent(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
if _, ok := item["metadata"].(map[string]any); !ok {
t.Fatalf("metadata must always be present for .msg, got %T", item["metadata"])
@@ -405,7 +405,7 @@ func TestEmailParser_MsgTextOutputExcludesMetadata(t *testing.T) {
// concatenated item text so the test survives the json-path Text removal.
var sb strings.Builder
for _, it := range result.JSON {
- if txt, _ := it["text"].(string); txt != "" {
+ if txt, _ := it.ToMap()["text"].(string); txt != "" {
sb.WriteString(txt)
sb.WriteString("\n")
}
@@ -463,7 +463,7 @@ func TestEmailParser_Base64Attachment(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
// attachments are dropped from the final ParseResult (consumed by
// rechunk, then deleted); verify the high-level result is clean...
@@ -541,7 +541,7 @@ func TestEmailParser_Base64AttachmentInMixedMultipart(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
// Verify body text was extracted from nested multipart/alternative
if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") {
@@ -598,7 +598,7 @@ func TestEmailParser_Multipart(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain text body") {
t.Errorf("text: got %q", v)
}
@@ -636,7 +636,7 @@ func TestEmailParser_MetadataAlwaysPresent(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
// metadata must exist even though it is not in fields (Python contract).
meta, ok := item["metadata"].(map[string]any)
@@ -696,7 +696,7 @@ func TestEmailParser_TextHTMLAlwaysPresent(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
if v, ok := item["text"].(string); !ok || !strings.Contains(v, "Plain body") {
t.Errorf("text: got %q", v)
@@ -760,13 +760,13 @@ func TestEmailParser_AttachmentSearchableJSON(t *testing.T) {
}
// Body must remain on the main item.
- if v, ok := result.JSON[0]["text"].(string); !ok || !strings.Contains(v, "Email body text") {
- t.Errorf("body missing on main item: %v", result.JSON[0]["text"])
+ if v, ok := result.JSON[0].ToMap()["text"].(string); !ok || !strings.Contains(v, "Email body text") {
+ t.Errorf("body missing on main item: %v", result.JSON[0].ToMap()["text"])
}
// Attachment text must appear as a separate retrievable JSON item.
found := false
- for _, it := range result.JSON {
+ for _, it := range ItemsToMaps(result.JSON) {
if txt, ok := it["text"].(string); ok && strings.Contains(txt, "quick brown fox") {
found = true
}
@@ -866,7 +866,7 @@ func TestEmailParser_AttachmentEmptyPayloadSkipped(t *testing.T) {
}
found := false
- for _, it := range result.JSON {
+ for _, it := range ItemsToMaps(result.JSON) {
if txt, ok := it["text"].(string); ok && strings.Contains(txt, "VALID attachment payload") {
found = true
}
@@ -922,7 +922,7 @@ func TestEmailParser_AttachmentUnknownExtSkipped(t *testing.T) {
}
found := false
- for _, it := range result.JSON {
+ for _, it := range ItemsToMaps(result.JSON) {
if txt, ok := it["text"].(string); ok && strings.Contains(txt, "KNOWN attachment payload") {
found = true
}
@@ -967,7 +967,7 @@ func TestEmailParser_AttachmentsWithoutBody(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
// text/text_html must be absent (gated by "body"), matching Python.
if _, ok := item["text"]; ok {
@@ -1054,7 +1054,7 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {
// result.JSON; text populates result.Text).
var indexed strings.Builder
if format == "json" {
- for _, it := range result.JSON {
+ for _, it := range ItemsToMaps(result.JSON) {
if txt, ok := it["text"].(string); ok {
indexed.WriteString(txt)
indexed.WriteString("\n")
@@ -1077,7 +1077,7 @@ func TestEmailParser_NestedEMLAttachmentRechunk(t *testing.T) {
// metadata key (the old bug leaked a "metadata:{...}" string as
// its text).
found := false
- for _, it := range result.JSON {
+ for _, it := range ItemsToMaps(result.JSON) {
if txt, ok := it["text"].(string); ok && strings.Contains(txt, "INNER BODY SECRET") {
found = true
if _, hasMeta := it["metadata"]; hasMeta {
@@ -1420,7 +1420,7 @@ func TestParseEML_SinglePartTransferEncoding(t *testing.T) {
if result.Err != nil {
t.Fatalf("unexpected error: %v", result.Err)
}
- item := result.JSON[0]
+ item := result.JSON[0].ToMap()
text, _ := item["text"].(string)
if strings.TrimRight(text, "\r\n") != strings.TrimRight(tt.wantText, "\r\n") {
diff --git a/internal/parser/parser/epub_parser.go b/internal/parser/parser/epub_parser.go
index d26574d37a..52b496cc89 100644
--- a/internal/parser/parser/epub_parser.go
+++ b/internal/parser/parser/epub_parser.go
@@ -64,7 +64,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data
"size": 0,
"encoding": "utf-8",
},
- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},
+ JSON: []DocItem{NewTextJSONItem("")},
}
}
@@ -97,7 +97,7 @@ func (p *EPUBParser) ParseWithResult(ctx context.Context, filename string, data
"size": len(data),
"encoding": detectedEnc,
},
- JSON: items,
+ JSON: MapsToItems(items),
}
}
diff --git a/internal/parser/parser/epub_parser_test.go b/internal/parser/parser/epub_parser_test.go
index 3e22fc8c6c..2d2170cf43 100644
--- a/internal/parser/parser/epub_parser_test.go
+++ b/internal/parser/parser/epub_parser_test.go
@@ -93,7 +93,7 @@ func parseTestEPUB(t *testing.T, chapters []epubTestChapter) ParseResult {
func epubTestText(res ParseResult) string {
var parts []string
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
parts = append(parts, text)
}
diff --git a/internal/parser/parser/html_charset_test.go b/internal/parser/parser/html_charset_test.go
index 604f4ba442..af5a46d250 100644
--- a/internal/parser/parser/html_charset_test.go
+++ b/internal/parser/parser/html_charset_test.go
@@ -62,7 +62,7 @@ func htmlEncodedFixture(t *testing.T, src string, enc encoding.Encoding) []byte
// itemsText concatenates the text of every item in res, newline separated.
func itemsText(res ParseResult) string {
var sb strings.Builder
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
sb.WriteString(renderedText(it))
sb.WriteString("\n")
}
@@ -159,7 +159,7 @@ func TestHTMLParser_DecodeTableSurvivesGBK(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var tableText string
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
if it["doc_type_kwd"] == "table" {
tableText = renderedText(it)
}
diff --git a/internal/parser/parser/html_media_test.go b/internal/parser/parser/html_media_test.go
index 024e28a859..c0cc61f7c8 100644
--- a/internal/parser/parser/html_media_test.go
+++ b/internal/parser/parser/html_media_test.go
@@ -21,14 +21,14 @@ func TestHTMLParser_EmitsInlineImagesInDocumentOrder(t *testing.T) {
if len(result.JSON) != 3 {
t.Fatalf("items = %+v, want text/image/text", result.JSON)
}
- if result.JSON[0]["text"] != "before" || result.JSON[0]["doc_type_kwd"] != "text" {
+ if result.JSON[0].ToMap()["text"] != "before" || result.JSON[0].ToMap()["doc_type_kwd"] != "text" {
t.Errorf("before item = %+v", result.JSON[0])
}
- imageItem := result.JSON[1]
+ imageItem := result.JSON[1].ToMap()
if imageItem["doc_type_kwd"] != "image" || imageItem["image"] != htmlMediaDataURI || imageItem["text"] != "figure alt" || imageItem["media_order"] != 1 {
t.Errorf("image item = %+v", imageItem)
}
- if result.JSON[2]["text"] != "after" || result.JSON[2]["doc_type_kwd"] != "text" {
+ if result.JSON[2].ToMap()["text"] != "after" || result.JSON[2].ToMap()["doc_type_kwd"] != "text" {
t.Errorf("after item = %+v", result.JSON[2])
}
}
@@ -42,20 +42,20 @@ func TestHTMLParser_EmitsTableCellImagesWithParentAndCellOrder(t *testing.T) {
if len(result.JSON) != 2 {
t.Fatalf("items = %+v, want table and one inline image item", result.JSON)
}
- if result.JSON[0]["doc_type_kwd"] != "table" || result.JSON[0]["source_table_id"] != "html-table-1" {
+ if result.JSON[0].ToMap()["doc_type_kwd"] != "table" || result.JSON[0].ToMap()["source_table_id"] != "html-table-1" {
t.Errorf("table item = %+v", result.JSON[0])
}
- tableText := renderedText(result.JSON[0])
+ tableText := renderedText(result.JSON[0].ToMap())
if strings.Contains(tableText, "data:image/") {
t.Error("table text duplicates the separately emitted inline image payload")
}
if !strings.Contains(tableText, "A") {
t.Errorf("table text lost the cell text: %q", tableText)
}
- if result.JSON[1]["image"] != htmlMediaDataURI {
- t.Errorf("inline image payload = %v, want data URI", result.JSON[1]["image"])
+ if result.JSON[1].ToMap()["image"] != htmlMediaDataURI {
+ t.Errorf("inline image payload = %v, want data URI", result.JSON[1].ToMap()["image"])
}
- item := result.JSON[1]
+ item := result.JSON[1].ToMap()
if item["doc_type_kwd"] != "image" || item["parent_table_id"] != "html-table-1" {
t.Errorf("inline image item = %+v", item)
}
@@ -71,7 +71,7 @@ func TestHTMLParser_RejectsOversizedInlineImagePayload(t *testing.T) {
if result.Err != nil {
t.Fatalf("ParseWithResult: %v", result.Err)
}
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] == "image" {
t.Fatal("oversized data URI emitted as an image item")
}
diff --git a/internal/parser/parser/html_parser.go b/internal/parser/parser/html_parser.go
index a41404d366..d7457a8e92 100644
--- a/internal/parser/parser/html_parser.go
+++ b/internal/parser/parser/html_parser.go
@@ -98,7 +98,7 @@ func (p *HTMLParser) ParseWithResult(ctx context.Context, filename string, data
"name": filename,
"encoding": encName,
},
- JSON: items,
+ JSON: MapsToItems(items),
Warnings: state.warnings(),
}
}
diff --git a/internal/parser/parser/html_parser_align_test.go b/internal/parser/parser/html_parser_align_test.go
index daf42abfe9..5c0d93a621 100644
--- a/internal/parser/parser/html_parser_align_test.go
+++ b/internal/parser/parser/html_parser_align_test.go
@@ -33,7 +33,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
var tableText string
tableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
- for i, it := range res.JSON {
+ for i, it := range ItemsToMaps(res.JSON) {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
@@ -61,7 +61,7 @@ func TestHTMLParser_TableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {
+ if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {
t.Errorf("structured table item ck_type = %v, want %v", got, want)
}
if !strings.Contains(tableText, "<table") ||
@@ -116,7 +116,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
var tableText string
tableIdx, headingIdx, trailingIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
- for i, it := range res.JSON {
+ for i, it := range ItemsToMaps(res.JSON) {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
@@ -144,7 +144,7 @@ func TestHTMLParser_NestedTableProducesStructuredItems(t *testing.T) {
if tableCount != 1 {
t.Fatalf("structured doc_type_kwd:\"table\" item count = %d, want exactly 1", tableCount)
}
- if got, want := res.JSON[tableIdx]["ck_type"], "table"; got != want {
+ if got, want := res.JSON[tableIdx].ToMap()["ck_type"], "table"; got != want {
t.Errorf("structured nested table item ck_type = %v, want %v", got, want)
}
if !strings.Contains(tableText, "<table") ||
@@ -202,7 +202,7 @@ func TestHTMLParser_NestedTableWithSurroundingText(t *testing.T) {
var tableText string
tableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
- for i, it := range res.JSON {
+ for i, it := range ItemsToMaps(res.JSON) {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
@@ -282,7 +282,7 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {
var tableItemIdx []int
titleIdx, middleIdx, endIdx := -1, -1, -1
- for i, it := range res.JSON {
+ for i, it := range ItemsToMaps(res.JSON) {
text, _ := it["text"].(string)
switch it["doc_type_kwd"] {
case "text":
@@ -312,8 +312,8 @@ func TestHTMLParser_MultipleTablesOrdering(t *testing.T) {
if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
}
- t1 := renderedText(res.JSON[tableItemIdx[0]])
- t2 := renderedText(res.JSON[tableItemIdx[1]])
+ t1 := renderedText(res.JSON[tableItemIdx[0]].ToMap())
+ t2 := renderedText(res.JSON[tableItemIdx[1]].ToMap())
if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
t.Errorf("first table item missing x/y cells: %q", t1)
}
@@ -356,7 +356,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {
// Join every item's text in order to check content + order + fusion.
var all string
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
if txt, _ := it["text"].(string); txt != "" {
all += txt + "\n"
}
@@ -390,7 +390,7 @@ func TestHTMLParser_NestedListItemsPreserved(t *testing.T) {
// therefore loses ck_type:"list" (becomes "text") — a fidelity nuance,
// not a collapse, since downstream chunking reads doc_type_kwd not ck_type.
var listText string
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
if it["ck_type"] == "list" {
listText, _ = it["text"].(string)
break
@@ -440,7 +440,7 @@ SELECT * FROM t
}
var preItem, textareaItem map[string]any
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
txt, _ := it["text"].(string)
switch {
case strings.Contains(txt, "def foo()"):
@@ -525,14 +525,14 @@ func TestHTMLParser_AlignmentGolden(t *testing.T) {
// the <table> markup inlined as a "text" item. filterTableDivergence drops
// the table content from both sides so the comparison stays on non-table prose.
drop := AcceptedDivergences(doc.Meta)
- goText := filterTableDivergence(res.JSON, drop)
+ goText := filterTableDivergence(ItemsToMaps(res.JSON), drop)
pyText := filterTableDivergence(doc.Items, drop)
// filterTableDivergence drops the table from the prose comparison;
// the table-equivalence guard below (assertTablesEquivalent) checks
// the table cell content still matches Python, so a collapse or
// dropped column on either side is caught independently.
- assertTablesEquivalent(t, res.JSON, doc.Items)
+ assertTablesEquivalent(t, ItemsToMaps(res.JSON), doc.Items)
if ok, diff := CompareAlignment(goText, pyText, HTMLAlignOptions()); !ok {
t.Fatalf("html parser not aligned with Python golden:%s", diff)
diff --git a/internal/parser/parser/html_parser_test.go b/internal/parser/parser/html_parser_test.go
index 6110678611..02eaadeead 100644
--- a/internal/parser/parser/html_parser_test.go
+++ b/internal/parser/parser/html_parser_test.go
@@ -50,7 +50,7 @@ func TestHTMLParser_RemoveHeaderFooter(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
for _, banned := range []string{"Site Navigation", "Copyright 2024", "Ad banner", "Legal info"} {
if strings.Contains(text, banned) {
@@ -60,7 +60,7 @@ func TestHTMLParser_RemoveHeaderFooter(t *testing.T) {
}
// Main content must survive.
found := false
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if text, _ := item["text"].(string); strings.Contains(text, "Main content here") {
found = true
}
@@ -82,7 +82,7 @@ func TestHTMLParser_RemoveHeaderFooter_DisabledByDefault(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
found := false
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if text, _ := item["text"].(string); strings.Contains(text, "Nav") {
found = true
}
@@ -113,7 +113,7 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {
}
// "目录" heading and "第一章 概述" (first entry after heading)
// should be removed by removeContentsTable.
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if text == "目录" {
t.Errorf("TOC heading %q was not removed", text)
@@ -124,7 +124,7 @@ func TestHTMLParser_RemoveTOC(t *testing.T) {
}
// "前言" and "正文开始" should survive.
wantTexts := map[string]bool{"前言": false, "正文开始": false}
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if _, ok := wantTexts[text]; ok {
wantTexts[text] = true
diff --git a/internal/parser/parser/html_parser_unified_test.go b/internal/parser/parser/html_parser_unified_test.go
index 82031e31d2..38599581f9 100644
--- a/internal/parser/parser/html_parser_unified_test.go
+++ b/internal/parser/parser/html_parser_unified_test.go
@@ -62,7 +62,7 @@ func TestHTMLParser_ParseWithResult_UnifiedSemantics(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("block count = %d, want 1: %#v", len(res.JSON), res.JSON)
}
- if got := res.JSON[0]["text"].(string); got != tc.Want {
+ if got := res.JSON[0].ToMap()["text"].(string); got != tc.Want {
t.Errorf("got %q, want %q", got, tc.Want)
}
})
@@ -90,7 +90,7 @@ func TestHTMLParser_ParseWithResult_RealisticSmoke(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var texts []string
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
texts = append(texts, item["text"].(string))
}
want := []string{
diff --git a/internal/parser/parser/json_parser.go b/internal/parser/parser/json_parser.go
index 22523895e0..491e03bc2b 100644
--- a/internal/parser/parser/json_parser.go
+++ b/internal/parser/parser/json_parser.go
@@ -63,7 +63,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data
"size": len(data),
"encoding": encName,
},
- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},
+ JSON: []DocItem{NewTextJSONItem("")},
}
}
@@ -78,7 +78,7 @@ func (p *JSONParser) ParseWithResult(ctx context.Context, filename string, data
"size": len(data),
"encoding": encName,
},
- JSON: items,
+ JSON: MapsToItems(items),
}
}
diff --git a/internal/parser/parser/json_parser_test.go b/internal/parser/parser/json_parser_test.go
index 6abb6b5ae3..08f12d3636 100644
--- a/internal/parser/parser/json_parser_test.go
+++ b/internal/parser/parser/json_parser_test.go
@@ -10,7 +10,7 @@ import (
func jsonTexts(res ParseResult) []string {
texts := make([]string, 0, len(res.JSON))
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
texts = append(texts, text)
}
diff --git a/internal/parser/parser/markdown_parser.go b/internal/parser/parser/markdown_parser.go
index 38843297e6..2a20b9df7d 100644
--- a/internal/parser/parser/markdown_parser.go
+++ b/internal/parser/parser/markdown_parser.go
@@ -132,7 +132,7 @@ func (p *MarkdownParser) ParseWithResult(ctx context.Context, filename string, d
File: map[string]any{
"name": filename,
},
- JSON: items,
+ JSON: MapsToItems(items),
Warnings: warnings,
}
}
diff --git a/internal/parser/parser/markdown_parser_test.go b/internal/parser/parser/markdown_parser_test.go
index c4bb08dc40..27689d4b3d 100644
--- a/internal/parser/parser/markdown_parser_test.go
+++ b/internal/parser/parser/markdown_parser_test.go
@@ -31,10 +31,10 @@ func TestMarkdownParser_ParseWithResult_Basic(t *testing.T) {
t.Fatal("JSON is empty; want at least one item")
}
// Verify heading
- if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
t.Fatalf("first item text = %q, want %q", got, "# Hello")
}
- if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {
+ if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {
t.Fatalf("first item ck_type = %q, want %q", got, "heading")
}
}
@@ -49,10 +49,10 @@ func TestMarkdownParser_ParseWithResult_UTF8BOM(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if got, _ := res.JSON[0]["text"].(string); got != "# Hello" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); got != "# Hello" {
t.Fatalf("first item text = %q, want %q", got, "# Hello")
}
- if got, _ := res.JSON[0]["ck_type"].(string); got != "heading" {
+ if got, _ := res.JSON[0].ToMap()["ck_type"].(string); got != "heading" {
t.Fatalf("first item ck_type = %q, want %q", got, "heading")
}
}
@@ -78,7 +78,7 @@ func TestMarkdownParser_ListItemsKeepWordBoundaries(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var lists []string
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if ck, _ := item["ck_type"].(string); ck == "list" {
text, _ := item["text"].(string)
lists = append(lists, text)
@@ -140,7 +140,7 @@ func TestMarkdownParser_OrderedListKeepsNumbers(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var lists []string
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if ck, _ := item["ck_type"].(string); ck == "list" {
text, _ := item["text"].(string)
lists = append(lists, text)
@@ -181,7 +181,7 @@ func TestMarkdownParser_TableKeepsCellsBeyondTheHeader(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var joined strings.Builder
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
joined.WriteString(renderedText(item))
joined.WriteString("\n")
}
@@ -222,7 +222,7 @@ func TestMarkdownParser_ParseWithResult_ImageDataURI(t *testing.T) {
}
// Find the image item
found := false
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
found = true
if img, ok := item["image"].(string); !ok || img != pixelB64 {
@@ -244,7 +244,7 @@ func TestMarkdownParser_ParseWithResult_NoImage(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
t.Fatal("unexpected image item in text-only markdown")
}
@@ -267,7 +267,7 @@ func TestMarkdownParser_ParseWithResult_RendersTableInline(t *testing.T) {
t.Fatalf("len(JSON) = %d, want 3 (leading paragraph, table, note)", len(res.JSON))
}
tableCount, inlineTableCount := 0, 0
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
kd, _ := item["doc_type_kwd"].(string)
text, _ := item["text"].(string)
switch kd {
@@ -320,7 +320,7 @@ func TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var sb strings.Builder
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
sb.WriteString(text)
sb.WriteString("\n")
@@ -344,7 +344,7 @@ func TestMarkdownParser_ParseWithResult_DollarIsCurrencyNotMath(t *testing.T) {
t.Fatalf("ParseWithResult: %v", resDisplay.Err)
}
var sbDisplay strings.Builder
- for _, item := range resDisplay.JSON {
+ for _, item := range ItemsToMaps(resDisplay.JSON) {
text, _ := item["text"].(string)
sbDisplay.WriteString(text)
}
@@ -372,7 +372,7 @@ func TestMarkdownParser_ParseWithResult_InlineRawHTMLSurvives(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var sb strings.Builder
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
sb.WriteString(text)
sb.WriteString("\n")
@@ -451,7 +451,7 @@ func TestMarkdownParser_FlattenMediaToText(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if kd, _ := item["doc_type_kwd"].(string); kd != tc.wantDocType {
t.Errorf("doc_type_kwd = %q, want %q (item text=%q)",
kd, tc.wantDocType, item["text"])
@@ -554,7 +554,7 @@ func TestFindBlockImage(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var imgItems, textItems int
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "image":
imgItems++
@@ -574,7 +574,7 @@ func TestFindBlockImage(t *testing.T) {
// A document with no image must not produce any image item.
noImg := p.ParseWithResult(ctx, "b.md", []byte("# T\n\nNo images here.\n"))
- for _, item := range noImg.JSON {
+ for _, item := range ItemsToMaps(noImg.JSON) {
if kd, _ := item["doc_type_kwd"].(string); kd == "image" {
t.Fatal("unexpected image item in text-only markdown")
}
@@ -623,7 +623,7 @@ func TestMarkdownParser_TableNotCollapsed(t *testing.T) {
var all strings.Builder
sawTitle, sawTrailing, sawTableItem, sawInlineTable := false, false, false, false
tableCount := 0
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
all.WriteString(text)
all.WriteString("\n")
@@ -687,7 +687,7 @@ func TestMarkdownParser_TableWithSurroundingText(t *testing.T) {
var tableText string
tableIdx, beforeIdx, afterIdx, inlineTableCount, tableCount := -1, -1, -1, 0, 0
- for i, item := range res.JSON {
+ for i, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "text":
@@ -751,7 +751,7 @@ func TestMarkdownParser_NonTableHTMLBlockNotTable(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if strings.Contains(text, "<div") || strings.Contains(text, "<style") {
if ck, ok := item["ck_type"].(string); ok && ck == "table" {
@@ -777,7 +777,7 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
// No table item may be emitted — the pipe rows live inside a code block.
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if kd, _ := item["doc_type_kwd"].(string); kd == "table" {
t.Fatalf("code-fence pipe rows wrongly emitted as table item: %q", item["text"])
}
@@ -785,7 +785,7 @@ func TestMarkdownParser_TableInCodeFenceNotRendered(t *testing.T) {
// The code block item must retain the raw pipe text and must NOT contain
// any <table> markup.
sawCode := false
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
if strings.Contains(text, "| A | B |") {
sawCode = true
@@ -814,7 +814,7 @@ func TestMarkdownParser_RawHTMLTableHandled(t *testing.T) {
}
sawInlined, sawTableItem := false, false
tableCount := 0
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "text":
@@ -859,7 +859,7 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
var tableItemIdx []int
var titleIdx, middleIdx, endIdx = -1, -1, -1
- for i, item := range res.JSON {
+ for i, item := range ItemsToMaps(res.JSON) {
text, _ := item["text"].(string)
switch kd, _ := item["doc_type_kwd"].(string); kd {
case "text":
@@ -887,8 +887,8 @@ func TestMarkdownParser_MultipleTablesOrdering(t *testing.T) {
if !(titleIdx < tableItemIdx[0] && tableItemIdx[0] < middleIdx && middleIdx < tableItemIdx[1] && tableItemIdx[1] < endIdx) {
t.Fatalf("table order wrong: tables=%v title=%d middle=%d end=%d", tableItemIdx, titleIdx, middleIdx, endIdx)
}
- t1 := renderedText(res.JSON[tableItemIdx[0]])
- t2 := renderedText(res.JSON[tableItemIdx[1]])
+ t1 := renderedText(res.JSON[tableItemIdx[0]].ToMap())
+ t2 := renderedText(res.JSON[tableItemIdx[1]].ToMap())
if !strings.Contains(t1, "x") || !strings.Contains(t1, "y") {
t.Fatalf("first table item missing x/y cells: %q", t1)
}
@@ -945,14 +945,14 @@ func TestMarkdownParser_AlignmentGolden(t *testing.T) {
// filterTableDivergence additionally drops the inlined <table> markup that
// Python keeps as a "text" item, so only non-table prose is compared.
ignore := AcceptedDivergences(gd.Meta)
- goText := filterTableDivergence(res.JSON, ignore)
+ goText := filterTableDivergence(ItemsToMaps(res.JSON), ignore)
pyText := filterTableDivergence(gd.Items, ignore)
// filterTableDivergence drops the table from the prose comparison;
// the table-equivalence guard below (assertTablesEquivalent) checks
// the table cell content still matches Python, so a collapse or
// dropped column on either side is caught independently.
- assertTablesEquivalent(t, res.JSON, gd.Items)
+ assertTablesEquivalent(t, ItemsToMaps(res.JSON), gd.Items)
if ok, diff := CompareAlignment(goText, pyText, MarkdownAlignOptions(DefaultMarkdownDelimiter)); !ok {
t.Fatalf("markdown parser not aligned with Python golden:%s", diff)
diff --git a/internal/parser/parser/office_image_fixture_cgo_test.go b/internal/parser/parser/office_image_fixture_cgo_test.go
index c7be30b61c..2241e27f08 100644
--- a/internal/parser/parser/office_image_fixture_cgo_test.go
+++ b/internal/parser/parser/office_image_fixture_cgo_test.go
@@ -43,7 +43,7 @@ func TestDOCXParser_RealTableImage(t *testing.T) {
t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
}
var tableID string
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] == "table" {
tableID, _ = item["source_table_id"].(string)
}
@@ -51,7 +51,7 @@ func TestDOCXParser_RealTableImage(t *testing.T) {
if tableID == "" {
t.Fatalf("table with image not found: %+v", result.JSON)
}
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] != "image" || item["parent_table_id"] != tableID {
continue
}
@@ -99,7 +99,7 @@ func TestPPTXParser_RealSlideImage(t *testing.T) {
if len(result.Warnings) != 0 {
t.Fatalf("warnings = %v, want none for an admitted image", result.Warnings)
}
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] != "image" {
continue
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 848ce14f7d..5eca25921f 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -203,9 +203,9 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int
grid = append(grid, cells)
}
td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}
- item := NewTableDataJSONItem(td, sheet, matrix)
- item["sheet_index"] = sheetIndex
- return item
+ ti := NewTableDataJSONItem(td, sheet, matrix).(TableItem)
+ ti.Extra = map[string]any{"sheet_index": sheetIndex}
+ return ti.ToMap()
}
prev := 0
diff --git a/internal/parser/parser/office_table_render_test.go b/internal/parser/parser/office_table_render_test.go
index 03727deaaf..ed8bf9cfd6 100644
--- a/internal/parser/parser/office_table_render_test.go
+++ b/internal/parser/parser/office_table_render_test.go
@@ -29,7 +29,7 @@ func TestXLSXParserEmitsSegmentedHTMLTable(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one segmented HTML table item", len(res.JSON))
}
- item := res.JSON[0]
+ item := res.JSON[0].ToMap()
if item["ck_type"] != "table" || item["doc_type_kwd"] != "table" {
t.Fatalf("item types = ck_type:%v doc_type:%v", item["ck_type"], item["doc_type_kwd"])
}
@@ -81,7 +81,7 @@ func TestXLSXParserHTML4ExcelIsIgnored(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one atomic table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table" || res.JSON[0]["table"] == nil {
+ if res.JSON[0].ToMap()["ck_type"] != "table" || res.JSON[0].ToMap()["table"] == nil {
t.Fatalf("html4excel item = %#v", res.JSON[0])
}
if !reflect.DeepEqual(res.JSON, plainRes.JSON) {
@@ -105,7 +105,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one HTML table item (parser must not pre-split rows)", len(res.JSON))
}
- positions := res.JSON[0]["positions"].([][]float64)
+ positions := res.JSON[0].ToMap()["positions"].([][]float64)
if len(positions) != dataRows+1 {
t.Fatalf("len(positions) = %d, want one tuple per <tr> (header plus %d rows)", len(positions), dataRows)
}
@@ -113,7 +113,7 @@ func TestXLSXParserDoesNotPrechunkRows(t *testing.T) {
if last[1] != float64(dataRows+1) || last[2] != float64(dataRows+1) {
t.Fatalf("last tuple = %v, want source row %d", last, dataRows+1)
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<td>258</td>") {
t.Fatalf("last data row missing from markup:\n%s", text)
}
@@ -138,7 +138,7 @@ func TestXLSXParserPreservesSheetOrderAndIdentity(t *testing.T) {
if len(res.JSON) != 2 {
t.Fatalf("items = %d, want one HTML table per sheet", len(res.JSON))
}
- first, second := res.JSON[0], res.JSON[1]
+ first, second := res.JSON[0].ToMap(), res.JSON[1].ToMap()
if first["sheet"] != "Sheet1" || first["sheet_index"] != 1 {
t.Fatalf("first sheet identity = %#v", first)
}
@@ -168,7 +168,7 @@ func TestXLSXParserEmitsHeaderOnlySegment(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("items = %d, want one header-only segment", len(res.JSON))
}
- item := res.JSON[0]
+ item := res.JSON[0].ToMap()
wantText := "<table><caption>Sheet1</caption>\n" +
"<tr><th>Name</th><th>Amount</th></tr>\n" +
"</table>\n"
@@ -233,7 +233,7 @@ func TestXLSXParserImageAnchorsSplitSegments(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- return res.JSON
+ return ItemsToMaps(res.JSON)
}
assertSegments := func(t *testing.T, items []map[string]any, wantFirst, wantSecond []string) {
@@ -332,7 +332,7 @@ func renderedText(item map[string]any) string {
func spreadsheetText(res ParseResult) string {
var out strings.Builder
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
if td, ok := item["table"].(*entity.TableData); ok {
out.WriteString(tableutil.RenderTableHTML(td))
}
@@ -344,7 +344,7 @@ func spreadsheetText(res ParseResult) string {
// joined by "; " — the rendered <th> cells of its first <tr> — or "" when no
// table item exists.
func spreadsheetHeaderText(res ParseResult) string {
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
td, ok := item["table"].(*entity.TableData)
if !ok {
continue
@@ -534,7 +534,7 @@ func TestXLSXParser_MergedDataCellInheritance(t *testing.T) {
if !strings.Contains(text, "<tr><td>G1</td><td>y</td></tr>") {
t.Fatalf("slave row did not inherit merged master text, got:\n%s", text)
}
- positions := res.JSON[0]["positions"].([][]float64)
+ positions := res.JSON[0].ToMap()["positions"].([][]float64)
if len(positions) != 3 {
t.Fatalf("len(positions) = %d, want one tuple per <tr>", len(positions))
}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index 09ba17d6ff..c09226bcdc 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -61,14 +61,15 @@ type ParseResult struct {
// echoed the requested extension.
File map[string]any
- // JSON is the structured payload when available.
- // Shape depends on the parser family: PDF emits
- // `[]map[string]any` with `text` + `doc_type_kwd` keys (and
- // optional `image` / `layout` / `positions` fields);
- // Markdown / text emit normalized
- // `{text, doc_type_kwd}` items; image emits an image payload and, after
- // ingestion enhancement, its VLM description.
- JSON []map[string]any
+ // JSON is the structured payload when available, as typed DocItems.
+ // Each item projects onto the historical wire form via DocItem.ToMap,
+ // so the external key shape (text + doc_type_kwd, optional image /
+ // layout / positions, table) is unchanged. Shape depends on the parser
+ // family: PDF emits `text` + `doc_type_kwd` items (and optional
+ // `image` / `layout` / `positions` fields); Markdown / HTML / text emit
+ // normalized `{text, doc_type_kwd}` items; image emits an image payload
+ // and, after ingestion enhancement, its VLM description.
+ JSON []DocItem
// Markdown is a backend response awaiting normalization.
Markdown string
@@ -101,30 +102,21 @@ const (
spreadsheetOutputFormat = "json"
)
-// NewTextJSONItem constructs a canonical text JSON item for parser output.
-func NewTextJSONItem(text string) map[string]any {
- return map[string]any{
- "text": text,
- DocTypeKey: DocTypeText,
- }
+// NewTextJSONItem constructs a canonical text DocItem for parser output.
+func NewTextJSONItem(text string) DocItem {
+ return TextItem{Text: text}
}
-// NewTableDataJSONItem constructs a canonical structured table JSON item for
+// NewTableDataJSONItem constructs a canonical structured table DocItem for
// parser output. It carries the parsed TableData contract (caption, header
// rows, cell text already normalized the same way the legacy chunker
// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,
// so the chunker reads rows directly without re-parsing any markup.
-func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) map[string]any {
- item := map[string]any{
- "table": td,
- DocTypeKey: DocTypeTable,
- "ck_type": DocTypeTable,
- }
- if sheet != "" {
- item["sheet"] = sheet
- }
- if len(positions) > 0 {
- item["positions"] = positions
+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) DocItem {
+ return TableItem{
+ Table: td,
+ Sheet: sheet,
+ Positions: positions,
+ CKType: DocTypeTable,
}
- return item
}
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 10d6caf0d1..257b8af6e1 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -30,21 +30,22 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
Caption: "Sheet1",
}
item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})
+ m := item.ToMap()
- if got, want := item[DocTypeKey], DocTypeTable; got != want {
+ if got, want := m[DocTypeKey], DocTypeTable; got != want {
t.Fatalf("item[%q] = %v, want %v", DocTypeKey, got, want)
}
- if got, want := item["ck_type"], DocTypeTable; got != want {
+ if got, want := m["ck_type"], DocTypeTable; got != want {
t.Fatalf("item[%q] = %v, want %v", "ck_type", got, want)
}
- got, ok := item["table"].(*entity.TableData)
+ got, ok := m["table"].(*entity.TableData)
if !ok {
- t.Fatalf("item[%q] = %T, want *entity.TableData", "table", item["table"])
+ t.Fatalf("item[%q] = %T, want *entity.TableData", "table", m["table"])
}
if got.Caption != "Sheet1" || len(got.Grid) != 2 {
t.Fatalf("item table = %+v, want caption Sheet1 and 2 rows", got)
}
- if _, ok := item["positions"].([][]float64); !ok {
+ if _, ok := m["positions"].([][]float64); !ok {
t.Fatalf("item[%q] missing positions matrix", "positions")
}
}
@@ -72,9 +73,9 @@ func TestParseResult_Contract(t *testing.T) {
name: "json family only",
in: ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: MapsToItems([]map[string]any{
{"text": "hello", "doc_type_kwd": "text"},
- },
+ }),
},
wantFmt: "json",
},
@@ -170,14 +171,14 @@ func TestMarkdownParser_ParseWithResult(t *testing.T) {
if len(res.JSON) != 3 {
t.Fatalf("JSON len = %d, want 3 (heading, paragraph, list)", len(res.JSON))
}
- heading := res.JSON[0]
+ heading := res.JSON[0].ToMap()
if txt, _ := heading["text"].(string); !strings.Contains(txt, "Title") {
t.Errorf("JSON[0].text = %q, want contains 'Title'", txt)
}
if got, _ := heading["ck_type"].(string); got != "heading" {
t.Errorf("JSON[0].ck_type = %q, want heading", got)
}
- for i, it := range res.JSON {
+ for i, it := range ItemsToMaps(res.JSON) {
if _, ok := it["doc_type_kwd"]; !ok {
t.Errorf("JSON[%d] missing doc_type_kwd: %+v", i, it)
}
diff --git a/internal/parser/parser/parse_with_result_test.go b/internal/parser/parser/parse_with_result_test.go
index d60782848b..7c51365021 100644
--- a/internal/parser/parser/parse_with_result_test.go
+++ b/internal/parser/parser/parse_with_result_test.go
@@ -61,10 +61,10 @@ func TestTextParser_ParseWithResult_PreservesOneTextUnit(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1: %#v", len(res.JSON), res.JSON)
}
- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {
+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {
t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)
}
- got, _ := res.JSON[0]["text"].(string)
+ got, _ := res.JSON[0].ToMap()["text"].(string)
want := "First sentence.\n\nSecond sentence!\nThird sentence?"
if got != want {
t.Errorf("JSON[0].text = %q, want %q", got, want)
@@ -103,7 +103,7 @@ func TestTextParser_ParseWithResult_NoSizeCap(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1 (no per-item size cap)", len(res.JSON))
}
- if txt, _ := res.JSON[0]["text"].(string); txt != long {
+ if txt, _ := res.JSON[0].ToMap()["text"].(string); txt != long {
t.Errorf("text len = %d, want %d (full content preserved, not sliced)", len(txt), len(long))
}
}
@@ -124,7 +124,7 @@ func TestTextParser_ParseWithResult_GBK(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("want non-empty items")
}
- got, _ := res.JSON[0]["text"].(string)
+ got, _ := res.JSON[0].ToMap()["text"].(string)
if got != "测试中文文本内容" {
t.Errorf("got %q, want %q", got, "测试中文文本内容")
}
@@ -154,22 +154,22 @@ func TestHTMLParser_ParseWithResult_BlockSplit(t *testing.T) {
if len(res.JSON) != 3 {
t.Fatalf("JSON len = %d, want 3 (h1, p, ul)", len(res.JSON))
}
- if got, want := res.JSON[0]["ck_type"], "heading"; got != want {
+ if got, want := res.JSON[0].ToMap()["ck_type"], "heading"; got != want {
t.Errorf("JSON[0].ck_type = %v, want %v", got, want)
}
- if got, want := res.JSON[0]["text"], "Title"; got != want {
+ if got, want := res.JSON[0].ToMap()["text"], "Title"; got != want {
t.Errorf("JSON[0].text = %v, want %v", got, want)
}
- if got, want := res.JSON[1]["ck_type"], "paragraph"; got != want {
+ if got, want := res.JSON[1].ToMap()["ck_type"], "paragraph"; got != want {
t.Errorf("JSON[1].ck_type = %v, want %v", got, want)
}
- if got, want := res.JSON[1]["text"], "First paragraph."; got != want {
+ if got, want := res.JSON[1].ToMap()["text"], "First paragraph."; got != want {
t.Errorf("JSON[1].text = %v, want %v", got, want)
}
- if got, want := res.JSON[2]["ck_type"], "list"; got != want {
+ if got, want := res.JSON[2].ToMap()["ck_type"], "list"; got != want {
t.Errorf("JSON[2].ck_type = %v, want %v", got, want)
}
- if got, want := res.JSON[2]["text"], "Item one"; got != want {
+ if got, want := res.JSON[2].ToMap()["text"], "Item one"; got != want {
t.Errorf("JSON[2].text = %v, want %v", got, want)
}
}
@@ -207,13 +207,13 @@ Tail text
t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, w := range want {
- if got := res.JSON[i]["text"]; got != w.text {
+ if got := res.JSON[i].ToMap()["text"]; got != w.text {
t.Errorf("JSON[%d].text = %v, want %v", i, got, w.text)
}
- if got := res.JSON[i]["doc_type_kwd"]; got != "text" {
+ if got := res.JSON[i].ToMap()["doc_type_kwd"]; got != "text" {
t.Errorf("JSON[%d].doc_type_kwd = %v, want text", i, got)
}
- if got := res.JSON[i]["ck_type"]; got != w.ckType {
+ if got := res.JSON[i].ToMap()["ck_type"]; got != w.ckType {
t.Errorf("JSON[%d].ck_type = %v, want %v", i, got, w.ckType)
}
}
@@ -235,7 +235,7 @@ Tail text`)
t.Fatalf("JSON len = %d, want %d: %#v", len(res.JSON), len(want), res.JSON)
}
for i, text := range want {
- if got := res.JSON[i]["text"]; got != text {
+ if got := res.JSON[i].ToMap()["text"]; got != text {
t.Errorf("JSON[%d].text = %v, want %v", i, got, text)
}
}
@@ -273,7 +273,7 @@ func TestHTMLParser_ParseWithResult_SkipsScriptAndStyle(t *testing.T) {
if len(res.JSON) != 3 {
t.Errorf("JSON len = %d, want 3 (script+style+noscript skipped)", len(res.JSON))
}
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
if txt, _ := it["text"].(string); strings.Contains(txt, "alert") ||
strings.Contains(txt, "color") ||
strings.Contains(txt, "inline alert") ||
@@ -316,7 +316,7 @@ func TestTextParser_ParseWithResult_NewlineNormalization(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
out := make([]string, 0, len(res.JSON))
- for _, it := range res.JSON {
+ for _, it := range ItemsToMaps(res.JSON) {
if txt, _ := it["text"].(string); txt != "" {
out = append(out, txt)
}
@@ -382,7 +382,7 @@ func TestTextParser_AlignmentGolden(t *testing.T) {
gd := LoadGoldenDoc(t, tc.golden)
ignore := AcceptedDivergences(gd.Meta)
- goText := FilterOutDocTypes(FilterByDocType(res.JSON, "text"), ignore)
+ goText := FilterOutDocTypes(FilterByDocType(ItemsToMaps(res.JSON), "text"), ignore)
pyText := FilterOutDocTypes(FilterByDocType(gd.Items, "text"), ignore)
delimiter, _ := gd.Meta["delimiter"].(string)
diff --git a/internal/parser/parser/pdf_parser_cgo_test.go b/internal/parser/parser/pdf_parser_cgo_test.go
index a9effa4822..b0e3b9a475 100644
--- a/internal/parser/parser/pdf_parser_cgo_test.go
+++ b/internal/parser/parser/pdf_parser_cgo_test.go
@@ -43,7 +43,7 @@ func TestPDFParser_ParseWithResult_CGOEmpty(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- if text, _ := res.JSON[0]["text"].(string); strings.TrimSpace(text) != "" {
+ if text, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(text) != "" {
t.Fatalf("JSON[0].text = %q, want empty", text)
}
}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 197d25c358..af5f7576f3 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -411,7 +411,7 @@ func emptyPDFResult(filename string) ParseResult {
"page_count": 0,
"outline": []map[string]any{},
},
- JSON: []map[string]any{{"text": "", "doc_type_kwd": "text"}},
+ JSON: []DocItem{NewTextJSONItem("")},
}
}
@@ -507,7 +507,7 @@ func pdfParseResultToJSONWithOptions(filename string, parsed *deepdoctype.ParseR
"page_count": len(processed.PageHeight),
"outline": outlinesToFileMeta(processed.Outlines),
},
- JSON: items,
+ JSON: MapsToItems(items),
}
}
@@ -1034,8 +1034,10 @@ func parsePDFWithDeepDocOptions(ctx context.Context, filename string, data []byt
return ParseResult{Err: fmt.Errorf("parser: unsupported PDF output_format %q", opts.outputFormat)}
}
for i := range res.JSON {
- if img, _ := res.JSON[i]["image"].(string); img != "" {
- res.JSON[i]["image"] = pdflayout.InlinePNGDataURL(img)
+ m := res.JSON[i].ToMap()
+ if img, _ := m["image"].(string); img != "" {
+ m["image"] = pdflayout.InlinePNGDataURL(img)
+ res.JSON[i] = DocItemFromMap(m)
}
}
return res
diff --git a/internal/parser/parser/pdf_parser_common_test.go b/internal/parser/parser/pdf_parser_common_test.go
index 584171bc68..64e43eb63c 100644
--- a/internal/parser/parser/pdf_parser_common_test.go
+++ b/internal/parser/parser/pdf_parser_common_test.go
@@ -64,39 +64,39 @@ func TestPDFParseResultToJSON_NormalizesCoreFields(t *testing.T) {
if len(res.JSON) != 2 {
t.Fatalf("JSON len = %d, want 2", len(res.JSON))
}
- if got, want := res.JSON[0]["layout"], "title"; got != want {
+ if got, want := res.JSON[0].ToMap()["layout"], "title"; got != want {
t.Fatalf("JSON[0].layout = %v, want %v", got, want)
}
- if got, want := res.JSON[0]["page_number"], 1; got != want {
+ if got, want := res.JSON[0].ToMap()["page_number"], 1; got != want {
t.Fatalf("JSON[0].page_number = %v, want %v", got, want)
}
- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {
+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {
t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)
}
- pdfPositions, ok := res.JSON[0]["_pdf_positions"].([][]any)
+ pdfPositions, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)
if !ok {
- t.Fatalf("JSON[0]._pdf_positions type = %T, want [][]any", res.JSON[0]["_pdf_positions"])
+ t.Fatalf("JSON[0]._pdf_positions type = %T, want [][]any", res.JSON[0].ToMap()["_pdf_positions"])
}
if len(pdfPositions) != 1 || pdfPositions[0][0] != 1 {
t.Fatalf("JSON[0]._pdf_positions = %+v, want canonical 1-based positions", pdfPositions)
}
- if got := res.JSON[0]["positions"]; got == nil {
+ if got := res.JSON[0].ToMap()["positions"]; got == nil {
t.Fatal("JSON[0].positions missing after normalization")
}
- if got, want := res.JSON[1]["doc_type_kwd"], "image"; got != want {
+ if got, want := res.JSON[1].ToMap()["doc_type_kwd"], "image"; got != want {
t.Fatalf("JSON[1].doc_type_kwd = %v, want %v", got, want)
}
- if got, want := res.JSON[1]["page_number"], 2; got != want {
+ if got, want := res.JSON[1].ToMap()["page_number"], 2; got != want {
t.Fatalf("JSON[1].page_number = %v, want %v", got, want)
}
- secondPDFPositions, ok := res.JSON[1]["_pdf_positions"].([][]any)
+ secondPDFPositions, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)
if !ok {
- t.Fatalf("JSON[1]._pdf_positions type = %T, want [][]any", res.JSON[1]["_pdf_positions"])
+ t.Fatalf("JSON[1]._pdf_positions type = %T, want [][]any", res.JSON[1].ToMap()["_pdf_positions"])
}
if len(secondPDFPositions) != 1 || secondPDFPositions[0][0] != 2 {
t.Fatalf("JSON[1]._pdf_positions = %+v, want canonical 1-based positions (DeepDoc page 1 → 2)", secondPDFPositions)
}
- if got, want := res.JSON[1]["image"], "data:image/png;base64,aGVsbG8="; got != want {
+ if got, want := res.JSON[1].ToMap()["image"], "data:image/png;base64,aGVsbG8="; got != want {
t.Fatalf("JSON[1].image = %v, want %v", got, want)
}
}
@@ -128,7 +128,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
if res.Err != nil {
t.Fatalf("pdfParseResultToJSON: %v", res.Err)
}
- if got := res.JSON[0]["doc_type_kwd"]; got != "image" {
+ if got := res.JSON[0].ToMap()["doc_type_kwd"]; got != "image" {
t.Fatalf("figure caption with positions doc_type_kwd = %v, want image", got)
}
@@ -148,7 +148,7 @@ func TestPDFParseResultToJSON_ClassifiesFigureCaptionWithImage(t *testing.T) {
if res2.Err != nil {
t.Fatalf("pdfParseResultToJSON: %v", res2.Err)
}
- if got := res2.JSON[0]["doc_type_kwd"]; got != "text" {
+ if got := res2.JSON[0].ToMap()["doc_type_kwd"]; got != "text" {
t.Fatalf("figure caption with image but no positions doc_type_kwd = %v, want text", got)
}
}
@@ -211,10 +211,10 @@ func TestPDFParseResultToJSON_PreservesPositivePageNumbers(t *testing.T) {
res := pdfParseResultToJSON("one-based.pdf", parsed)
// DeepDoc page 3 is 0-indexed (the 4th page); normalizePDFPageNumber
// converts it to 1-indexed page 4.
- if got, want := res.JSON[0]["page_number"], 4; got != want {
+ if got, want := res.JSON[0].ToMap()["page_number"], 4; got != want {
t.Fatalf("JSON[0].page_number = %v, want %v", got, want)
}
- if got, want := res.JSON[0]["doc_type_kwd"], "table"; got != want {
+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "table"; got != want {
t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)
}
}
@@ -227,7 +227,7 @@ func TestPDFParseResultToJSON_EmptySectionsStillEmitPlaceholder(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON len = %d, want 1", len(res.JSON))
}
- if got, want := res.JSON[0]["doc_type_kwd"], "text"; got != want {
+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "text"; got != want {
t.Fatalf("JSON[0].doc_type_kwd = %v, want %v", got, want)
}
}
@@ -261,13 +261,13 @@ func TestPDFParseResultToJSON_DefaultKeepsHeaderFooterLikePython(t *testing.T) {
// Sections are now sorted by (page, top, left). Header and Footer have
// no position data (page=0, top=0), Body has top=30, so the sorted order
// is Header/Footer (tied top=0, stable) then Body (top=30).
- if got, want := res.JSON[0]["text"], "Header"; got != want {
+ if got, want := res.JSON[0].ToMap()["text"], "Header"; got != want {
t.Fatalf("JSON[0].text = %v, want %v", got, want)
}
- if got, want := res.JSON[1]["text"], "Footer"; got != want {
+ if got, want := res.JSON[1].ToMap()["text"], "Footer"; got != want {
t.Fatalf("JSON[1].text = %v, want %v", got, want)
}
- if got, want := res.JSON[2]["text"], "Body"; got != want {
+ if got, want := res.JSON[2].ToMap()["text"], "Body"; got != want {
t.Fatalf("JSON[2].text = %v, want %v", got, want)
}
}
@@ -671,25 +671,25 @@ func TestPDFParseResultToJSON_NoInlineRetainsPositions(t *testing.T) {
}
// Figure must retain positions but NOT inline an image.
- figPos, ok := res.JSON[0]["_pdf_positions"].([][]any)
+ figPos, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)
if !ok || len(figPos) == 0 {
- t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0]["_pdf_positions"])
+ t.Fatalf("Figure should retain _pdf_positions, got %v", res.JSON[0].ToMap()["_pdf_positions"])
}
- if img, _ := res.JSON[0]["image"].(string); img != "" {
+ if img, _ := res.JSON[0].ToMap()["image"].(string); img != "" {
t.Fatalf("Figure should not inline image under cgo, got %q", img)
}
// Table likewise retains positions, no inline image.
- tblPos, ok := res.JSON[1]["_pdf_positions"].([][]any)
+ tblPos, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)
if !ok || len(tblPos) == 0 {
- t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1]["_pdf_positions"])
+ t.Fatalf("Table should retain _pdf_positions, got %v", res.JSON[1].ToMap()["_pdf_positions"])
}
- if img, _ := res.JSON[1]["image"].(string); img != "" {
+ if img, _ := res.JSON[1].ToMap()["image"].(string); img != "" {
t.Fatalf("Table should not inline image under cgo, got %q", img)
}
// Plain text: no image, positions retained.
- if img, _ := res.JSON[2]["image"].(string); img != "" {
+ if img, _ := res.JSON[2].ToMap()["image"].(string); img != "" {
t.Fatalf("Text should not inline image, got %q", img)
}
}
@@ -719,13 +719,13 @@ func TestPDFParseResultToJSON_FigureCaptionNoInline(t *testing.T) {
if res.Err != nil {
t.Fatalf("pdfParseResultToJSON: %v", res.Err)
}
- if got, want := res.JSON[0]["doc_type_kwd"], "image"; got != want {
+ if got, want := res.JSON[0].ToMap()["doc_type_kwd"], "image"; got != want {
t.Fatalf("doc_type_kwd = %v, want %v", got, want)
}
- if pos, ok := res.JSON[0]["_pdf_positions"].([][]any); !ok || len(pos) == 0 {
+ if pos, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any); !ok || len(pos) == 0 {
t.Fatal("figure caption should retain _pdf_positions for on-demand crop")
}
- if image, _ := res.JSON[0]["image"].(string); image != "" {
+ if image, _ := res.JSON[0].ToMap()["image"].(string); image != "" {
t.Fatalf("figure caption should not be inlined by the parser, got %q", image)
}
}
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index ac49389613..b08022df2f 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -261,7 +261,7 @@ func doclingTextsToResult(filename string, texts []string, outputFormat string,
return ParseResult{
OutputFormat: "json",
File: fileMeta,
- JSON: items,
+ JSON: MapsToItems(items),
}
case "markdown":
return ParseResult{
diff --git a/internal/parser/parser/pdf_parser_fixture_cgo_test.go b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
index 2de37ad80b..83219849a5 100644
--- a/internal/parser/parser/pdf_parser_fixture_cgo_test.go
+++ b/internal/parser/parser/pdf_parser_fixture_cgo_test.go
@@ -50,7 +50,7 @@ func TestPDFParser_ParseWithResult_CGOFixture(t *testing.T) {
if got := res.File["page_count"]; got == nil {
t.Fatal("File.page_count missing")
}
- if positions, ok := res.JSON[0]["_pdf_positions"].([][]any); ok && len(positions) == 0 {
+ if positions, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any); ok && len(positions) == 0 {
t.Fatal("JSON[0]._pdf_positions is empty; want normalized positions for fixture text")
}
}
@@ -103,7 +103,7 @@ func TestPDFParser_ParseWithResult_CGOFixturePlainText(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want page text items")
}
- if got, _ := res.JSON[0]["text"].(string); strings.TrimSpace(got) == "" {
+ if got, _ := res.JSON[0].ToMap()["text"].(string); strings.TrimSpace(got) == "" {
t.Fatal("plain_text first page is empty")
}
}
diff --git a/internal/parser/parser/pdf_parser_mineru_test.go b/internal/parser/parser/pdf_parser_mineru_test.go
index 87304f7090..adaefaf5a6 100644
--- a/internal/parser/parser/pdf_parser_mineru_test.go
+++ b/internal/parser/parser/pdf_parser_mineru_test.go
@@ -135,7 +135,7 @@ func TestPDFParser_ParseWithResult_MinerUJSONIntegration(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want markdown-normalized items")
}
- if got := res.JSON[0]["text"]; got == nil {
+ if got := res.JSON[0].ToMap()["text"]; got == nil {
t.Fatal("JSON[0].text missing")
}
}
diff --git a/internal/parser/parser/pdf_parser_paddleocr_test.go b/internal/parser/parser/pdf_parser_paddleocr_test.go
index 2bdfee283c..172c1d7c53 100644
--- a/internal/parser/parser/pdf_parser_paddleocr_test.go
+++ b/internal/parser/parser/pdf_parser_paddleocr_test.go
@@ -125,7 +125,7 @@ func TestPDFParser_ParseWithResult_PaddleOCRJSONIntegration(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want markdown-normalized items")
}
- if got := res.JSON[0]["text"]; got == nil {
+ if got := res.JSON[0].ToMap()["text"]; got == nil {
t.Fatal("JSON[0].text missing")
}
}
diff --git a/internal/parser/parser/pdf_parser_pages_test.go b/internal/parser/parser/pdf_parser_pages_test.go
index 88316ae0fb..d12b0e01ef 100644
--- a/internal/parser/parser/pdf_parser_pages_test.go
+++ b/internal/parser/parser/pdf_parser_pages_test.go
@@ -101,7 +101,7 @@ func TestPDFParser_PlainTextHonorsPages(t *testing.T) {
t.Fatalf("ParseWithResult: %v", res.Err)
}
var pageNumbers []any
- for _, item := range res.JSON {
+ for _, item := range ItemsToMaps(res.JSON) {
pageNumbers = append(pageNumbers, item["page_number"])
}
if want := []any{2, 3}; !reflect.DeepEqual(pageNumbers, want) {
diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.go
index a26af71a1d..57d6df306f 100644
--- a/internal/parser/parser/pdf_parser_remote_common.go
+++ b/internal/parser/parser/pdf_parser_remote_common.go
@@ -46,7 +46,7 @@ func pdfItemsToResult(filename string, items []map[string]any, outputFormat stri
return ParseResult{
OutputFormat: "json",
File: pdfFileMeta(filename, pageCount),
- JSON: items,
+ JSON: MapsToItems(items),
}
case "markdown":
var b strings.Builder
diff --git a/internal/parser/parser/picture_parser.go b/internal/parser/parser/picture_parser.go
index cb97a28118..c5f7bc1e0a 100644
--- a/internal/parser/parser/picture_parser.go
+++ b/internal/parser/parser/picture_parser.go
@@ -114,7 +114,7 @@ func (p *PictureParser) ParseWithResult(ctx context.Context, filename string, da
"size": len(data),
"doc_type_kwd": DocTypeImage,
},
- JSON: []map[string]any{item},
+ JSON: []DocItem{DocItemFromMap(item)},
}
}
diff --git a/internal/parser/parser/picture_parser_test.go b/internal/parser/parser/picture_parser_test.go
index 6190b410ea..5dc809790c 100644
--- a/internal/parser/parser/picture_parser_test.go
+++ b/internal/parser/parser/picture_parser_test.go
@@ -28,12 +28,12 @@ func TestPictureParserKeepsImageWithoutOCR(t *testing.T) {
if res.Err != nil {
t.Fatal(res.Err)
}
- if len(res.JSON) != 1 || res.JSON[0]["text"] != "" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "" {
t.Fatalf("picture text = %+v, want empty text", res.JSON)
}
wantImage := "data:image/png;base64," + base64.StdEncoding.EncodeToString(data)
- if res.JSON[0]["image"] != wantImage {
- t.Fatalf("picture image = %v, want %q", res.JSON[0]["image"], wantImage)
+ if res.JSON[0].ToMap()["image"] != wantImage {
+ t.Fatalf("picture image = %v, want %q", res.JSON[0].ToMap()["image"], wantImage)
}
if len(res.Warnings) != 0 {
t.Fatalf("picture warnings = %v, want none", res.Warnings)
diff --git a/internal/parser/parser/ppt_parser_cgo_test.go b/internal/parser/parser/ppt_parser_cgo_test.go
index 010761deba..5aad089300 100644
--- a/internal/parser/parser/ppt_parser_cgo_test.go
+++ b/internal/parser/parser/ppt_parser_cgo_test.go
@@ -85,7 +85,7 @@ func TestPPTXParser_ParseWithResult_MultiSlide_CGO(t *testing.T) {
{"Slide One Title\nFirst slide body about the Alpha topic.", 1},
{"Slide Two Title\nSecond slide body about the Beta topic.", 2},
} {
- it := res.JSON[i]
+ it := res.JSON[i].ToMap()
if got := it["text"]; got != want.text {
t.Errorf("item %d text = %q, want %q", i, got, want.text)
}
@@ -124,10 +124,10 @@ func TestPPTXParser_ParseWithResult_EmptySlide_CGO(t *testing.T) {
if len(res.JSON) != 3 {
t.Fatalf("JSON items = %d, want 3 (one per slide): %#v", len(res.JSON), res.JSON)
}
- if got := res.JSON[1]["text"]; got != "" {
+ if got := res.JSON[1].ToMap()["text"]; got != "" {
t.Errorf("empty slide text = %q, want empty", got)
}
- if got := res.JSON[1]["slide_number"]; got != 2 {
+ if got := res.JSON[1].ToMap()["slide_number"]; got != 2 {
t.Errorf("empty slide slide_number = %v, want 2", got)
}
}
diff --git a/internal/parser/parser/pptx_parser.go b/internal/parser/parser/pptx_parser.go
index 5a2420df7c..61cc0a4a77 100644
--- a/internal/parser/parser/pptx_parser.go
+++ b/internal/parser/parser/pptx_parser.go
@@ -158,7 +158,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{
OutputFormat: "json",
File: map[string]any{"name": filename, "format": effFormat},
- JSON: itemsFromPlainText(plainText),
+ JSON: MapsToItems(itemsFromPlainText(plainText)),
}
}
budget := newEmbeddedMediaBudget()
@@ -184,7 +184,7 @@ func (p *PPTXParser) ParseWithResult(ctx context.Context, filename string, data
return ParseResult{
OutputFormat: "json",
File: map[string]any{"name": filename, "format": effFormat},
- JSON: items,
+ JSON: MapsToItems(items),
Warnings: budget.warnings(),
}
}
diff --git a/internal/parser/parser/text_parser.go b/internal/parser/parser/text_parser.go
index 1c1efd33b0..e1f00eff0e 100644
--- a/internal/parser/parser/text_parser.go
+++ b/internal/parser/parser/text_parser.go
@@ -53,7 +53,7 @@ func (p *TextParser) ParseWithResult(ctx context.Context, filename string, data
"size": len(data),
"encoding": encName,
},
- JSON: []map[string]any{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},
+ JSON: []DocItem{NewTextJSONItem(normalizeTextNewlines(string(decoded)))},
}
}
diff --git a/internal/parser/parser/xls_parser.go b/internal/parser/parser/xls_parser.go
index dfaf702cac..d22e1758c6 100644
--- a/internal/parser/parser/xls_parser.go
+++ b/internal/parser/parser/xls_parser.go
@@ -115,7 +115,7 @@ func (p *XLSParser) ParseWithResult(ctx context.Context, filename string, data [
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
File: map[string]any{"name": filename, "format": "xls", "sheets": sheetsCount},
- JSON: items,
+ JSON: MapsToItems(items),
Warnings: warnings,
}
}
diff --git a/internal/parser/parser/xls_parser_test.go b/internal/parser/parser/xls_parser_test.go
index 98b7b4ee29..95224ff5b2 100644
--- a/internal/parser/parser/xls_parser_test.go
+++ b/internal/parser/parser/xls_parser_test.go
@@ -140,7 +140,7 @@ func TestXLSParser_PreservesFormulaErrors(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("res.JSON len = %d, want 1", len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, "<tr><td>#DIV/0!</td></tr>") {
t.Fatalf("table markup = %q, want formula error row", text)
}
@@ -173,7 +173,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
t.Fatalf("res.JSON len = %d, want one table for each sheet", len(res.JSON))
}
- first := res.JSON[0]
+ first := res.JSON[0].ToMap()
text := tableItemHTML(t, first)
if !strings.Contains(text, "<caption>Inventory</caption>") ||
!strings.Contains(text, "<tr><th>Name</th><th>Count</th></tr>") ||
@@ -194,7 +194,7 @@ func TestXLSParser_GenuineBIFF8SpreadsheetJSONOutput(t *testing.T) {
t.Fatalf("last row position = %v, want %v", got, want)
}
- second := res.JSON[1]
+ second := res.JSON[1].ToMap()
secondText := tableItemHTML(t, second)
if !strings.Contains(secondText, "<caption>Notes</caption>") ||
!strings.Contains(secondText, "<tr><td>Legacy</td><td>BIFF8</td></tr>") {
@@ -224,7 +224,7 @@ func TestXLSParser_SpreadsheetJSONOutput(t *testing.T) {
if len(res.JSON) == 0 {
t.Fatalf("res.JSON should have structured table items")
}
- item := res.JSON[0]
+ item := res.JSON[0].ToMap()
if item["ck_type"] != "table" {
t.Errorf("table item ck_type = %v, want table", item["ck_type"])
}
@@ -271,8 +271,8 @@ func TestXLSXParser_JSONOutput(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("res.JSON = %#v, want one segmented HTML table item", res.JSON)
}
- if res.JSON[0]["ck_type"] != "table" {
- t.Errorf("item ck_type = %v, want table", res.JSON[0]["ck_type"])
+ if res.JSON[0].ToMap()["ck_type"] != "table" {
+ t.Errorf("item ck_type = %v, want table", res.JSON[0].ToMap()["ck_type"])
}
// Even if configured to legacy "html", format is unified to "json"
@@ -326,7 +326,7 @@ func TestXLSParser_ImagesDoNotRunOCR(t *testing.T) {
t.Fatalf("ParseWithResult: %v", result.Err)
}
var imageItem map[string]any
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] == "image" {
imageItem = item
break
@@ -375,7 +375,7 @@ func TestXLSParser_CapsEmbeddedImageCount(t *testing.T) {
t.Fatalf("ParseWithResult: %v", result.Err)
}
imageCount := 0
- for _, item := range result.JSON {
+ for _, item := range ItemsToMaps(result.JSON) {
if item["doc_type_kwd"] == "image" {
imageCount++
}
diff --git a/internal/parser/parser/xls_tcadp_test.go b/internal/parser/parser/xls_tcadp_test.go
index f623c3ce98..7768be6cc6 100644
--- a/internal/parser/parser/xls_tcadp_test.go
+++ b/internal/parser/parser/xls_tcadp_test.go
@@ -228,7 +228,7 @@ func TestCSVParser_ParseWithResult_DefaultCSVBehavior(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table" {
+ if res.JSON[0].ToMap()["ck_type"] != "table" {
t.Fatalf("JSON item = %#v; want a table item", res.JSON[0])
}
}
diff --git a/internal/parser/parser/xlsx_parse_method_test.go b/internal/parser/parser/xlsx_parse_method_test.go
index e07b2875d7..43ea5bf7a4 100644
--- a/internal/parser/parser/xlsx_parse_method_test.go
+++ b/internal/parser/parser/xlsx_parse_method_test.go
@@ -80,7 +80,7 @@ func TestXLSXParser_DeepDocParseMethod(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON item count = %d, want header-only result", len(res.JSON))
}
- text := tableItemHTML(t, res.JSON[0])
+ text := tableItemHTML(t, res.JSON[0].ToMap())
if !strings.Contains(text, tc.cellValue) {
t.Fatalf("JSON = %#v, want it to contain cell content %q", res.JSON, tc.cellValue)
}
@@ -113,7 +113,7 @@ func TestXLSXParser_ExtractsFloatingImages(t *testing.T) {
if len(res.JSON) != 2 {
t.Fatalf("JSON item count = %d, want table and image", len(res.JSON))
}
- image := res.JSON[1]
+ image := res.JSON[1].ToMap()
if image["text"] != "sheet image" || image["doc_type_kwd"] != "image" {
t.Fatalf("unexpected image item: %#v", image)
}
@@ -145,7 +145,7 @@ func TestXLSXParser_ImageWithoutAltTextUsesAnchorCell(t *testing.T) {
if res.Err != nil {
t.Fatalf("ParseWithResult: %v", res.Err)
}
- if len(res.JSON) != 1 || res.JSON[0]["text"] != "C3" {
+ if len(res.JSON) != 1 || res.JSON[0].ToMap()["text"] != "C3" {
t.Fatalf("image item = %#v, want anchor-cell text C3", res.JSON)
}
}
@@ -177,7 +177,7 @@ func TestCSVParser_DeepDocParseMethod(t *testing.T) {
if len(res.JSON) != 1 {
t.Fatalf("JSON items = %d, want one HTML table item", len(res.JSON))
}
- if res.JSON[0]["ck_type"] != "table" {
+ if res.JSON[0].ToMap()["ck_type"] != "table" {
t.Fatalf("JSON item = %#v, want a table item", res.JSON[0])
}
}
diff --git a/internal/parser/parser/xlsx_parser.go b/internal/parser/parser/xlsx_parser.go
index e8cc2ee7ed..b0dc873283 100644
--- a/internal/parser/parser/xlsx_parser.go
+++ b/internal/parser/parser/xlsx_parser.go
@@ -177,7 +177,7 @@ func xlsxParseResult(filename string, items []map[string]any, warnings []string,
return ParseResult{
OutputFormat: spreadsheetOutputFormat,
File: map[string]any{"name": filename, "format": "xlsx", "sheets": sheets},
- JSON: items,
+ JSON: MapsToItems(items),
Warnings: warnings,
}
}
diff --git a/internal/service/file/file_content.go b/internal/service/file/file_content.go
index 60cde67bd4..4a702252cd 100644
--- a/internal/service/file/file_content.go
+++ b/internal/service/file/file_content.go
@@ -235,15 +235,16 @@ func parseResultText(res parser.ParseResult) (string, error) {
if len(res.JSON) > 0 {
parts := make([]string, 0, len(res.JSON))
for _, item := range res.JSON {
- if text, ok := item["text"].(string); ok {
+ m := item.ToMap()
+ if text, ok := m["text"].(string); ok {
parts = append(parts, text)
continue
}
- if td, ok := item["table"].(*entity.TableData); ok && td != nil {
+ if td, ok := m["table"].(*entity.TableData); ok && td != nil {
parts = append(parts, tableutil.RenderTableHTML(td))
continue
}
- raw, err := json.Marshal(item)
+ raw, err := json.Marshal(m)
if err != nil {
return "", err
}
diff --git a/internal/service/file/file_content_test.go b/internal/service/file/file_content_test.go
index b0421c0681..b50ea119e9 100644
--- a/internal/service/file/file_content_test.go
+++ b/internal/service/file/file_content_test.go
@@ -60,11 +60,11 @@ func TestParseFileContent_JSONItems(t *testing.T) {
func TestParseResultText_JSONItemsPreservesOrderAndFallsBackToJSON(t *testing.T) {
result, err := parseResultText(parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": "first"},
{"text": "second"},
{"value": 3},
- },
+ }),
})
if err != nil {
t.Fatalf("parseResultText: %v", err)
@@ -82,9 +82,9 @@ func TestParseResultText_SpreadsheetSegmentsAreTheirOwnMarkup(t *testing.T) {
segment := "<table><caption>Data</caption>\n<tr><th>a</th><th>b</th></tr>\n<tr><td>1</td><td>2</td></tr>\n</table>\n"
result, err := parseResultText(parser.ParseResult{
OutputFormat: "json",
- JSON: []map[string]any{
+ JSON: parser.MapsToItems([]map[string]any{
{"text": segment, "doc_type_kwd": "table", "ck_type": "table", "sheet": "Data", "sheet_index": 1},
- },
+ }),
})
if err != nil {
t.Fatalf("parseResultText: %v", err)
diff --git a/tools/mdprobe/main.go b/tools/mdprobe/main.go
index 85d519411f..c9e12088f3 100644
--- a/tools/mdprobe/main.go
+++ b/tools/mdprobe/main.go
@@ -47,10 +47,11 @@ func main() {
fmt.Printf("file=%s bytes=%d output_format=%s items=%d\n", filepath.Base(path), len(data), res.OutputFormat, len(res.JSON))
var total int
for i, item := range res.JSON {
- text, _ := item["text"].(string)
+ m := item.ToMap()
+ text, _ := m["text"].(string)
total += len(text)
- ck := fmt.Sprint(item["ck_type"])
- docType := fmt.Sprint(item["doc_type_kwd"])
+ ck := fmt.Sprint(m["ck_type"])
+ docType := fmt.Sprint(m["doc_type_kwd"])
fmt.Printf(" [%3d] %-7s %-6s %6d chars %q\n", i, ck, docType, len(text), truncate(text, 90))
}
fmt.Printf("parser emitted %d chars total (file %d)\n", total, len(data))
@@ -59,7 +60,7 @@ func main() {
lower := strings.ToLower(needle)
found := -1
for i, item := range res.JSON {
- if text, _ := item["text"].(string); strings.Contains(strings.ToLower(text), lower) {
+ if text, _ := item.ToMap()["text"].(string); strings.Contains(strings.ToLower(text), lower) {
found = i
break
}