Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(parser): introduce typed Position API for DocItem positions (#21049)

xugangqiangb073eaabc08b
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.go
index c9fcc66968..fafa593fdb 100644
--- a/internal/ingestion/component/parser_dispatch.go
+++ b/internal/ingestion/component/parser_dispatch.go
@@ -348,7 +348,8 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name
// normalizeParserJSON returns the parser output as the typed []DocItem. It no
// longer round-trips through map[string]any (the historical ItemsToMaps call),
// so the downstream chunker receives structured items and converts them via
-// schema.ChunkDocsFromDocItem without ever calling DocItem.ToMap.
+// schema.ChunkDocsFromDocItem, which projects each item through its canonical
+// DocItem.ToMap wire shape before decoding it into a ChunkDoc.
func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []parser.DocItem {
if len(dispatched.JSON) > 0 {
return dispatched.JSON
diff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.go
index 42d8b5e27a..fcb37dded4 100644
--- a/internal/ingestion/component/schema/chunk_docitem_test.go
+++ b/internal/ingestion/component/schema/chunk_docitem_test.go
@@ -52,7 +52,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {
{
name: "table with sheet and positions",
items: []parser.DocItem{
- parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: [][]float64{{1, 2}}, Extra: map[string]any{"sheet_index": 0}},
+ parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: parser.SpreadsheetPositions([][]float64{{1, 2, 2, 1, 3}}), Extra: map[string]any{"sheet_index": 0}},
},
},
{
@@ -210,7 +210,7 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
- got, found, err := ChunkDocsFromDocItem(parser.DocItemsFromMaps(tc.maps))
+ got, found, err := ChunkDocsFromDocItem(parser.MapsToItems(tc.maps))
if err != nil {
t.Fatalf("ChunkDocsFromDocItem err: %v", err)
}
diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.go
index 8ca5ae09f7..b5a66e122c 100644
--- a/internal/ingestion/component/schema/chunk_types.go
+++ b/internal/ingestion/component/schema/chunk_types.go
@@ -242,10 +242,11 @@ func ChunkDocsFromAny(in any) ([]ChunkDoc, bool, error) {
}
// ChunkDocsFromDocItem converts a parser-produced []DocItem directly into a
-// typed chunk slice, without routing through DocItem.ToMap. It reads the
-// typed fields of each item and feeds the canonical wire map to ChunkDocFromMap,
-// so the resulting []ChunkDoc is identical to what the historical
-// ChunkDocsFromAny([]DocItem) path produced (which round-tripped via ToMap).
+// typed chunk slice by projecting each item through its canonical DocItem.ToMap
+// wire shape and decoding the result via ChunkDocFromMap (see chunkDocFromDocItem).
+// The resulting []ChunkDoc is therefore identical to what the historical
+// ChunkDocsFromAny([]DocItem) path produced, which round-tripped through the same
+// ToMap projection.
// This is the consumer-side entry the chunker uses once parsers emit DocItems
// directly; other callers that still pass []map[string]any / []any / []ChunkDoc
// keep using ChunkDocsFromAny.
diff --git a/internal/ingestion/component/schema/file.go b/internal/ingestion/component/schema/file.go
index e64a7cb7ac..40e11c84d7 100644
--- a/internal/ingestion/component/schema/file.go
+++ b/internal/ingestion/component/schema/file.go
@@ -19,9 +19,10 @@
// Pydantic schema in rag/flow/<component>/schema.py (or, when none exists,
// the runtime contract implied by the component's _invoke signature).
//
-// Files in this package are pure data definitions — they import only stdlib.
-// Business logic (the component Implementations) lives elsewhere in
-// internal/ingestion/component/.
+// Files in this package are pure data definitions. They depend on
+// ragflow/internal/common, ragflow/internal/entity, and
+// ragflow/internal/parser/parser for shared types, but hold no business logic;
+// the component Implementations live elsewhere in internal/ingestion/component/.
package schema
// FileFromUpstream is the upstream payload consumed by the File component.
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index 97ea6d8ca8..ddedcb0bf2 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -6,6 +6,9 @@ import (
"time"
"ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/parser/parser"
)
func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {
@@ -597,6 +600,87 @@ func TestProcessChunkPositions_SpreadsheetKeepsRowIndex(t *testing.T) {
}
}
+// TestProcessChunkPositions_ParserSpreadsheetWire guards the PARSER wire shape:
+// a spreadsheet DocItem must project positions as [][]float64 (not [][]any), and
+// processChunkPositions must accept that shape. This feeds the parser wire
+// directly (the direct-json path in NormalizeChunks that no built-in template
+// exercises); the production path that flows through the chunker is covered by
+// TestProcessChunkPositions_ChunkerRoundTrip below.
+func TestProcessChunkPositions_ParserSpreadsheetWire(t *testing.T) {
+ td := &entity.TableData{}
+ item := parser.NewTableDataJSONItem(td, "S", parser.SpreadsheetPositions([][]float64{{1, 1, 1, 1, 3}})).ToMap()
+ raw, ok := item["positions"].([][]float64)
+ if !ok {
+ t.Fatalf("parser spreadsheet wire = %T, want [][]float64", item["positions"])
+ }
+ chunk := map[string]any{
+ "positions": raw,
+ "top_int": []int{41},
+ }
+ processChunkPositions(chunk, true)
+
+ matrix, ok := chunk["position_int"].([][]int)
+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][4] != 3 {
+ t.Fatalf("position_int = %v, want the sheet tuple [1 1 1 1 3]", chunk["position_int"])
+ }
+ if _, exists := chunk["positions"]; exists {
+ t.Error("raw positions must be removed")
+ }
+}
+
+// TestProcessChunkPositions_ChunkerRoundTrip exercises the PRODUCTION path the
+// ingestion pipeline actually uses: parser DocItems cross the chunker boundary
+// as ChunkDoc (positions stored as json.RawMessage), and ChunkDoc.ToMap
+// re-renders them through decodeStructuredValue, which unmarshals into
+// [][]float64 for any all-numeric array. So by the time processChunkPositions
+// runs, spreadsheet positions are [][]float64 and reach position_int — and the
+// parser's [][]any (PDF) is likewise normalized to [][]float64 here. This is why
+// the parser's per-kind container type is irrelevant to indexing: the chunker
+// erases it. Regression guard for the positions restructuring's real seam.
+func TestProcessChunkPositions_ChunkerRoundTrip(t *testing.T) {
+ // Spreadsheet: RowColumnBox -> [][]float64 on the parser wire.
+ ssItem := parser.NewTableDataJSONItem(&entity.TableData{}, "S",
+ parser.SpreadsheetPositions([][]float64{{1, 1, 1, 1, 3}}))
+ docs, found, err := schema.ChunkDocsFromDocItem([]parser.DocItem{ssItem})
+ if err != nil || !found || len(docs) != 1 {
+ t.Fatalf("ChunkDocsFromDocItem(spreadsheet): found=%v err=%v", found, err)
+ }
+ m := docs[0].ToMap()
+ raw, ok := m["positions"].([][]float64)
+ if !ok {
+ t.Fatalf("chunker-rendered positions = %T, want [][]float64 (decodeStructuredValue)", m["positions"])
+ }
+ chunk := map[string]any{"positions": raw, "top_int": []int{41}, "sheet_index": 1}
+ processChunkPositions(chunk, true)
+ matrix, ok := chunk["position_int"].([][]int)
+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][4] != 3 {
+ t.Fatalf("position_int = %v, want [[1 1 1 1 3]]", chunk["position_int"])
+ }
+ if _, exists := chunk["positions"]; exists {
+ t.Error("raw positions must be removed")
+ }
+
+ // PDF: AbsoluteBox -> [][]any on the parser wire, but decodeStructuredValue
+ // normalizes it to [][]float64 at the chunker boundary too, so the parser
+ // container type never reaches the indexer.
+ pdfItem := parser.ImageItem{
+ Text: "fig",
+ Image: "",
+ CKType: "image",
+ Positions: parser.Positions{
+ parser.AbsoluteBox{PageNumber: 1, Left: 100, Right: 50, Top: 200, Bottom: 150},
+ },
+ }
+ pdfDocs, _, err := schema.ChunkDocsFromDocItem([]parser.DocItem{pdfItem})
+ if err != nil || len(pdfDocs) != 1 {
+ t.Fatalf("ChunkDocsFromDocItem(pdf): err=%v", err)
+ }
+ pm := pdfDocs[0].ToMap()
+ if _, ok := pm["positions"].([][]float64); !ok {
+ t.Fatalf("PDF positions after chunker = %T, want [][]float64 (normalized by decodeStructuredValue)", pm["positions"])
+ }
+}
+
// TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex pins the whole
// chunk loop: the spreadsheet identity has to be read before
// stripPipelineOnlyFields removes sheet_index, or the positions of a
diff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.go
index 3eb9b23d75..9e26ebe55e 100644
--- a/internal/parser/parser/doc_item.go
+++ b/internal/parser/parser/doc_item.go
@@ -34,8 +34,6 @@ import (
// historical map shape. A later pass may promote the well-known Extra keys to
// named fields; that is out of scope for the boundary-adapter change.
type DocItem interface {
- // DocType returns the item's doc_type_kwd discriminator (text/table/image).
- DocType() string
// ToMap projects the item onto the historical wire shape. Callers must not
// mutate the returned map's shared Extra contents.
ToMap() map[string]any
@@ -50,8 +48,6 @@ type TextItem struct {
Extra map[string]any
}
-func (t TextItem) DocType() string { return DocTypeText }
-
func (t TextItem) ToMap() map[string]any {
m := map[string]any{
"text": t.Text,
@@ -69,13 +65,11 @@ func (t TextItem) ToMap() map[string]any {
type TableItem struct {
Table *entity.TableData
Sheet string
- Positions [][]float64
+ Positions Positions
CKType string
Extra map[string]any
}
-func (t TableItem) DocType() string { return DocTypeTable }
-
func (t TableItem) ToMap() map[string]any {
m := map[string]any{
"table": t.Table,
@@ -86,7 +80,7 @@ func (t TableItem) ToMap() map[string]any {
m["sheet"] = t.Sheet
}
if len(t.Positions) > 0 {
- m["positions"] = t.Positions
+ m["positions"] = positionsToWire(t.Positions)
}
return mergeExtra(m, t.Extra)
}
@@ -99,13 +93,11 @@ type ImageItem struct {
Text string
Image string
Sheet string
- Positions [][]float64
+ Positions Positions
CKType string
Extra map[string]any
}
-func (i ImageItem) DocType() string { return DocTypeImage }
-
func (i ImageItem) ToMap() map[string]any {
m := map[string]any{
"text": i.Text,
@@ -119,7 +111,7 @@ func (i ImageItem) ToMap() map[string]any {
m["sheet"] = i.Sheet
}
if len(i.Positions) > 0 {
- m["positions"] = i.Positions
+ m["positions"] = positionsToWire(i.Positions)
}
return mergeExtra(m, i.Extra)
}
@@ -127,14 +119,11 @@ func (i ImageItem) ToMap() map[string]any {
// RawItem is a passthrough for items the parser constructs as an opaque map
// (the email content item, or any parser still emitting map literals during
// the migration). ToMap returns the map unchanged, so the wire shape is
-// preserved exactly. DocType is empty because the discriminator lives in the
-// map under DocTypeKey.
+// preserved exactly. The discriminator lives in the map under DocTypeKey.
type RawItem struct {
M map[string]any
}
-func (RawItem) DocType() string { return "" }
-
func (r RawItem) ToMap() map[string]any { return r.M }
// MarshalJSON projects each item onto its wire shape so that downstream
@@ -146,16 +135,6 @@ func (t TableItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()
func (i ImageItem) MarshalJSON() ([]byte, error) { return json.Marshal(i.ToMap()) }
func (r RawItem) MarshalJSON() ([]byte, error) { return json.Marshal(r.M) }
-// FromMap wraps an opaque payload map as a DocItem. Used by boundary readers
-// that receive a []map[string]any and must hand it to DocItem-typed APIs.
-func FromMap(m map[string]any) DocItem { return RawItem{M: m} }
-
-// Raw wraps an opaque payload map as a DocItem. Parsers that still build items
-// as map literals use it at the assignment site (e.g. `JSON: MapsToItems(items)`
-// or `res.JSON = append(res.JSON, Raw(m))`), so their internal logic stays
-// unchanged while the field gains the DocItem type.
-func Raw(m map[string]any) DocItem { return RawItem{M: m} }
-
// MapsToItems wraps a slice of payload maps as typed DocItems (one
// TextItem / TableItem / ImageItem / RawItem each, detected via doc_type_kwd).
// Parsers that accumulate results as []map[string]any use it at the assignment
@@ -164,7 +143,11 @@ func Raw(m map[string]any) DocItem { return RawItem{M: m} }
// ImageItem constructors become the emission path. Opaque maps without a known
// discriminator stay RawItem, preserving the wire shape exactly.
func MapsToItems(maps []map[string]any) []DocItem {
- return DocItemsFromMaps(maps)
+ out := make([]DocItem, 0, len(maps))
+ for _, m := range maps {
+ out = append(out, DocItemFromMap(m))
+ }
+ return out
}
// DocItemFromMap detects the item's doc_type_kwd and builds the matching typed
@@ -212,30 +195,46 @@ func textItemFromMap(m map[string]any) DocItem {
}
func imageItemFromMap(m map[string]any) DocItem {
+ pos := wireToPositions(m["positions"], hasSheetIndex(m))
+ skip := []string{"text", "image", DocTypeKey, "ck_type", "sheet"}
+ if pos != nil {
+ // Parsed into the typed field: it owns "positions", so strip it from
+ // Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions
+ // from normalizePDFPositions) parse here, so the old [][]any-only
+ // Extra fallback is gone.
+ skip = append(skip, "positions")
+ }
return ImageItem{
Text: stringField(m, "text"),
Image: stringField(m, "image"),
CKType: stringField(m, "ck_type"),
Sheet: stringField(m, "sheet"),
- Positions: positionsField(m),
- // "positions" is intentionally NOT stripped: when the map carries
- // positions as [][]any (e.g. PDF crop regions from normalizePDFPositions)
- // the typed field cannot capture them, and stripping would drop the key
- // from ToMap. Leaving it in Extra preserves the wire shape exactly.
- Extra: extraFromMap(m, "text", "image", DocTypeKey, "ck_type", "sheet"),
+ Positions: pos,
+ Extra: extraFromMap(m, skip...),
}
}
func tableItemFromMap(m map[string]any) DocItem {
- td, _ := m["table"].(*entity.TableData)
+ td, ok := m["table"].(*entity.TableData)
+ // A table-typed map whose "table" value is not a *entity.TableData (e.g. a
+ // JSON-decoded map[string]any) cannot be coerced into a TableItem without
+ // silently dropping the data — TableItem.ToMap would emit "table": null.
+ // Fall back to RawItem, preserving the map verbatim, exactly like the
+ // text/image branches do when their canonical field is absent.
+ if !ok {
+ return RawItem{M: m}
+ }
+ pos := wireToPositions(m["positions"], hasSheetIndex(m))
+ skip := []string{"table", DocTypeKey, "ck_type", "sheet"}
+ if pos != nil {
+ skip = append(skip, "positions")
+ }
return TableItem{
Table: td,
Sheet: stringField(m, "sheet"),
- Positions: positionsField(m),
+ Positions: pos,
CKType: stringField(m, "ck_type"),
- // Same as imageItemFromMap: keep "positions" in Extra so non-[][]float64
- // position matrices survive the typed conversion unchanged.
- Extra: extraFromMap(m, "table", DocTypeKey, "ck_type", "sheet"),
+ Extra: extraFromMap(m, skip...),
}
}
@@ -244,11 +243,6 @@ func stringField(m map[string]any, key string) string {
return s
}
-func positionsField(m map[string]any) [][]float64 {
- p, _ := m["positions"].([][]float64)
- return p
-}
-
// extraFromMap returns every key of m that is not a canonical field for the
// item type, so it lands in the item's Extra passthrough (matching mergeExtra
// semantics downstream). A nil result means "no extra keys".
@@ -270,22 +264,6 @@ func extraFromMap(m map[string]any, canonical ...string) map[string]any {
return extra
}
-// DocItemsFromMaps maps a []map[string]any to typed DocItems, preserving order,
-// via DocItemFromMap. Parsers that accumulate results as []map[string]any use
-// it at the assignment site (`JSON: DocItemsFromMaps(items)`) so their internal
-// logic is unchanged while the field gains the DocItem type and the concrete
-// constructors become the emission path.
-func DocItemsFromMaps(maps []map[string]any) []DocItem {
- if maps == nil {
- return nil
- }
- out := make([]DocItem, 0, len(maps))
- for _, m := range maps {
- out = append(out, DocItemFromMap(m))
- }
- return out
-}
-
// ItemsToMaps projects a slice of DocItems onto the historical wire form. The
// ingestion boundary calls this once so downstream consumers keep reading
// []map[string]any unchanged.
@@ -324,3 +302,288 @@ func firstNonEmptyStr(values ...string) string {
}
return ""
}
+
+// ---------------------------------------------------------------------------
+// Position: strong-typed replacement for the loosely-typed `positions` wire
+// field. The single wire shape is a 5-tuple array, but it carries TWO
+// vocabularies that share the same five slots:
+// - AbsoluteBox: [page, left, right, top, bottom] (PDF layout bbox)
+// - RowColumnBox: [sheet, rowStart, rowEnd, colStart, colEnd] (spreadsheet)
+// `Kind()` is a method on the concrete type (not a stored field) so it can
+// never disagree with the type. The only consumer-visible seam is Wire(),
+// which renders the historical tuple; ToMap projects through it so the JSON
+// wire shape is unchanged.
+// ---------------------------------------------------------------------------
+
+type PositionKind int
+
+const (
+ // KindAbsolute marks a [page, left, right, top, bottom] bounding box.
+ KindAbsolute PositionKind = iota
+ // KindRowColumn marks a [sheet, rowStart, rowEnd, colStart, colEnd] range.
+ KindRowColumn
+)
+
+// Position is one structured position entry. Implementations are concrete
+// (AbsoluteBox / RowColumnBox); the interface exists only to give ToMap a
+// single projection seam (Wire) and a type-safe discriminator (Kind).
+type Position interface {
+ // Wire renders the historical 5-tuple. AbsoluteBox keeps the page as an
+ // int; RowColumnBox emits all ints.
+ Wire() []any
+ Kind() PositionKind
+}
+
+// AbsoluteBox is a PDF-style absolute bounding box: page number plus the box
+// edges in page coordinates.
+type AbsoluteBox struct {
+ PageNumber int
+ Left float64
+ Right float64
+ Top float64
+ Bottom float64
+}
+
+func (AbsoluteBox) Kind() PositionKind { return KindAbsolute }
+
+func (b AbsoluteBox) Wire() []any {
+ return []any{b.PageNumber, b.Left, b.Right, b.Top, b.Bottom}
+}
+
+// RowColumnBox is a spreadsheet-style cell range: the 1-indexed sheet ordinal
+// plus the row/column span. Sheet is the Nth sheet in the workbook, NOT the
+// Nth table (one sheet emits one table unit; sheet_index is shared).
+type RowColumnBox struct {
+ Sheet int
+ R0 int
+ R1 int
+ C0 int
+ C1 int
+}
+
+func (RowColumnBox) Kind() PositionKind { return KindRowColumn }
+
+func (b RowColumnBox) Wire() []any {
+ return []any{b.Sheet, b.R0, b.R1, b.C0, b.C1}
+}
+
+// Positions is the structured position list carried by an ImageItem/TableItem.
+type Positions []Position
+
+// SpreadsheetPositions converts producer-side spreadsheet tuples
+// ([sheet, r0, r1, c0, c1] as [][]float64) into RowColumnBoxes. Producers call
+// this explicitly so the vocabulary is set at the source, never inferred.
+func SpreadsheetPositions(tuples [][]float64) Positions {
+ return rowColumnBoxesFromTyped(tuples)
+}
+
+// hasSheetIndex reports whether a payload map carries the spreadsheet
+// discriminator. wireToPositions uses it to pick the vocabulary without
+// guessing from the tuple shape (PDF and spreadsheet tuples are both five
+// numbers). It mirrors the downstream isSpreadsheetChunk (task/indexdoc)
+// exactly — sheet_index present and non-nil, or a non-empty "sheet" string —
+// so the parser's classification and the indexer's spreadsheet/PDF decision
+// use identical criteria and can never diverge. A present-but-nil sheet_index
+// is not a spreadsheet signal.
+func hasSheetIndex(m map[string]any) bool {
+ if v, ok := m["sheet_index"]; ok && v != nil {
+ return true
+ }
+ sheet, ok := m["sheet"].(string)
+ return ok && sheet != ""
+}
+
+// wireToPositions is the in-bound seam: it converts a raw `positions` value
+// into structured Positions. A nil / non-5-tuple input returns nil so the
+// caller leaves the original value in Extra verbatim (positions-dropping
+// regression guard). It never panics.
+func wireToPositions(raw any, hasSheet bool) Positions {
+ if hasSheet {
+ return rowColumnBoxesFrom(raw)
+ }
+ return absoluteBoxesFrom(raw)
+}
+
+// positionsToWire is the out-bound seam: it renders structured Positions back
+// into the historical wire form for ToMap. The wire container type is preserved
+// per vocabulary so the parser output is byte-for-byte unchanged:
+// - AbsoluteBox (PDF) -> [][]any (int page + float64 coords), matching the
+// shape normalizePDFPositions already emitted and keeping the page a
+// genuine int (a [][]float64 container would coerce it to float64).
+// - RowColumnBox (xlsx/csv) -> [][]float64, matching the historical
+// spreadsheet wire the ingestion indexer consumes.
+//
+// Preserving each producer's historical container keeps the wire unchanged and
+// avoids churning the existing [][]float64 assertions; it is NOT a correctness
+// requirement. Downstream, the chunker boundary stores positions as
+// json.RawMessage and ChunkDoc.ToMap re-renders them via decodeStructuredValue,
+// which unmarshals any all-numeric array into [][]float64 — so by the time the
+// indexer runs, both vocabularies are already [][]float64 regardless of the
+// container chosen here.
+//
+// A mixed slice (first element decides) cannot occur — a single item is either
+// PDF or spreadsheet — so the [][]any fallback below is purely defensive and is
+// never reached in practice.
+func positionsToWire(p Positions) any {
+ if len(p) == 0 {
+ return nil
+ }
+ if p[0].Kind() == KindRowColumn {
+ out := make([][]float64, 0, len(p))
+ for _, pos := range p {
+ b, ok := pos.(RowColumnBox)
+ if !ok {
+ return positionsToWireAny(p)
+ }
+ out = append(out, []float64{float64(b.Sheet), float64(b.R0), float64(b.R1), float64(b.C0), float64(b.C1)})
+ }
+ return out
+ }
+ return positionsToWireAny(p)
+}
+
+func positionsToWireAny(p Positions) []any {
+ out := make([]any, 0, len(p))
+ for _, pos := range p {
+ out = append(out, pos.Wire())
+ }
+ return out
+}
+
+func absoluteBoxesFrom(raw any) Positions {
+ switch v := raw.(type) {
+ case [][]float64:
+ return absoluteBoxesFromTyped(v)
+ case [][]any:
+ return absoluteBoxesFromAny(v)
+ default:
+ return nil
+ }
+}
+
+func rowColumnBoxesFrom(raw any) Positions {
+ switch v := raw.(type) {
+ case [][]float64:
+ return rowColumnBoxesFromTyped(v)
+ case [][]any:
+ return rowColumnBoxesFromAny(v)
+ default:
+ return nil
+ }
+}
+
+func absoluteBoxesFromTyped(v [][]float64) Positions {
+ out := make(Positions, 0, len(v))
+ for _, t := range v {
+ if len(t) != 5 {
+ return nil
+ }
+ out = append(out, AbsoluteBox{
+ PageNumber: int(t[0]),
+ Left: t[1],
+ Right: t[2],
+ Top: t[3],
+ Bottom: t[4],
+ })
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+func absoluteBoxesFromAny(v [][]any) Positions {
+ out := make(Positions, 0, len(v))
+ for _, t := range v {
+ if len(t) != 5 {
+ return nil
+ }
+ page, ok := t[0].(int)
+ if !ok {
+ return nil
+ }
+ l, lok := positionFloat(t[1])
+ r, rok := positionFloat(t[2])
+ tp, tok := positionFloat(t[3])
+ b, bok := positionFloat(t[4])
+ if !lok || !rok || !tok || !bok {
+ return nil
+ }
+ out = append(out, AbsoluteBox{PageNumber: page, Left: l, Right: r, Top: tp, Bottom: b})
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+func rowColumnBoxesFromTyped(v [][]float64) Positions {
+ out := make(Positions, 0, len(v))
+ for _, t := range v {
+ if len(t) != 5 {
+ return nil
+ }
+ out = append(out, RowColumnBox{
+ Sheet: int(t[0]),
+ R0: int(t[1]),
+ R1: int(t[2]),
+ C0: int(t[3]),
+ C1: int(t[4]),
+ })
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+func rowColumnBoxesFromAny(v [][]any) Positions {
+ out := make(Positions, 0, len(v))
+ for _, t := range v {
+ if len(t) != 5 {
+ return nil
+ }
+ s, ok := toInt(t[0])
+ if !ok {
+ return nil
+ }
+ r0, ok0 := toInt(t[1])
+ r1, ok1 := toInt(t[2])
+ c0, ok2 := toInt(t[3])
+ c1, ok3 := toInt(t[4])
+ if !ok0 || !ok1 || !ok2 || !ok3 {
+ return nil
+ }
+ out = append(out, RowColumnBox{Sheet: s, R0: r0, R1: r1, C0: c0, C1: c1})
+ }
+ if len(out) == 0 {
+ return nil
+ }
+ return out
+}
+
+func positionFloat(v any) (float64, bool) {
+ switch n := v.(type) {
+ case float64:
+ return n, true
+ case int:
+ return float64(n), true
+ case int64:
+ return float64(n), true
+ default:
+ return 0, false
+ }
+}
+
+func toInt(v any) (int, bool) {
+ switch n := v.(type) {
+ case int:
+ return n, true
+ case int64:
+ return int(n), true
+ case float64:
+ return int(n), true
+ default:
+ return 0, false
+ }
+}
diff --git a/internal/parser/parser/doc_item_positions_test.go b/internal/parser/parser/doc_item_positions_test.go
new file mode 100644
index 0000000000..c78fb2a303
--- /dev/null
+++ b/internal/parser/parser/doc_item_positions_test.go
@@ -0,0 +1,173 @@
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package parser
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+)
+
+// AbsoluteBox.Wire projects the PDF absolute bounding box [page, l, r, t, b]
+// onto the historical wire tuple, keeping the page as an int (never float64).
+func TestAbsoluteBoxWire(t *testing.T) {
+ b := AbsoluteBox{PageNumber: 1, Left: 2, Right: 3, Top: 4, Bottom: 5}
+ got := b.Wire()
+ want := []any{1, 2.0, 3.0, 4.0, 5.0}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("AbsoluteBox.Wire() = %#v, want %#v", got, want)
+ }
+}
+
+// RowColumnBox.Wire projects the spreadsheet range [sheet, r0, r1, c0, c1]
+// onto the historical wire tuple as all-int values.
+func TestRowColumnBoxWire(t *testing.T) {
+ b := RowColumnBox{Sheet: 3, R0: 1, R1: 9, C0: 2, C1: 7}
+ got := b.Wire()
+ want := []any{3, 1, 9, 2, 7}
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("RowColumnBox.Wire() = %#v, want %#v", got, want)
+ }
+}
+
+// Kind() is a method on the concrete type (single source of truth), not a
+// stored field, so it can never disagree with the real type.
+func TestPositionKindMethod(t *testing.T) {
+ if (AbsoluteBox{}).Kind() != KindAbsolute {
+ t.Fatal("AbsoluteBox.Kind() != KindAbsolute")
+ }
+ if (RowColumnBox{}).Kind() != KindRowColumn {
+ t.Fatal("RowColumnBox.Kind() != KindRowColumn")
+ }
+ var _ Position = AbsoluteBox{}
+ var _ Position = RowColumnBox{}
+}
+
+// PDF path positions arrive as [][]any (int page + float64 coords) from
+// normalizePDFPositions. wireToPositions must build AbsoluteBox and round-trip
+// byte-exact, with the page preserved as int.
+func TestWireToPositionsAbsoluteAny(t *testing.T) {
+ raw := [][]any{{1, 2.0, 3.0, 4.0, 5.0}}
+ got := wireToPositions(raw, false)
+ if len(got) != 1 {
+ t.Fatalf("len = %d, want 1", len(got))
+ }
+ ab, ok := got[0].(AbsoluteBox)
+ if !ok {
+ t.Fatalf("got[0] = %T, want AbsoluteBox", got[0])
+ }
+ if ab.PageNumber != 1 || ab.Left != 2 || ab.Top != 4 || ab.Bottom != 5 {
+ t.Fatalf("AbsoluteBox = %+v", ab)
+ }
+ if !reflect.DeepEqual(positionsToWire(got), []any{[]any{1, 2.0, 3.0, 4.0, 5.0}}) {
+ t.Fatalf("round-trip = %#v", positionsToWire(got))
+ }
+}
+
+// Excel path positions arrive as [][]float64 of [sheet, r0, r1, c0, c1]. With
+// the sheet_index signal present, wireToPositions must build RowColumnBox and
+// the wire must be all-int []any, JSON-equal to the historical [][]float64.
+func TestWireToPositionsRowColumn(t *testing.T) {
+ raw := [][]float64{{3, 1, 9, 2, 7}}
+ got := wireToPositions(raw, true)
+ if len(got) != 1 {
+ t.Fatalf("len = %d, want 1", len(got))
+ }
+ rb, ok := got[0].(RowColumnBox)
+ if !ok {
+ t.Fatalf("got[0] = %T, want RowColumnBox", got[0])
+ }
+ if rb.Sheet != 3 || rb.R0 != 1 || rb.R1 != 9 || rb.C0 != 2 || rb.C1 != 7 {
+ t.Fatalf("RowColumnBox = %+v", rb)
+ }
+ out := positionsToWire(got)
+ if !reflect.DeepEqual(out, [][]float64{{3, 1, 9, 2, 7}}) {
+ t.Fatalf("positionsToWire = %#v, want [][]float64", out)
+ }
+ b, _ := json.Marshal(out)
+ if string(b) != "[[3,1,9,2,7]]" {
+ t.Fatalf("json = %s", b)
+ }
+}
+
+// Malformed / non-5-tuple input must not panic and must return nil so the
+// caller keeps the raw value in Extra verbatim (positions-dropping regression
+// guard). Covers: short tuple, nil, and a completely wrong type.
+func TestWireToPositionsMalformed(t *testing.T) {
+ if got := wireToPositions([][]any{{1, 2, 3, 4}}, false); got != nil {
+ t.Fatalf("4-tuple absolute = %#v, want nil", got)
+ }
+ if got := wireToPositions([][]float64{{1, 2}}, true); got != nil {
+ t.Fatalf("2-tuple rowcolumn = %#v, want nil", got)
+ }
+ if got := wireToPositions(nil, false); got != nil {
+ t.Fatalf("nil = %#v, want nil", got)
+ }
+ if got := wireToPositions("not a slice", false); got != nil {
+ t.Fatalf("string = %#v, want nil", got)
+ }
+}
+
+// SpreadsheetPositions is the producer-explicit helper (office_table_render):
+// it builds RowColumnBoxes without relying on the legacy-map inference. A mixed
+// Positions slice must render each element through its own Wire().
+func TestSpreadsheetPositions(t *testing.T) {
+ got := SpreadsheetPositions([][]float64{{2, 0, 5, 1, 3}, {2, 6, 9, 1, 3}})
+ if len(got) != 2 {
+ t.Fatalf("len = %d, want 2", len(got))
+ }
+ rb := got[0].(RowColumnBox)
+ if rb.Sheet != 2 || rb.R0 != 0 || rb.R1 != 5 || rb.C0 != 1 || rb.C1 != 3 {
+ t.Fatalf("RowColumnBox = %+v", rb)
+ }
+ mixed := Positions{
+ AbsoluteBox{PageNumber: 1, Left: 1, Top: 1, Right: 2, Bottom: 2},
+ RowColumnBox{Sheet: 2, R0: 0, R1: 1, C0: 0, C1: 1},
+ }
+ out := positionsToWire(mixed)
+ want := []any{[]any{1, 1.0, 2.0, 1.0, 2.0}, []any{2, 0, 1, 0, 1}}
+ if !reflect.DeepEqual(out, want) {
+ t.Fatalf("mixed positionsToWire = %#v, want %#v", out, want)
+ }
+}
+
+// hasSheetIndex picks the positions vocabulary for the in-bound conversion. It
+// MUST mirror the downstream isSpreadsheetChunk (task/indexdoc/process.go)
+// exactly — same criteria — so the parser's classification and the indexer's
+// spreadsheet/PDF decision can never diverge: sheet_index present and non-nil,
+// OR a non-empty "sheet" string. A present-but-nil sheet_index is NOT a
+// spreadsheet signal, and the speculative capital "SheetIndex" key is not one
+// either (isSpreadsheetChunk does not read it).
+func TestHasSheetIndex_MirrorsIsSpreadsheetChunk(t *testing.T) {
+ cases := []struct {
+ name string
+ m map[string]any
+ want bool
+ }{
+ {"sheet_index int", map[string]any{"sheet_index": 1}, true},
+ {"sheet_index zero", map[string]any{"sheet_index": 0}, true},
+ {"sheet_index nil", map[string]any{"sheet_index": nil}, false},
+ {"sheet only", map[string]any{"sheet": "Sheet1"}, true},
+ {"sheet empty", map[string]any{"sheet": ""}, false},
+ {"both present", map[string]any{"sheet_index": 2, "sheet": "S"}, true},
+ {"neither", map[string]any{}, false},
+ {"capital SheetIndex ignored", map[string]any{"SheetIndex": 1}, false},
+ }
+ for _, tc := range cases {
+ if got := hasSheetIndex(tc.m); got != tc.want {
+ t.Errorf("%s: hasSheetIndex(%v) = %v, want %v", tc.name, tc.m, got, tc.want)
+ }
+ }
+}
diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.go
index dfbb5db632..500c405942 100644
--- a/internal/parser/parser/doc_item_test.go
+++ b/internal/parser/parser/doc_item_test.go
@@ -104,7 +104,7 @@ func TestTableItem_ToMap(t *testing.T) {
}, TableItem{
CKType: "table",
Sheet: "Sheet1",
- Positions: [][]float64{{0, 1, 1, 2, 2}},
+ Positions: SpreadsheetPositions([][]float64{{0, 1, 1, 2, 2}}),
}.ToMap())
})
t.Run("empty sheet/positions omitted", func(t *testing.T) {
@@ -144,7 +144,7 @@ func TestImageItem_ToMap(t *testing.T) {
Image: "data:image/png;base64,xxx",
CKType: "image",
Sheet: "Sheet2",
- Positions: [][]float64{{1, 0, 0, 3, 4}},
+ Positions: SpreadsheetPositions([][]float64{{1, 0, 0, 3, 4}}),
}.ToMap())
})
t.Run("extra passthrough (media_order/parent_table_id)", func(t *testing.T) {
@@ -174,11 +174,6 @@ func TestRawItem_Passthrough(t *testing.T) {
got := RawItem{M: in}.ToMap()
mustMap(t, in, got)
})
- t.Run("DocType is empty (lives in map)", func(t *testing.T) {
- if (RawItem{M: map[string]any{DocTypeKey: DocTypeText}}).DocType() != "" {
- t.Error("RawItem.DocType should be empty; discriminator is in the map")
- }
- })
t.Run("nil map does not panic", func(t *testing.T) {
var ri RawItem
if ri.ToMap() != nil {
@@ -222,13 +217,15 @@ func TestConstructorWireShapeRegression(t *testing.T) {
})
t.Run("NewTableDataJSONItem", func(t *testing.T) {
td := &entity.TableData{}
+ // Spreadsheet positions render as [][]float64 on the wire (the historical
+ // shape the ingestion indexer consumes); only PDF positions stay [][]any.
mustMap(t, map[string]any{
"table": td,
DocTypeKey: DocTypeTable,
"ck_type": DocTypeTable,
"sheet": "S",
"positions": [][]float64{{0, 0, 0, 1, 1}},
- }, NewTableDataJSONItem(td, "S", [][]float64{{0, 0, 0, 1, 1}}).ToMap())
+ }, NewTableDataJSONItem(td, "S", SpreadsheetPositions([][]float64{{0, 0, 0, 1, 1}})).ToMap())
})
}
@@ -293,8 +290,20 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
"ck_type": DocTypeTable,
"positions": [][]any{{1, 2, 3, 4}},
}
+ // A table whose "table" value is a JSON-decoded map[string]any (not a typed
+ // *entity.TableData — the shape a component-boundary JSON payload produces)
+ // must NOT be silently coerced into a TableItem with a nil table, which would
+ // emit "table": null and drop the data. The failed type assertion must fall
+ // back to RawItem, preserving the map verbatim — the same round-trip guard
+ // the text/image branches uphold. Regression guard for the silent
+ // table-dropping path.
+ tableMapJSONDecoded := map[string]any{
+ "table": map[string]any{"grid": []any{}, "header_rows": float64(1)},
+ DocTypeKey: DocTypeTable,
+ "ck_type": DocTypeTable,
+ }
- for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny} {
+ for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny, tableMapJSONDecoded} {
got := DocItemFromMap(m).ToMap()
if !reflect.DeepEqual(got, m) {
t.Fatalf("DocItemFromMap round-trip mismatch for %q\n got=%#v\nwant=%#v", m[DocTypeKey], got, m)
@@ -318,12 +327,12 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {
// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way
// DocItemFromMap does, preserving order and skipping nothing.
-func TestDocItemsFromMaps_OrderAndType(t *testing.T) {
+func TestMapsToItems_OrderAndType(t *testing.T) {
maps := []map[string]any{
{"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},
{"foo": "bar"},
}
- items := DocItemsFromMaps(maps)
+ items := MapsToItems(maps)
if len(items) != 2 {
t.Fatalf("expected 2 items, got %d", len(items))
}
diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.go
index 251307faaa..14b9f60119 100644
--- a/internal/parser/parser/office_table_render.go
+++ b/internal/parser/parser/office_table_render.go
@@ -203,7 +203,7 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow int
grid = append(grid, cells)
}
td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}
- ti := NewTableDataJSONItem(td, sheet, matrix).(TableItem)
+ ti := NewTableDataJSONItem(td, sheet, SpreadsheetPositions(matrix)).(TableItem)
ti.Extra = map[string]any{"sheet_index": sheetIndex}
return ti.ToMap()
}
diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.go
index c09226bcdc..3afa7225da 100644
--- a/internal/parser/parser/parse_result.go
+++ b/internal/parser/parser/parse_result.go
@@ -112,7 +112,7 @@ func NewTextJSONItem(text string) DocItem {
// rows, cell text already normalized the same way the legacy chunker
// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,
// so the chunker reads rows directly without re-parsing any markup.
-func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) DocItem {
+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions Positions) DocItem {
return TableItem{
Table: td,
Sheet: sheet,
diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.go
index 257b8af6e1..6f47acd44c 100644
--- a/internal/parser/parser/parse_result_test.go
+++ b/internal/parser/parser/parse_result_test.go
@@ -29,7 +29,7 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {
HeaderRows: 1,
Caption: "Sheet1",
}
- item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})
+ item := NewTableDataJSONItem(td, "Sheet1", SpreadsheetPositions([][]float64{{1, 2, 2, 1, 1}}))
m := item.ToMap()
if got, want := m[DocTypeKey], DocTypeTable; got != want {