RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor(parser): introduce typed Position API for DocItem positions (#21049)
xugangqiangb073eaabc08b
diff --git a/internal/ingestion/component/parser_dispatch.go b/internal/ingestion/component/parser_dispatch.goindex c9fcc66968..fafa593fdb 100644--- a/internal/ingestion/component/parser_dispatch.go+++ b/internal/ingestion/component/parser_dispatch.go@@ -348,7 +348,8 @@ func buildParserOutputs(ctx context.Context, dispatched parser.ParseResult, name// normalizeParserJSON returns the parser output as the typed []DocItem. It no// longer round-trips through map[string]any (the historical ItemsToMaps call),// so the downstream chunker receives structured items and converts them via-// schema.ChunkDocsFromDocItem without ever calling DocItem.ToMap.+// schema.ChunkDocsFromDocItem, which projects each item through its canonical+// DocItem.ToMap wire shape before decoding it into a ChunkDoc.func normalizeParserJSON(ctx context.Context, filename string, dispatched parser.ParseResult) []parser.DocItem {if len(dispatched.JSON) > 0 {return dispatched.JSONdiff --git a/internal/ingestion/component/schema/chunk_docitem_test.go b/internal/ingestion/component/schema/chunk_docitem_test.goindex 42d8b5e27a..fcb37dded4 100644--- a/internal/ingestion/component/schema/chunk_docitem_test.go+++ b/internal/ingestion/component/schema/chunk_docitem_test.go@@ -52,7 +52,7 @@ func TestChunkDocsFromDocItem_ParallelsMapPath(t *testing.T) {{name: "table with sheet and positions",items: []parser.DocItem{- parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: [][]float64{{1, 2}}, Extra: map[string]any{"sheet_index": 0}},+ parser.TableItem{Table: &entity.TableData{}, Sheet: "s", Positions: parser.SpreadsheetPositions([][]float64{{1, 2, 2, 1, 3}}), Extra: map[string]any{"sheet_index": 0}},},},{@@ -210,7 +210,7 @@ func TestDocItemsFromMaps_ParallelsMapPath(t *testing.T) {}for _, tc := range cases {t.Run(tc.name, func(t *testing.T) {- got, found, err := ChunkDocsFromDocItem(parser.DocItemsFromMaps(tc.maps))+ got, found, err := ChunkDocsFromDocItem(parser.MapsToItems(tc.maps))if err != nil {t.Fatalf("ChunkDocsFromDocItem err: %v", err)}diff --git a/internal/ingestion/component/schema/chunk_types.go b/internal/ingestion/component/schema/chunk_types.goindex 8ca5ae09f7..b5a66e122c 100644--- a/internal/ingestion/component/schema/chunk_types.go+++ b/internal/ingestion/component/schema/chunk_types.go@@ -242,10 +242,11 @@ func ChunkDocsFromAny(in any) ([]ChunkDoc, bool, error) {}// ChunkDocsFromDocItem converts a parser-produced []DocItem directly into a-// typed chunk slice, without routing through DocItem.ToMap. It reads the-// typed fields of each item and feeds the canonical wire map to ChunkDocFromMap,-// so the resulting []ChunkDoc is identical to what the historical-// ChunkDocsFromAny([]DocItem) path produced (which round-tripped via ToMap).+// typed chunk slice by projecting each item through its canonical DocItem.ToMap+// wire shape and decoding the result via ChunkDocFromMap (see chunkDocFromDocItem).+// The resulting []ChunkDoc is therefore identical to what the historical+// ChunkDocsFromAny([]DocItem) path produced, which round-tripped through the same+// ToMap projection.// This is the consumer-side entry the chunker uses once parsers emit DocItems// directly; other callers that still pass []map[string]any / []any / []ChunkDoc// keep using ChunkDocsFromAny.diff --git a/internal/ingestion/component/schema/file.go b/internal/ingestion/component/schema/file.goindex e64a7cb7ac..40e11c84d7 100644--- a/internal/ingestion/component/schema/file.go+++ b/internal/ingestion/component/schema/file.go@@ -19,9 +19,10 @@// Pydantic schema in rag/flow/<component>/schema.py (or, when none exists,// the runtime contract implied by the component's _invoke signature).//-// Files in this package are pure data definitions — they import only stdlib.-// Business logic (the component Implementations) lives elsewhere in-// internal/ingestion/component/.+// Files in this package are pure data definitions. They depend on+// ragflow/internal/common, ragflow/internal/entity, and+// ragflow/internal/parser/parser for shared types, but hold no business logic;+// the component Implementations live elsewhere in internal/ingestion/component/.package schema// FileFromUpstream is the upstream payload consumed by the File component.diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 97ea6d8ca8..ddedcb0bf2 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -6,6 +6,9 @@ import ("time""ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/parser/parser")func TestProcessChunksForPipeline_PreservesTypedMetadata(t *testing.T) {@@ -597,6 +600,87 @@ func TestProcessChunkPositions_SpreadsheetKeepsRowIndex(t *testing.T) {}}+// TestProcessChunkPositions_ParserSpreadsheetWire guards the PARSER wire shape:+// a spreadsheet DocItem must project positions as [][]float64 (not [][]any), and+// processChunkPositions must accept that shape. This feeds the parser wire+// directly (the direct-json path in NormalizeChunks that no built-in template+// exercises); the production path that flows through the chunker is covered by+// TestProcessChunkPositions_ChunkerRoundTrip below.+func TestProcessChunkPositions_ParserSpreadsheetWire(t *testing.T) {+ td := &entity.TableData{}+ item := parser.NewTableDataJSONItem(td, "S", parser.SpreadsheetPositions([][]float64{{1, 1, 1, 1, 3}})).ToMap()+ raw, ok := item["positions"].([][]float64)+ if !ok {+ t.Fatalf("parser spreadsheet wire = %T, want [][]float64", item["positions"])+ }+ chunk := map[string]any{+ "positions": raw,+ "top_int": []int{41},+ }+ processChunkPositions(chunk, true)++ matrix, ok := chunk["position_int"].([][]int)+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][4] != 3 {+ t.Fatalf("position_int = %v, want the sheet tuple [1 1 1 1 3]", chunk["position_int"])+ }+ if _, exists := chunk["positions"]; exists {+ t.Error("raw positions must be removed")+ }+}++// TestProcessChunkPositions_ChunkerRoundTrip exercises the PRODUCTION path the+// ingestion pipeline actually uses: parser DocItems cross the chunker boundary+// as ChunkDoc (positions stored as json.RawMessage), and ChunkDoc.ToMap+// re-renders them through decodeStructuredValue, which unmarshals into+// [][]float64 for any all-numeric array. So by the time processChunkPositions+// runs, spreadsheet positions are [][]float64 and reach position_int — and the+// parser's [][]any (PDF) is likewise normalized to [][]float64 here. This is why+// the parser's per-kind container type is irrelevant to indexing: the chunker+// erases it. Regression guard for the positions restructuring's real seam.+func TestProcessChunkPositions_ChunkerRoundTrip(t *testing.T) {+ // Spreadsheet: RowColumnBox -> [][]float64 on the parser wire.+ ssItem := parser.NewTableDataJSONItem(&entity.TableData{}, "S",+ parser.SpreadsheetPositions([][]float64{{1, 1, 1, 1, 3}}))+ docs, found, err := schema.ChunkDocsFromDocItem([]parser.DocItem{ssItem})+ if err != nil || !found || len(docs) != 1 {+ t.Fatalf("ChunkDocsFromDocItem(spreadsheet): found=%v err=%v", found, err)+ }+ m := docs[0].ToMap()+ raw, ok := m["positions"].([][]float64)+ if !ok {+ t.Fatalf("chunker-rendered positions = %T, want [][]float64 (decodeStructuredValue)", m["positions"])+ }+ chunk := map[string]any{"positions": raw, "top_int": []int{41}, "sheet_index": 1}+ processChunkPositions(chunk, true)+ matrix, ok := chunk["position_int"].([][]int)+ if !ok || len(matrix) != 1 || matrix[0][0] != 1 || matrix[0][4] != 3 {+ t.Fatalf("position_int = %v, want [[1 1 1 1 3]]", chunk["position_int"])+ }+ if _, exists := chunk["positions"]; exists {+ t.Error("raw positions must be removed")+ }++ // PDF: AbsoluteBox -> [][]any on the parser wire, but decodeStructuredValue+ // normalizes it to [][]float64 at the chunker boundary too, so the parser+ // container type never reaches the indexer.+ pdfItem := parser.ImageItem{+ Text: "fig",+ Image: "",+ CKType: "image",+ Positions: parser.Positions{+ parser.AbsoluteBox{PageNumber: 1, Left: 100, Right: 50, Top: 200, Bottom: 150},+ },+ }+ pdfDocs, _, err := schema.ChunkDocsFromDocItem([]parser.DocItem{pdfItem})+ if err != nil || len(pdfDocs) != 1 {+ t.Fatalf("ChunkDocsFromDocItem(pdf): err=%v", err)+ }+ pm := pdfDocs[0].ToMap()+ if _, ok := pm["positions"].([][]float64); !ok {+ t.Fatalf("PDF positions after chunker = %T, want [][]float64 (normalized by decodeStructuredValue)", pm["positions"])+ }+}+// TestProcessChunksForPipeline_SpreadsheetPositionsKeepRowIndex pins the whole// chunk loop: the spreadsheet identity has to be read before// stripPipelineOnlyFields removes sheet_index, or the positions of adiff --git a/internal/parser/parser/doc_item.go b/internal/parser/parser/doc_item.goindex 3eb9b23d75..9e26ebe55e 100644--- a/internal/parser/parser/doc_item.go+++ b/internal/parser/parser/doc_item.go@@ -34,8 +34,6 @@ import (// historical map shape. A later pass may promote the well-known Extra keys to// named fields; that is out of scope for the boundary-adapter change.type DocItem interface {- // DocType returns the item's doc_type_kwd discriminator (text/table/image).- DocType() string// ToMap projects the item onto the historical wire shape. Callers must not// mutate the returned map's shared Extra contents.ToMap() map[string]any@@ -50,8 +48,6 @@ type TextItem struct {Extra map[string]any}-func (t TextItem) DocType() string { return DocTypeText }-func (t TextItem) ToMap() map[string]any {m := map[string]any{"text": t.Text,@@ -69,13 +65,11 @@ func (t TextItem) ToMap() map[string]any {type TableItem struct {Table *entity.TableDataSheet string- Positions [][]float64+ Positions PositionsCKType stringExtra map[string]any}-func (t TableItem) DocType() string { return DocTypeTable }-func (t TableItem) ToMap() map[string]any {m := map[string]any{"table": t.Table,@@ -86,7 +80,7 @@ func (t TableItem) ToMap() map[string]any {m["sheet"] = t.Sheet}if len(t.Positions) > 0 {- m["positions"] = t.Positions+ m["positions"] = positionsToWire(t.Positions)}return mergeExtra(m, t.Extra)}@@ -99,13 +93,11 @@ type ImageItem struct {Text stringImage stringSheet string- Positions [][]float64+ Positions PositionsCKType stringExtra map[string]any}-func (i ImageItem) DocType() string { return DocTypeImage }-func (i ImageItem) ToMap() map[string]any {m := map[string]any{"text": i.Text,@@ -119,7 +111,7 @@ func (i ImageItem) ToMap() map[string]any {m["sheet"] = i.Sheet}if len(i.Positions) > 0 {- m["positions"] = i.Positions+ m["positions"] = positionsToWire(i.Positions)}return mergeExtra(m, i.Extra)}@@ -127,14 +119,11 @@ func (i ImageItem) ToMap() map[string]any {// RawItem is a passthrough for items the parser constructs as an opaque map// (the email content item, or any parser still emitting map literals during// the migration). ToMap returns the map unchanged, so the wire shape is-// preserved exactly. DocType is empty because the discriminator lives in the-// map under DocTypeKey.+// preserved exactly. The discriminator lives in the map under DocTypeKey.type RawItem struct {M map[string]any}-func (RawItem) DocType() string { return "" }-func (r RawItem) ToMap() map[string]any { return r.M }// MarshalJSON projects each item onto its wire shape so that downstream@@ -146,16 +135,6 @@ func (t TableItem) MarshalJSON() ([]byte, error) { return json.Marshal(t.ToMap()func (i ImageItem) MarshalJSON() ([]byte, error) { return json.Marshal(i.ToMap()) }func (r RawItem) MarshalJSON() ([]byte, error) { return json.Marshal(r.M) }-// FromMap wraps an opaque payload map as a DocItem. Used by boundary readers-// that receive a []map[string]any and must hand it to DocItem-typed APIs.-func FromMap(m map[string]any) DocItem { return RawItem{M: m} }--// Raw wraps an opaque payload map as a DocItem. Parsers that still build items-// as map literals use it at the assignment site (e.g. `JSON: MapsToItems(items)`-// or `res.JSON = append(res.JSON, Raw(m))`), so their internal logic stays-// unchanged while the field gains the DocItem type.-func Raw(m map[string]any) DocItem { return RawItem{M: m} }-// MapsToItems wraps a slice of payload maps as typed DocItems (one// TextItem / TableItem / ImageItem / RawItem each, detected via doc_type_kwd).// Parsers that accumulate results as []map[string]any use it at the assignment@@ -164,7 +143,11 @@ func Raw(m map[string]any) DocItem { return RawItem{M: m} }// ImageItem constructors become the emission path. Opaque maps without a known// discriminator stay RawItem, preserving the wire shape exactly.func MapsToItems(maps []map[string]any) []DocItem {- return DocItemsFromMaps(maps)+ out := make([]DocItem, 0, len(maps))+ for _, m := range maps {+ out = append(out, DocItemFromMap(m))+ }+ return out}// DocItemFromMap detects the item's doc_type_kwd and builds the matching typed@@ -212,30 +195,46 @@ func textItemFromMap(m map[string]any) DocItem {}func imageItemFromMap(m map[string]any) DocItem {+ pos := wireToPositions(m["positions"], hasSheetIndex(m))+ skip := []string{"text", "image", DocTypeKey, "ck_type", "sheet"}+ if pos != nil {+ // Parsed into the typed field: it owns "positions", so strip it from+ // Extra. Both [][]float64 (office/xlsx) and [][]any (PDF crop regions+ // from normalizePDFPositions) parse here, so the old [][]any-only+ // Extra fallback is gone.+ skip = append(skip, "positions")+ }return ImageItem{Text: stringField(m, "text"),Image: stringField(m, "image"),CKType: stringField(m, "ck_type"),Sheet: stringField(m, "sheet"),- Positions: positionsField(m),- // "positions" is intentionally NOT stripped: when the map carries- // positions as [][]any (e.g. PDF crop regions from normalizePDFPositions)- // the typed field cannot capture them, and stripping would drop the key- // from ToMap. Leaving it in Extra preserves the wire shape exactly.- Extra: extraFromMap(m, "text", "image", DocTypeKey, "ck_type", "sheet"),+ Positions: pos,+ Extra: extraFromMap(m, skip...),}}func tableItemFromMap(m map[string]any) DocItem {- td, _ := m["table"].(*entity.TableData)+ td, ok := m["table"].(*entity.TableData)+ // A table-typed map whose "table" value is not a *entity.TableData (e.g. a+ // JSON-decoded map[string]any) cannot be coerced into a TableItem without+ // silently dropping the data — TableItem.ToMap would emit "table": null.+ // Fall back to RawItem, preserving the map verbatim, exactly like the+ // text/image branches do when their canonical field is absent.+ if !ok {+ return RawItem{M: m}+ }+ pos := wireToPositions(m["positions"], hasSheetIndex(m))+ skip := []string{"table", DocTypeKey, "ck_type", "sheet"}+ if pos != nil {+ skip = append(skip, "positions")+ }return TableItem{Table: td,Sheet: stringField(m, "sheet"),- Positions: positionsField(m),+ Positions: pos,CKType: stringField(m, "ck_type"),- // Same as imageItemFromMap: keep "positions" in Extra so non-[][]float64- // position matrices survive the typed conversion unchanged.- Extra: extraFromMap(m, "table", DocTypeKey, "ck_type", "sheet"),+ Extra: extraFromMap(m, skip...),}}@@ -244,11 +243,6 @@ func stringField(m map[string]any, key string) string {return s}-func positionsField(m map[string]any) [][]float64 {- p, _ := m["positions"].([][]float64)- return p-}-// extraFromMap returns every key of m that is not a canonical field for the// item type, so it lands in the item's Extra passthrough (matching mergeExtra// semantics downstream). A nil result means "no extra keys".@@ -270,22 +264,6 @@ func extraFromMap(m map[string]any, canonical ...string) map[string]any {return extra}-// DocItemsFromMaps maps a []map[string]any to typed DocItems, preserving order,-// via DocItemFromMap. Parsers that accumulate results as []map[string]any use-// it at the assignment site (`JSON: DocItemsFromMaps(items)`) so their internal-// logic is unchanged while the field gains the DocItem type and the concrete-// constructors become the emission path.-func DocItemsFromMaps(maps []map[string]any) []DocItem {- if maps == nil {- return nil- }- out := make([]DocItem, 0, len(maps))- for _, m := range maps {- out = append(out, DocItemFromMap(m))- }- return out-}-// ItemsToMaps projects a slice of DocItems onto the historical wire form. The// ingestion boundary calls this once so downstream consumers keep reading// []map[string]any unchanged.@@ -324,3 +302,288 @@ func firstNonEmptyStr(values ...string) string {}return ""}++// ---------------------------------------------------------------------------+// Position: strong-typed replacement for the loosely-typed `positions` wire+// field. The single wire shape is a 5-tuple array, but it carries TWO+// vocabularies that share the same five slots:+// - AbsoluteBox: [page, left, right, top, bottom] (PDF layout bbox)+// - RowColumnBox: [sheet, rowStart, rowEnd, colStart, colEnd] (spreadsheet)+// `Kind()` is a method on the concrete type (not a stored field) so it can+// never disagree with the type. The only consumer-visible seam is Wire(),+// which renders the historical tuple; ToMap projects through it so the JSON+// wire shape is unchanged.+// ---------------------------------------------------------------------------++type PositionKind int++const (+ // KindAbsolute marks a [page, left, right, top, bottom] bounding box.+ KindAbsolute PositionKind = iota+ // KindRowColumn marks a [sheet, rowStart, rowEnd, colStart, colEnd] range.+ KindRowColumn+)++// Position is one structured position entry. Implementations are concrete+// (AbsoluteBox / RowColumnBox); the interface exists only to give ToMap a+// single projection seam (Wire) and a type-safe discriminator (Kind).+type Position interface {+ // Wire renders the historical 5-tuple. AbsoluteBox keeps the page as an+ // int; RowColumnBox emits all ints.+ Wire() []any+ Kind() PositionKind+}++// AbsoluteBox is a PDF-style absolute bounding box: page number plus the box+// edges in page coordinates.+type AbsoluteBox struct {+ PageNumber int+ Left float64+ Right float64+ Top float64+ Bottom float64+}++func (AbsoluteBox) Kind() PositionKind { return KindAbsolute }++func (b AbsoluteBox) Wire() []any {+ return []any{b.PageNumber, b.Left, b.Right, b.Top, b.Bottom}+}++// RowColumnBox is a spreadsheet-style cell range: the 1-indexed sheet ordinal+// plus the row/column span. Sheet is the Nth sheet in the workbook, NOT the+// Nth table (one sheet emits one table unit; sheet_index is shared).+type RowColumnBox struct {+ Sheet int+ R0 int+ R1 int+ C0 int+ C1 int+}++func (RowColumnBox) Kind() PositionKind { return KindRowColumn }++func (b RowColumnBox) Wire() []any {+ return []any{b.Sheet, b.R0, b.R1, b.C0, b.C1}+}++// Positions is the structured position list carried by an ImageItem/TableItem.+type Positions []Position++// SpreadsheetPositions converts producer-side spreadsheet tuples+// ([sheet, r0, r1, c0, c1] as [][]float64) into RowColumnBoxes. Producers call+// this explicitly so the vocabulary is set at the source, never inferred.+func SpreadsheetPositions(tuples [][]float64) Positions {+ return rowColumnBoxesFromTyped(tuples)+}++// hasSheetIndex reports whether a payload map carries the spreadsheet+// discriminator. wireToPositions uses it to pick the vocabulary without+// guessing from the tuple shape (PDF and spreadsheet tuples are both five+// numbers). It mirrors the downstream isSpreadsheetChunk (task/indexdoc)+// exactly — sheet_index present and non-nil, or a non-empty "sheet" string —+// so the parser's classification and the indexer's spreadsheet/PDF decision+// use identical criteria and can never diverge. A present-but-nil sheet_index+// is not a spreadsheet signal.+func hasSheetIndex(m map[string]any) bool {+ if v, ok := m["sheet_index"]; ok && v != nil {+ return true+ }+ sheet, ok := m["sheet"].(string)+ return ok && sheet != ""+}++// wireToPositions is the in-bound seam: it converts a raw `positions` value+// into structured Positions. A nil / non-5-tuple input returns nil so the+// caller leaves the original value in Extra verbatim (positions-dropping+// regression guard). It never panics.+func wireToPositions(raw any, hasSheet bool) Positions {+ if hasSheet {+ return rowColumnBoxesFrom(raw)+ }+ return absoluteBoxesFrom(raw)+}++// positionsToWire is the out-bound seam: it renders structured Positions back+// into the historical wire form for ToMap. The wire container type is preserved+// per vocabulary so the parser output is byte-for-byte unchanged:+// - AbsoluteBox (PDF) -> [][]any (int page + float64 coords), matching the+// shape normalizePDFPositions already emitted and keeping the page a+// genuine int (a [][]float64 container would coerce it to float64).+// - RowColumnBox (xlsx/csv) -> [][]float64, matching the historical+// spreadsheet wire the ingestion indexer consumes.+//+// Preserving each producer's historical container keeps the wire unchanged and+// avoids churning the existing [][]float64 assertions; it is NOT a correctness+// requirement. Downstream, the chunker boundary stores positions as+// json.RawMessage and ChunkDoc.ToMap re-renders them via decodeStructuredValue,+// which unmarshals any all-numeric array into [][]float64 — so by the time the+// indexer runs, both vocabularies are already [][]float64 regardless of the+// container chosen here.+//+// A mixed slice (first element decides) cannot occur — a single item is either+// PDF or spreadsheet — so the [][]any fallback below is purely defensive and is+// never reached in practice.+func positionsToWire(p Positions) any {+ if len(p) == 0 {+ return nil+ }+ if p[0].Kind() == KindRowColumn {+ out := make([][]float64, 0, len(p))+ for _, pos := range p {+ b, ok := pos.(RowColumnBox)+ if !ok {+ return positionsToWireAny(p)+ }+ out = append(out, []float64{float64(b.Sheet), float64(b.R0), float64(b.R1), float64(b.C0), float64(b.C1)})+ }+ return out+ }+ return positionsToWireAny(p)+}++func positionsToWireAny(p Positions) []any {+ out := make([]any, 0, len(p))+ for _, pos := range p {+ out = append(out, pos.Wire())+ }+ return out+}++func absoluteBoxesFrom(raw any) Positions {+ switch v := raw.(type) {+ case [][]float64:+ return absoluteBoxesFromTyped(v)+ case [][]any:+ return absoluteBoxesFromAny(v)+ default:+ return nil+ }+}++func rowColumnBoxesFrom(raw any) Positions {+ switch v := raw.(type) {+ case [][]float64:+ return rowColumnBoxesFromTyped(v)+ case [][]any:+ return rowColumnBoxesFromAny(v)+ default:+ return nil+ }+}++func absoluteBoxesFromTyped(v [][]float64) Positions {+ out := make(Positions, 0, len(v))+ for _, t := range v {+ if len(t) != 5 {+ return nil+ }+ out = append(out, AbsoluteBox{+ PageNumber: int(t[0]),+ Left: t[1],+ Right: t[2],+ Top: t[3],+ Bottom: t[4],+ })+ }+ if len(out) == 0 {+ return nil+ }+ return out+}++func absoluteBoxesFromAny(v [][]any) Positions {+ out := make(Positions, 0, len(v))+ for _, t := range v {+ if len(t) != 5 {+ return nil+ }+ page, ok := t[0].(int)+ if !ok {+ return nil+ }+ l, lok := positionFloat(t[1])+ r, rok := positionFloat(t[2])+ tp, tok := positionFloat(t[3])+ b, bok := positionFloat(t[4])+ if !lok || !rok || !tok || !bok {+ return nil+ }+ out = append(out, AbsoluteBox{PageNumber: page, Left: l, Right: r, Top: tp, Bottom: b})+ }+ if len(out) == 0 {+ return nil+ }+ return out+}++func rowColumnBoxesFromTyped(v [][]float64) Positions {+ out := make(Positions, 0, len(v))+ for _, t := range v {+ if len(t) != 5 {+ return nil+ }+ out = append(out, RowColumnBox{+ Sheet: int(t[0]),+ R0: int(t[1]),+ R1: int(t[2]),+ C0: int(t[3]),+ C1: int(t[4]),+ })+ }+ if len(out) == 0 {+ return nil+ }+ return out+}++func rowColumnBoxesFromAny(v [][]any) Positions {+ out := make(Positions, 0, len(v))+ for _, t := range v {+ if len(t) != 5 {+ return nil+ }+ s, ok := toInt(t[0])+ if !ok {+ return nil+ }+ r0, ok0 := toInt(t[1])+ r1, ok1 := toInt(t[2])+ c0, ok2 := toInt(t[3])+ c1, ok3 := toInt(t[4])+ if !ok0 || !ok1 || !ok2 || !ok3 {+ return nil+ }+ out = append(out, RowColumnBox{Sheet: s, R0: r0, R1: r1, C0: c0, C1: c1})+ }+ if len(out) == 0 {+ return nil+ }+ return out+}++func positionFloat(v any) (float64, bool) {+ switch n := v.(type) {+ case float64:+ return n, true+ case int:+ return float64(n), true+ case int64:+ return float64(n), true+ default:+ return 0, false+ }+}++func toInt(v any) (int, bool) {+ switch n := v.(type) {+ case int:+ return n, true+ case int64:+ return int(n), true+ case float64:+ return int(n), true+ default:+ return 0, false+ }+}diff --git a/internal/parser/parser/doc_item_positions_test.go b/internal/parser/parser/doc_item_positions_test.gonew file mode 100644index 0000000000..c78fb2a303--- /dev/null+++ b/internal/parser/parser/doc_item_positions_test.go@@ -0,0 +1,173 @@+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package parser++import (+ "encoding/json"+ "reflect"+ "testing"+)++// AbsoluteBox.Wire projects the PDF absolute bounding box [page, l, r, t, b]+// onto the historical wire tuple, keeping the page as an int (never float64).+func TestAbsoluteBoxWire(t *testing.T) {+ b := AbsoluteBox{PageNumber: 1, Left: 2, Right: 3, Top: 4, Bottom: 5}+ got := b.Wire()+ want := []any{1, 2.0, 3.0, 4.0, 5.0}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("AbsoluteBox.Wire() = %#v, want %#v", got, want)+ }+}++// RowColumnBox.Wire projects the spreadsheet range [sheet, r0, r1, c0, c1]+// onto the historical wire tuple as all-int values.+func TestRowColumnBoxWire(t *testing.T) {+ b := RowColumnBox{Sheet: 3, R0: 1, R1: 9, C0: 2, C1: 7}+ got := b.Wire()+ want := []any{3, 1, 9, 2, 7}+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("RowColumnBox.Wire() = %#v, want %#v", got, want)+ }+}++// Kind() is a method on the concrete type (single source of truth), not a+// stored field, so it can never disagree with the real type.+func TestPositionKindMethod(t *testing.T) {+ if (AbsoluteBox{}).Kind() != KindAbsolute {+ t.Fatal("AbsoluteBox.Kind() != KindAbsolute")+ }+ if (RowColumnBox{}).Kind() != KindRowColumn {+ t.Fatal("RowColumnBox.Kind() != KindRowColumn")+ }+ var _ Position = AbsoluteBox{}+ var _ Position = RowColumnBox{}+}++// PDF path positions arrive as [][]any (int page + float64 coords) from+// normalizePDFPositions. wireToPositions must build AbsoluteBox and round-trip+// byte-exact, with the page preserved as int.+func TestWireToPositionsAbsoluteAny(t *testing.T) {+ raw := [][]any{{1, 2.0, 3.0, 4.0, 5.0}}+ got := wireToPositions(raw, false)+ if len(got) != 1 {+ t.Fatalf("len = %d, want 1", len(got))+ }+ ab, ok := got[0].(AbsoluteBox)+ if !ok {+ t.Fatalf("got[0] = %T, want AbsoluteBox", got[0])+ }+ if ab.PageNumber != 1 || ab.Left != 2 || ab.Top != 4 || ab.Bottom != 5 {+ t.Fatalf("AbsoluteBox = %+v", ab)+ }+ if !reflect.DeepEqual(positionsToWire(got), []any{[]any{1, 2.0, 3.0, 4.0, 5.0}}) {+ t.Fatalf("round-trip = %#v", positionsToWire(got))+ }+}++// Excel path positions arrive as [][]float64 of [sheet, r0, r1, c0, c1]. With+// the sheet_index signal present, wireToPositions must build RowColumnBox and+// the wire must be all-int []any, JSON-equal to the historical [][]float64.+func TestWireToPositionsRowColumn(t *testing.T) {+ raw := [][]float64{{3, 1, 9, 2, 7}}+ got := wireToPositions(raw, true)+ if len(got) != 1 {+ t.Fatalf("len = %d, want 1", len(got))+ }+ rb, ok := got[0].(RowColumnBox)+ if !ok {+ t.Fatalf("got[0] = %T, want RowColumnBox", got[0])+ }+ if rb.Sheet != 3 || rb.R0 != 1 || rb.R1 != 9 || rb.C0 != 2 || rb.C1 != 7 {+ t.Fatalf("RowColumnBox = %+v", rb)+ }+ out := positionsToWire(got)+ if !reflect.DeepEqual(out, [][]float64{{3, 1, 9, 2, 7}}) {+ t.Fatalf("positionsToWire = %#v, want [][]float64", out)+ }+ b, _ := json.Marshal(out)+ if string(b) != "[[3,1,9,2,7]]" {+ t.Fatalf("json = %s", b)+ }+}++// Malformed / non-5-tuple input must not panic and must return nil so the+// caller keeps the raw value in Extra verbatim (positions-dropping regression+// guard). Covers: short tuple, nil, and a completely wrong type.+func TestWireToPositionsMalformed(t *testing.T) {+ if got := wireToPositions([][]any{{1, 2, 3, 4}}, false); got != nil {+ t.Fatalf("4-tuple absolute = %#v, want nil", got)+ }+ if got := wireToPositions([][]float64{{1, 2}}, true); got != nil {+ t.Fatalf("2-tuple rowcolumn = %#v, want nil", got)+ }+ if got := wireToPositions(nil, false); got != nil {+ t.Fatalf("nil = %#v, want nil", got)+ }+ if got := wireToPositions("not a slice", false); got != nil {+ t.Fatalf("string = %#v, want nil", got)+ }+}++// SpreadsheetPositions is the producer-explicit helper (office_table_render):+// it builds RowColumnBoxes without relying on the legacy-map inference. A mixed+// Positions slice must render each element through its own Wire().+func TestSpreadsheetPositions(t *testing.T) {+ got := SpreadsheetPositions([][]float64{{2, 0, 5, 1, 3}, {2, 6, 9, 1, 3}})+ if len(got) != 2 {+ t.Fatalf("len = %d, want 2", len(got))+ }+ rb := got[0].(RowColumnBox)+ if rb.Sheet != 2 || rb.R0 != 0 || rb.R1 != 5 || rb.C0 != 1 || rb.C1 != 3 {+ t.Fatalf("RowColumnBox = %+v", rb)+ }+ mixed := Positions{+ AbsoluteBox{PageNumber: 1, Left: 1, Top: 1, Right: 2, Bottom: 2},+ RowColumnBox{Sheet: 2, R0: 0, R1: 1, C0: 0, C1: 1},+ }+ out := positionsToWire(mixed)+ want := []any{[]any{1, 1.0, 2.0, 1.0, 2.0}, []any{2, 0, 1, 0, 1}}+ if !reflect.DeepEqual(out, want) {+ t.Fatalf("mixed positionsToWire = %#v, want %#v", out, want)+ }+}++// hasSheetIndex picks the positions vocabulary for the in-bound conversion. It+// MUST mirror the downstream isSpreadsheetChunk (task/indexdoc/process.go)+// exactly — same criteria — so the parser's classification and the indexer's+// spreadsheet/PDF decision can never diverge: sheet_index present and non-nil,+// OR a non-empty "sheet" string. A present-but-nil sheet_index is NOT a+// spreadsheet signal, and the speculative capital "SheetIndex" key is not one+// either (isSpreadsheetChunk does not read it).+func TestHasSheetIndex_MirrorsIsSpreadsheetChunk(t *testing.T) {+ cases := []struct {+ name string+ m map[string]any+ want bool+ }{+ {"sheet_index int", map[string]any{"sheet_index": 1}, true},+ {"sheet_index zero", map[string]any{"sheet_index": 0}, true},+ {"sheet_index nil", map[string]any{"sheet_index": nil}, false},+ {"sheet only", map[string]any{"sheet": "Sheet1"}, true},+ {"sheet empty", map[string]any{"sheet": ""}, false},+ {"both present", map[string]any{"sheet_index": 2, "sheet": "S"}, true},+ {"neither", map[string]any{}, false},+ {"capital SheetIndex ignored", map[string]any{"SheetIndex": 1}, false},+ }+ for _, tc := range cases {+ if got := hasSheetIndex(tc.m); got != tc.want {+ t.Errorf("%s: hasSheetIndex(%v) = %v, want %v", tc.name, tc.m, got, tc.want)+ }+ }+}diff --git a/internal/parser/parser/doc_item_test.go b/internal/parser/parser/doc_item_test.goindex dfbb5db632..500c405942 100644--- a/internal/parser/parser/doc_item_test.go+++ b/internal/parser/parser/doc_item_test.go@@ -104,7 +104,7 @@ func TestTableItem_ToMap(t *testing.T) {}, TableItem{CKType: "table",Sheet: "Sheet1",- Positions: [][]float64{{0, 1, 1, 2, 2}},+ Positions: SpreadsheetPositions([][]float64{{0, 1, 1, 2, 2}}),}.ToMap())})t.Run("empty sheet/positions omitted", func(t *testing.T) {@@ -144,7 +144,7 @@ func TestImageItem_ToMap(t *testing.T) {Image: "data:image/png;base64,xxx",CKType: "image",Sheet: "Sheet2",- Positions: [][]float64{{1, 0, 0, 3, 4}},+ Positions: SpreadsheetPositions([][]float64{{1, 0, 0, 3, 4}}),}.ToMap())})t.Run("extra passthrough (media_order/parent_table_id)", func(t *testing.T) {@@ -174,11 +174,6 @@ func TestRawItem_Passthrough(t *testing.T) {got := RawItem{M: in}.ToMap()mustMap(t, in, got)})- t.Run("DocType is empty (lives in map)", func(t *testing.T) {- if (RawItem{M: map[string]any{DocTypeKey: DocTypeText}}).DocType() != "" {- t.Error("RawItem.DocType should be empty; discriminator is in the map")- }- })t.Run("nil map does not panic", func(t *testing.T) {var ri RawItemif ri.ToMap() != nil {@@ -222,13 +217,15 @@ func TestConstructorWireShapeRegression(t *testing.T) {})t.Run("NewTableDataJSONItem", func(t *testing.T) {td := &entity.TableData{}+ // Spreadsheet positions render as [][]float64 on the wire (the historical+ // shape the ingestion indexer consumes); only PDF positions stay [][]any.mustMap(t, map[string]any{"table": td,DocTypeKey: DocTypeTable,"ck_type": DocTypeTable,"sheet": "S","positions": [][]float64{{0, 0, 0, 1, 1}},- }, NewTableDataJSONItem(td, "S", [][]float64{{0, 0, 0, 1, 1}}).ToMap())+ }, NewTableDataJSONItem(td, "S", SpreadsheetPositions([][]float64{{0, 0, 0, 1, 1}})).ToMap())})}@@ -293,8 +290,20 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {"ck_type": DocTypeTable,"positions": [][]any{{1, 2, 3, 4}},}+ // A table whose "table" value is a JSON-decoded map[string]any (not a typed+ // *entity.TableData — the shape a component-boundary JSON payload produces)+ // must NOT be silently coerced into a TableItem with a nil table, which would+ // emit "table": null and drop the data. The failed type assertion must fall+ // back to RawItem, preserving the map verbatim — the same round-trip guard+ // the text/image branches uphold. Regression guard for the silent+ // table-dropping path.+ tableMapJSONDecoded := map[string]any{+ "table": map[string]any{"grid": []any{}, "header_rows": float64(1)},+ DocTypeKey: DocTypeTable,+ "ck_type": DocTypeTable,+ }- for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny} {+ for _, m := range []map[string]any{textMap, imageMap, tableMap, opaqueMap, imageMapAny, tableMapAny, tableMapJSONDecoded} {got := DocItemFromMap(m).ToMap()if !reflect.DeepEqual(got, m) {t.Fatalf("DocItemFromMap round-trip mismatch for %q\n got=%#v\nwant=%#v", m[DocTypeKey], got, m)@@ -318,12 +327,12 @@ func TestDocItemFromMap_WireShapePreserved(t *testing.T) {// DocItemsFromMaps maps a []map[string]any to typed DocItems the same way// DocItemFromMap does, preserving order and skipping nothing.-func TestDocItemsFromMaps_OrderAndType(t *testing.T) {+func TestMapsToItems_OrderAndType(t *testing.T) {maps := []map[string]any{{"text": "a", DocTypeKey: DocTypeText, "ck_type": "text"},{"foo": "bar"},}- items := DocItemsFromMaps(maps)+ items := MapsToItems(maps)if len(items) != 2 {t.Fatalf("expected 2 items, got %d", len(items))}diff --git a/internal/parser/parser/office_table_render.go b/internal/parser/parser/office_table_render.goindex 251307faaa..14b9f60119 100644--- a/internal/parser/parser/office_table_render.go+++ b/internal/parser/parser/office_table_render.go@@ -203,7 +203,7 @@ func buildSheetItems(records [][]string, sheet string, sheetIndex, headerRow intgrid = append(grid, cells)}td := &entity.TableData{Grid: grid, HeaderRows: 1, Caption: sheet}- ti := NewTableDataJSONItem(td, sheet, matrix).(TableItem)+ ti := NewTableDataJSONItem(td, sheet, SpreadsheetPositions(matrix)).(TableItem)ti.Extra = map[string]any{"sheet_index": sheetIndex}return ti.ToMap()}diff --git a/internal/parser/parser/parse_result.go b/internal/parser/parser/parse_result.goindex c09226bcdc..3afa7225da 100644--- a/internal/parser/parser/parse_result.go+++ b/internal/parser/parser/parse_result.go@@ -112,7 +112,7 @@ func NewTextJSONItem(text string) DocItem {// rows, cell text already normalized the same way the legacy chunker// normalized HTML <td>/<th> text) instead of the legacy HTML markup string,// so the chunker reads rows directly without re-parsing any markup.-func NewTableDataJSONItem(td *entity.TableData, sheet string, positions [][]float64) DocItem {+func NewTableDataJSONItem(td *entity.TableData, sheet string, positions Positions) DocItem {return TableItem{Table: td,Sheet: sheet,diff --git a/internal/parser/parser/parse_result_test.go b/internal/parser/parser/parse_result_test.goindex 257b8af6e1..6f47acd44c 100644--- a/internal/parser/parser/parse_result_test.go+++ b/internal/parser/parser/parse_result_test.go@@ -29,7 +29,7 @@ func TestNewTableDataJSONItem_IncludesDocTypeAndChunkType(t *testing.T) {HeaderRows: 1,Caption: "Sheet1",}- item := NewTableDataJSONItem(td, "Sheet1", [][]float64{{1, 2, 2, 1, 1}})+ item := NewTableDataJSONItem(td, "Sheet1", SpreadsheetPositions([][]float64{{1, 2, 2, 1, 1}}))m := item.ToMap()if got, want := m[DocTypeKey], DocTypeTable; got != want {