Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(chunker): preserve list-form PDF positions in title chunkers (#21258)

jay7772193b314f3061c
diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.go
index 58256d756f..8f402ea953 100644
--- a/internal/ingestion/component/chunker/crop_title_test.go
+++ b/internal/ingestion/component/chunker/crop_title_test.go
@@ -53,6 +53,23 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {
}
}
+// TestCropTitleChunks_CropsImageWithPageList covers the image variant: a
+// title-family image chunk whose page cell is a list must still be cropped.
+func TestCropTitleChunks_CropsImageWithPageList(t *testing.T) {
+ pos := [][]any{{[]any{1}, 10, 100, 10, 100}}
+ in := []map[string]any{
+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},
+ }
+ got := cropTitleChunks(context.Background(), mockCropEngine{}, in)
+ if len(got) != 1 {
+ t.Fatalf("len = %d, want 1", len(got))
+ }
+ img, _ := got[0]["image"].(string)
+ if !strings.HasPrefix(img, "data:image/png;base64,") {
+ t.Errorf("image = %q, want a cropped data URL", img)
+ }
+}
+
// TestCropTitleChunks_NilEnginePassthrough asserts the helper is a no-op
// when no PDF engine is available (best-effort contract).
func TestCropTitleChunks_NilEnginePassthrough(t *testing.T) {
diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.go
index 9a63747458..146a3c7f00 100644
--- a/internal/ingestion/component/chunker/general.go
+++ b/internal/ingestion/component/chunker/general.go
@@ -1273,7 +1273,7 @@ func mergeGeneralMetadata(dst *schema.ChunkDoc, src schema.ChunkDoc) {
}
func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {
- if positions := mergePositionMatrix(first, second); len(positions) > 0 {
+ if positions := mergePositionMatrix(first, second); positions != nil {
encoded, err := json.Marshal(positions)
if err == nil {
return encoded
diff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.go
index dec608e3e0..b35498985e 100644
--- a/internal/ingestion/component/chunker/group.go
+++ b/internal/ingestion/component/chunker/group.go
@@ -420,22 +420,29 @@ func pdfPosRowLess(a, b []float64) bool {
// [page,left,right,top,bottom]; duplicates (by the first five columns)
// are dropped and rows are sorted by (page, top, left). Returns nil when
// no source carries any usable coordinates.
-func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {
- var out [][]float64
+//
+// The page cell may be a scalar (deepdoc's normalized form) or a list (the
+// raw layout.SectionsToJSON form, also seen in external/historical payloads);
+// both are preserved verbatim so the downstream PositionsFromMatrix can decode
+// either. A source may also be a single flat 5-tuple rather than a list of
+// rows (TCADP and some external producers emit that shape).
+//
+// Rows stay []any because a page cell may itself be a list; the generic shape
+// is what ChunkDoc.ToMap emits for such a matrix, so one representation flows
+// through the chunker and the index boundary.
+func mergePositionMatrix(sources ...json.RawMessage) [][]any {
+ var out [][]any
seen := make(map[string]bool)
for _, src := range sources {
- if len(src) == 0 {
- continue
- }
- var mat [][]float64
- if err := json.Unmarshal(src, &mat); err != nil {
+ rows, ok := decodePositionRows(src)
+ if !ok {
continue
}
- for _, row := range mat {
+ for _, row := range rows {
if len(row) < 5 {
continue
}
- key := fmt.Sprintf("%v|%v|%v|%v|%v", row[0], row[1], row[2], row[3], row[4])
+ key := fmt.Sprint(row[:5])
if seen[key] {
continue
}
@@ -446,12 +453,101 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {
if len(out) == 0 {
return nil
}
- sort.Slice(out, func(i, j int) bool {
- return pdfPosRowLess(out[i], out[j])
+ sort.SliceStable(out, func(i, j int) bool {
+ return positionRowLess(out[i], out[j])
})
return out
}
+// decodePositionRows accepts the nested matrix form, a single flat 5-tuple,
+// and a flat sequence of multiple 5-tuples (TCADP/external producers emit a
+// flat array). JSON-decoded numbers are float64, and a page cell may itself
+// be a list, so the rows stay []any rather than being coerced to [][]float64.
+//
+// A flat sequence whose length is a multiple of five is grouped by five so
+// every tuple survives; a flat length that is not a multiple of five has
+// ambiguous tuple boundaries and is preserved verbatim as one row so the
+// first tuple still reaches cropping (sorting abstains via firstPositionRow).
+func decodePositionRows(src json.RawMessage) ([][]any, bool) {
+ if len(src) == 0 {
+ return nil, false
+ }
+ var rows [][]any
+ if err := json.Unmarshal(src, &rows); err == nil {
+ return rows, true
+ }
+ var flat []any
+ if err := json.Unmarshal(src, &flat); err != nil || len(flat) < 5 {
+ return nil, false
+ }
+ if len(flat)%5 != 0 {
+ return [][]any{flat}, true
+ }
+ grouped := make([][]any, 0, len(flat)/5)
+ for i := 0; i < len(flat); i += 5 {
+ grouped = append(grouped, flat[i:i+5])
+ }
+ return grouped, true
+}
+
+// positionRowLess orders rows by (first page, top, left), matching
+// pdfPosRowLess for scalar page numbers while tolerating a list page cell.
+func positionRowLess(a, b []any) bool {
+ ap, bp := positionRowPage(a), positionRowPage(b)
+ if ap != bp {
+ return ap < bp
+ }
+ at, bt := positionRowFloat(a, 3), positionRowFloat(b, 3)
+ if at != bt {
+ return at < bt
+ }
+ return positionRowFloat(a, 1) < positionRowFloat(b, 1)
+}
+
+// positionRowPage decodes the page cell for ordering only; the original cell
+// is kept in the emitted row. A list page uses its first entry.
+func positionRowPage(row []any) float64 {
+ if len(row) == 0 {
+ return 0
+ }
+ switch v := row[0].(type) {
+ case float64:
+ return v
+ case int:
+ return float64(v)
+ case []any:
+ if len(v) == 0 {
+ return 0
+ }
+ return positionRowPage([]any{v[0]})
+ default:
+ return 0
+ }
+}
+
+func positionRowFloat(row []any, index int) float64 {
+ if index >= len(row) {
+ return 0
+ }
+ value, _ := positionRowNumber(row[index])
+ return value
+}
+
+// positionRowNumber decodes one numeric matrix element. JSON-decoded numbers
+// are float64; Go-constructed callers may use int/int64.
+func positionRowNumber(raw any) (float64, bool) {
+ switch v := raw.(type) {
+ case float64:
+ return v, true
+ case int:
+ return float64(v), true
+ case int64:
+ return float64(v), true
+ default:
+ return 0, false
+ }
+}
+
// extractLineRecords converts the decoded upstream payload into records in the
// same order the python BaseTitleChunker.extract_line_records uses: the field
// named by output_format selects the payload (chunks / json / markdown / text
diff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.go
index c14d4e8bbb..1da22158b6 100644
--- a/internal/ingestion/component/chunker/group_test.go
+++ b/internal/ingestion/component/chunker/group_test.go
@@ -367,7 +367,7 @@ func TestGroupChunker_MergesPDFPositionsAndRemovesTags(t *testing.T) {
t.Errorf("parser position tags leaked into chunk text: %q", text)
}
// (b) positions must be merged across both records.
- pos, ok := ck["positions"].([][]float64)
+ pos, ok := ck["positions"].([][]any)
if !ok {
t.Fatalf("positions missing or wrong type %T on merged group chunk", ck["positions"])
}
diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.go
index 10bf71c4e5..e896728fa1 100644
--- a/internal/ingestion/component/chunker/manual_test.go
+++ b/internal/ingestion/component/chunker/manual_test.go
@@ -264,7 +264,7 @@ func TestManualChunker_MergesPositions(t *testing.T) {
if strings.Contains(text, "@@") {
t.Errorf("parser position tags leaked into chunk text: %q", text)
}
- pos, ok := ck["_pdf_positions"].([][]float64)
+ pos, ok := ck["_pdf_positions"].([][]any)
if !ok {
t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"])
}
@@ -274,8 +274,8 @@ func TestManualChunker_MergesPositions(t *testing.T) {
t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos))
}
// Merged matrix must itself be sorted by (page, top, left) — the
- // same key pdfPosRowLess uses when merging.
- if pdfPosRowLess(pos[1], pos[0]) {
+ // same key positionRowLess uses when merging.
+ if positionRowLess(pos[1], pos[0]) {
t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos)
}
}
diff --git a/internal/ingestion/component/chunker/position_merge_test.go b/internal/ingestion/component/chunker/position_merge_test.go
new file mode 100644
index 0000000000..88532829df
--- /dev/null
+++ b/internal/ingestion/component/chunker/position_merge_test.go
@@ -0,0 +1,275 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package chunker
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+
+ "ragflow/internal/ingestion/component/schema"
+)
+
+// TestMergePositionMatrixAcceptsPageLists pins the Title-family parity fix:
+// positions produced by layout.SectionsToJSON keep the page as a list until
+// normalization, and a source may be a single flat 5-tuple (TCADP). Both must
+// survive the merge instead of being dropped, so the on-demand PDF crop can
+// still run.
+func TestMergePositionMatrixAcceptsPageLists(t *testing.T) {
+ listPage := json.RawMessage(`[[[1],10,100,10,100]]`)
+ flatRow := json.RawMessage(`[2,5,50,5,50]`)
+ duplicate := json.RawMessage(`[[[1],10,100,10,100]]`)
+
+ got := mergePositionMatrix(listPage, flatRow, duplicate)
+ want := [][]any{
+ {[]any{float64(1)}, float64(10), float64(100), float64(10), float64(100)},
+ {float64(2), float64(5), float64(50), float64(5), float64(50)},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("mergePositionMatrix = %#v, want %#v", got, want)
+ }
+}
+
+// TestMergePositionMatrixPreservesNonNumericRows pins that a malformed row is
+// kept verbatim instead of being silently coerced to zeros.
+func TestMergePositionMatrixPreservesNonNumericRows(t *testing.T) {
+ src := json.RawMessage(`[[1,"bad",100,10,100]]`)
+ got := mergePositionMatrix(src)
+ if len(got) != 1 {
+ t.Fatalf("mergePositionMatrix = %#v, want the raw row", got)
+ }
+ raw, err := json.Marshal(got)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if string(raw) != `[[1,"bad",100,10,100]]` {
+ t.Errorf("raw rows = %s, want the source unchanged", raw)
+ }
+}
+
+// TestTitleFamilyAcceptsFlatRowPositions covers the TCADP shape: a source may
+// carry one flat 5-tuple instead of a list of rows. The title merge must nest
+// it so the emitted chunk is still [[page,left,right,top,bottom]].
+func TestTitleFamilyAcceptsFlatRowPositions(t *testing.T) {
+ pos := []float64{1, 10, 100, 10, 100}
+ items := []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},
+ }
+ title, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ out, err := title.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.pdf", "output_format": "json", "json": items,
+ })
+ if err != nil {
+ t.Fatalf("Invoke: %v", err)
+ }
+ chunks, _ := out["chunks"].([]map[string]any)
+ var image map[string]any
+ for _, ck := range chunks {
+ if ck["doc_type_kwd"] == "image" {
+ image = ck
+ }
+ }
+ if image == nil {
+ t.Fatalf("no image chunk emitted: %#v", chunks)
+ }
+ raw, err := json.Marshal(image["positions"])
+ if err != nil {
+ t.Fatalf("marshal positions: %v", err)
+ }
+ if string(raw) != `[[1,10,100,10,100]]` {
+ t.Errorf("positions = %s, want [[1,10,100,10,100]]", raw)
+ }
+}
+
+// TestPositionOrderingAcceptsListPages pins the ordering paths to the same
+// shape handling as the merge: a list page cell must be seen as positioned,
+// otherwise Manual/General leave records in parser order.
+func TestPositionOrderingAcceptsListPages(t *testing.T) {
+ first := json.RawMessage(`[[[1],10,100,100,200]]`) // top 100
+ second := json.RawMessage(`[[[1],10,100,300,400]]`) // top 300
+
+ recs := []lineRecord{
+ {text: "second", pdfPositions: second},
+ {text: "first", pdfPositions: first},
+ }
+ if !hasPdfPositions(recs) {
+ t.Fatal("hasPdfPositions = false for list-page positions")
+ }
+ if row, ok := firstPositionRow(recs[0]); !ok || row[0] != 1 || row[3] != 300 {
+ t.Fatalf("firstPositionRow = %v, %v; want page 1 top 300", row, ok)
+ }
+ sortRecordsByPosition(recs)
+ if recs[0].text != "first" || recs[1].text != "second" {
+ t.Fatalf("sorted records = %q, %q; want first, second", recs[0].text, recs[1].text)
+ }
+
+ units := []schema.ChunkDoc{
+ {Text: "second", PDFPositions: second},
+ {Text: "first", PDFPositions: first},
+ }
+ if !hasPDFPositions(units) {
+ t.Fatal("hasPDFPositions = false for list-page positions")
+ }
+ sorted := sortPDFUnits(units)
+ if sorted[0].Text != "first" || sorted[1].Text != "second" {
+ t.Fatalf("sortPDFUnits = %q, %q; want first, second", sorted[0].Text, sorted[1].Text)
+ }
+}
+
+// TestMergePositionMatrixGroupsFlatMultiTuples pins the flat multi-tuple
+// handling: a flat array carrying several 5-tuples (TCADP/external form) must
+// be grouped by five instead of treated as one row, so no tuple is dropped
+// and distinct sources sharing a first tuple are not deduplicated into one.
+func TestMergePositionMatrixGroupsFlatMultiTuples(t *testing.T) {
+ src := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)
+ got := mergePositionMatrix(src)
+ if len(got) != 2 {
+ t.Fatalf("mergePositionMatrix flat multi-tuple = %#v, want 2 rows", got)
+ }
+ first, err := json.Marshal(got[0])
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ second, err := json.Marshal(got[1])
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if string(first) != `[1,10,100,20,80]` || string(second) != `[2,30,120,40,90]` {
+ t.Fatalf("mergePositionMatrix = %s %s, want [1,10,100,20,80] [2,30,120,40,90]", first, second)
+ }
+
+ a := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)
+ b := json.RawMessage(`[1,10,100,20,80,3,30,120,40,90]`)
+ got = mergePositionMatrix(a, b)
+ if len(got) != 3 {
+ t.Fatalf("mergePositionMatrix distinct flat sources = %#v, want 3 rows", got)
+ }
+}
+
+// TestPositionOrderingAcceptsFlatMultiTuples pins that a flat multi-tuple
+// record participates in physical-order sorting by its first tuple, while a
+// malformed flat sequence (ambiguous boundaries) reports unpositioned instead
+// of sorting by the wrong keys.
+func TestPositionOrderingAcceptsFlatMultiTuples(t *testing.T) {
+ multi := json.RawMessage(`[2,30,120,40,90,1,10,100,20,80]`)
+ single := json.RawMessage(`[[1,10,100,10,100]]`)
+
+ if row, ok := firstPositionRow(lineRecord{pdfPositions: multi}); !ok || row[0] != 2 || row[3] != 40 {
+ t.Fatalf("firstPositionRow flat multi-tuple = %v, %v; want page 2 top 40", row, ok)
+ }
+
+ recs := []lineRecord{
+ {text: "multi", pdfPositions: multi},
+ {text: "single", pdfPositions: single},
+ }
+ sortRecordsByPosition(recs)
+ if recs[0].text != "single" || recs[1].text != "multi" {
+ t.Fatalf("sorted records = %q, %q; want single, multi", recs[0].text, recs[1].text)
+ }
+
+ units := []schema.ChunkDoc{
+ {Text: "multi", PDFPositions: multi},
+ {Text: "single", PDFPositions: single},
+ }
+ sorted := sortPDFUnits(units)
+ if sorted[0].Text != "single" || sorted[1].Text != "multi" {
+ t.Fatalf("sortPDFUnits = %q, %q; want single, multi", sorted[0].Text, sorted[1].Text)
+ }
+
+ malformed := json.RawMessage(`[1,10,100,20,80,2,30]`)
+ if _, ok := firstPositionRow(lineRecord{pdfPositions: malformed}); ok {
+ t.Fatal("firstPositionRow malformed flat = ok, want false for ambiguous boundaries")
+ }
+}
+
+// TestTitleFamilyKeepsListPagePositions is the end-to-end guard: an image
+// record whose page cell is a list must keep its coordinates on the emitted
+// chunk, matching General's pass-through behavior.
+func TestTitleFamilyKeepsListPagePositions(t *testing.T) {
+ pos := [][]any{{[]any{1}, 10, 100, 10, 100}}
+ items := []map[string]any{
+ {"text": "before", "doc_type_kwd": "text"},
+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},
+ }
+ levels := [][]string{{`^# `}}
+
+ title, err := NewGroupTitleChunker(map[string]any{"levels": levels})
+ if err != nil {
+ t.Fatalf("NewGroupTitleChunker: %v", err)
+ }
+ out, err := title.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.pdf", "output_format": "json", "json": items,
+ })
+ if err != nil {
+ t.Fatalf("Title Invoke: %v", err)
+ }
+ titleChunks, _ := out["chunks"].([]map[string]any)
+ var titleImage map[string]any
+ for _, ck := range titleChunks {
+ if ck["doc_type_kwd"] == "image" {
+ titleImage = ck
+ }
+ }
+ if titleImage == nil {
+ t.Fatalf("Title emitted no image chunk: %#v", titleChunks)
+ }
+ assertListPagePositions(t, "title", titleImage)
+
+ general, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})
+ if err != nil {
+ t.Fatalf("NewGeneralChunker: %v", err)
+ }
+ gout, err := general.Invoke(t.Context(), nil, map[string]any{
+ "name": "doc.pdf", "file_type": "pdf", "output_format": "json", "json": items,
+ })
+ if err != nil {
+ t.Fatalf("General Invoke: %v", err)
+ }
+ generalChunks, _ := gout["chunks"].([]map[string]any)
+ var generalImage map[string]any
+ for _, ck := range generalChunks {
+ if ck["doc_type_kwd"] == "image" {
+ generalImage = ck
+ }
+ }
+ if generalImage == nil {
+ t.Fatalf("General emitted no image chunk: %#v", generalChunks)
+ }
+ assertListPagePositions(t, "general", generalImage)
+}
+
+func assertListPagePositions(t *testing.T, name string, ck map[string]any) {
+ t.Helper()
+ want := []any{[]any{[]any{float64(1)}, float64(10), float64(100), float64(10), float64(100)}}
+ for _, key := range []string{"positions", "_pdf_positions"} {
+ raw, err := json.Marshal(ck[key])
+ if err != nil {
+ t.Fatalf("%s %s: marshal: %v", name, key, err)
+ }
+ var got any
+ if err := json.Unmarshal(raw, &got); err != nil {
+ t.Fatalf("%s %s: unmarshal: %v", name, key, err)
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("%s %s = %s, want [[[1],10,100,10,100]]", name, key, raw)
+ }
+ }
+}
diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.go
index 34a2529f98..2db586ed10 100644
--- a/internal/ingestion/component/chunker/title_structured_media_test.go
+++ b/internal/ingestion/component/chunker/title_structured_media_test.go
@@ -136,7 +136,7 @@ func TestTitleFamilyRendersStructuredTableAtomic(t *testing.T) {
if _, ok := ck["table"]; ok {
t.Errorf("structured TableData must not leak onto the chunk: %#v", ck["table"])
}
- if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]float64{{1, 10, 200, 10, 40}}) {
+ if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]any{{float64(1), float64(10), float64(200), float64(10), float64(40)}}) {
t.Errorf("_pdf_positions = %#v, want the source matrix", got)
}
for _, other := range chunks {
@@ -345,7 +345,10 @@ func TestTitleFamilyPreservesSheetIdentity(t *testing.T) {
if _, ok := ck["_pdf_positions"]; ok {
t.Errorf("spreadsheet selections leaked into _pdf_positions: %#v", ck["_pdf_positions"])
}
- if got := ck["positions"]; !reflect.DeepEqual(got, [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}}) {
+ if got := ck["positions"]; !reflect.DeepEqual(got, [][]any{
+ {float64(1), float64(1), float64(1), float64(1), float64(2)},
+ {float64(1), float64(2), float64(2), float64(1), float64(2)},
+ }) {
t.Errorf("positions = %#v, want the source matrix", got)
}
}
diff --git a/internal/ingestion/task/indexdoc/position.go b/internal/ingestion/task/indexdoc/position.go
index c68bb8d074..55799dfd8d 100644
--- a/internal/ingestion/task/indexdoc/position.go
+++ b/internal/ingestion/task/indexdoc/position.go
@@ -34,6 +34,131 @@ func positionsToIntMatrix(positions []float64) [][]int {
return matrix
}
+// positionRowsFromAny normalizes the generic []any shape produced by
+// ChunkDoc.ToMap (JSON decoding into `any`) into [][]any. It accepts a list of
+// rows (each []any/[]float64), a single flat 5-tuple, or a flat sequence of
+// multiple 5-tuples (TCADP/external producers).
+//
+// Element 1 distinguishes the shapes: a row list has another row (or an
+// invalid entry) there, while a flat sequence has a numeric coordinate. A flat
+// sequence is then grouped by five; a length that is not a multiple of five is
+// malformed, matching positionsToIntMatrix.
+func positionRowsFromAny(v []any) ([][]any, bool) {
+ if len(v) == 0 {
+ return nil, false
+ }
+ if len(v) == 1 {
+ if row, ok := positionRowFromAny(v[0]); ok {
+ return [][]any{row}, true
+ }
+ }
+ if len(v) >= 2 {
+ _, secondIsRow := positionRowFromAny(v[1])
+ if secondIsRow || v[1] == nil {
+ _, firstIsRow := positionRowFromAny(v[0])
+ if firstIsRow || v[0] == nil {
+ rows := make([][]any, 0, len(v))
+ for _, e := range v {
+ if row, ok := positionRowFromAny(e); ok {
+ rows = append(rows, row)
+ }
+ }
+ return rows, len(rows) > 0
+ }
+ }
+ }
+ if len(v)%5 != 0 {
+ return nil, false
+ }
+ rows := make([][]any, 0, len(v)/5)
+ for i := 0; i < len(v); i += 5 {
+ rows = append(rows, v[i:i+5])
+ }
+ return rows, len(rows) > 0
+}
+
+func positionRowFromAny(e any) ([]any, bool) {
+ switch row := e.(type) {
+ case []any:
+ return row, true
+ case []float64:
+ out := make([]any, len(row))
+ for i, value := range row {
+ out[i] = value
+ }
+ return out, true
+ default:
+ return nil, false
+ }
+}
+
+// flattenMixedPositions expands a [][]any position matrix into the flat
+// [page,left,right,top,bottom,...] form addPDFPositions expects. This is the
+// shape left by the title-family merge when a page cell is a list (or rows are
+// otherwise non-float); each page in a list becomes its own scalar tuple so
+// page_num_int/top_int can be built without discarding the whole matrix.
+func flattenMixedPositions(rows [][]any) ([]float64, bool) {
+ flat := make([]float64, 0, len(rows)*5)
+ for _, row := range rows {
+ if len(row) < 5 {
+ continue
+ }
+ left, lok := positionNumber(row[1])
+ right, rok := positionNumber(row[2])
+ top, tok := positionNumber(row[3])
+ bottom, bok := positionNumber(row[4])
+ if !lok || !rok || !tok || !bok {
+ continue
+ }
+ pages, ok := positionPages(row[0])
+ if !ok {
+ continue
+ }
+ for _, page := range pages {
+ flat = append(flat, page, left, right, top, bottom)
+ }
+ }
+ return flat, len(flat) > 0
+}
+
+func positionPages(raw any) ([]float64, bool) {
+ switch v := raw.(type) {
+ case []any:
+ pages := make([]float64, 0, len(v))
+ for _, e := range v {
+ page, ok := positionNumber(e)
+ if !ok {
+ return nil, false
+ }
+ pages = append(pages, page)
+ }
+ return pages, len(pages) > 0
+ default:
+ page, ok := positionNumber(v)
+ if !ok {
+ return nil, false
+ }
+ return []float64{page}, true
+ }
+}
+
+func positionNumber(raw any) (float64, bool) {
+ switch v := raw.(type) {
+ case float64:
+ return v, true
+ case float32:
+ return float64(v), true
+ case int:
+ return float64(v), true
+ case int32:
+ return float64(v), true
+ case int64:
+ return float64(v), true
+ default:
+ return 0, false
+ }
+}
+
// addPDFPositions stores PDF layout boxes: positions is a flat []float64
// grouped by five as [page, left, right, top, bottom] and is written to
// page_num_int / top_int / position_int. Page numbers are ALREADY 1-indexed on
diff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.go
index 1415fe7476..e18bab93b2 100644
--- a/internal/ingestion/task/indexdoc/process.go
+++ b/internal/ingestion/task/indexdoc/process.go
@@ -232,6 +232,23 @@ func processChunkPositions(ck map[string]any, spreadsheet bool) {
flat = append(flat, group...)
}
storePositions(ck, flat, spreadsheet)
+ case [][]any:
+ if flat, ok := flattenMixedPositions(v); ok {
+ storePositions(ck, flat, spreadsheet)
+ } else {
+ common.Warn(fmt.Sprintf("chunk positions %T contained no usable rows; discarding", poss))
+ }
+ case []any:
+ rows, ok := positionRowsFromAny(v)
+ if !ok {
+ common.Warn(fmt.Sprintf("chunk positions %T contained no usable rows; discarding", poss))
+ break
+ }
+ if flat, ok := flattenMixedPositions(rows); ok {
+ storePositions(ck, flat, spreadsheet)
+ } else {
+ common.Warn("chunk positions contained no usable rows; discarding")
+ }
default:
common.Warn(fmt.Sprintf("chunk positions unexpected type %T; discarding", poss))
}
diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.go
index 31635a7cf8..a909ad7b19 100644
--- a/internal/ingestion/task/indexdoc/process_test.go
+++ b/internal/ingestion/task/indexdoc/process_test.go
@@ -1,6 +1,7 @@
package indexdoc
import (
+ "encoding/json"
"reflect"
"testing"
"time"
@@ -456,6 +457,135 @@ func TestProcessChunkPositions_2DFloat64(t *testing.T) {
}
}
+// TestProcessChunkPositions_ToMapMixedMatrix pins the real production type
+// boundary: ChunkDoc.ToMap() decodes a list-page matrix through the generic
+// JSON path, so the indexer receives []any (not [][]any). Every valid scalar
+// and list-page row must still reach page_num_int/top_int/position_int.
+func TestProcessChunkPositions_ToMapMixedMatrix(t *testing.T) {
+ raw := json.RawMessage(`[[[1,2],10,100,20,80],[3,30,120,40,90]]`)
+ doc := schema.ChunkDoc{
+ Text: "Figure 1: caption",
+ DocType: "image",
+ Positions: raw,
+ PDFPositions: raw,
+ }
+ ck := doc.ToMap()
+ if _, ok := ck["positions"].([]any); !ok {
+ t.Fatalf("production positions type = %T, want []any", ck["positions"])
+ }
+
+ processChunkPositions(ck, false)
+
+ pageNum, ok := ck["page_num_int"].([]int)
+ if !ok {
+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])
+ }
+ if !reflect.DeepEqual(pageNum, []int{1, 2, 3}) {
+ t.Errorf("page_num_int = %v, want [1 2 3]", pageNum)
+ }
+ top, ok := ck["top_int"].([]int)
+ if !ok {
+ t.Fatalf("top_int = %#v, want []int", ck["top_int"])
+ }
+ if !reflect.DeepEqual(top, []int{20, 20, 40}) {
+ t.Errorf("top_int = %v, want [20 20 40]", top)
+ }
+ matrix, ok := ck["position_int"].([][]int)
+ if !ok || len(matrix) != 3 {
+ t.Fatalf("position_int = %#v, want 3 rows", ck["position_int"])
+ }
+ if !reflect.DeepEqual(matrix[1], []int{2, 10, 100, 20, 80}) {
+ t.Errorf("expanded row = %v, want [2 10 100 20 80]", matrix[1])
+ }
+}
+
+// TestProcessChunkPositions_FlatMultiTuple pins the flat multi-row shape:
+// ChunkDoc.ToMap produces a flat []any of numbers, and every five-tuple must
+// reach the index fields, not just the first.
+func TestProcessChunkPositions_FlatMultiTuple(t *testing.T) {
+ raw := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)
+ doc := schema.ChunkDoc{
+ Text: "two rows",
+ Positions: raw,
+ PDFPositions: raw,
+ }
+ ck := doc.ToMap()
+ processChunkPositions(ck, false)
+
+ pageNum, ok := ck["page_num_int"].([]int)
+ if !ok {
+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])
+ }
+ if !reflect.DeepEqual(pageNum, []int{1, 2}) {
+ t.Errorf("page_num_int = %v, want [1 2]", pageNum)
+ }
+ matrix, ok := ck["position_int"].([][]int)
+ if !ok || len(matrix) != 2 {
+ t.Fatalf("position_int = %#v, want two rows", ck["position_int"])
+ }
+ if !reflect.DeepEqual(matrix[1], []int{2, 30, 120, 40, 90}) {
+ t.Errorf("second row = %v, want [2 30 120 40 90]", matrix[1])
+ }
+}
+
+// TestProcessChunkPositions_SkipsInvalidRows pins the partial-loss guard: a
+// null row must not discard the valid rows around it.
+func TestProcessChunkPositions_SkipsInvalidRows(t *testing.T) {
+ raw := json.RawMessage(`[[1,10,100,20,80],null,[2,30,120,40,90]]`)
+ doc := schema.ChunkDoc{
+ Text: "Figure 1: caption",
+ DocType: "image",
+ Positions: raw,
+ PDFPositions: raw,
+ }
+ ck := doc.ToMap()
+ processChunkPositions(ck, false)
+
+ pageNum, ok := ck["page_num_int"].([]int)
+ if !ok {
+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])
+ }
+ if !reflect.DeepEqual(pageNum, []int{1, 2}) {
+ t.Errorf("page_num_int = %v, want [1 2]", pageNum)
+ }
+ matrix, ok := ck["position_int"].([][]int)
+ if !ok || len(matrix) != 2 {
+ t.Fatalf("position_int = %#v, want the two valid rows", ck["position_int"])
+ }
+}
+
+// TestProcessChunkPositions_MixedPageList covers the matrix shape left by the
+// title-family merge when a row spans pages: a list page expands into one
+// scalar tuple per page instead of discarding the whole matrix.
+func TestProcessChunkPositions_MixedPageList(t *testing.T) {
+ chunk := map[string]any{
+ "positions": [][]any{
+ {[]any{1, 2}, 100, 50, 200, 150},
+ {3, 200, 60, 300, 250},
+ },
+ }
+ processChunkPositions(chunk, false)
+
+ if _, exists := chunk["positions"]; exists {
+ t.Fatal("positions key must be removed")
+ }
+ pageNum := chunk["page_num_int"].([]int)
+ if len(pageNum) != 3 || pageNum[0] != 1 || pageNum[1] != 2 || pageNum[2] != 3 {
+ t.Errorf("page_num_int = %v, want [1 2 3]", pageNum)
+ }
+ top := chunk["top_int"].([]int)
+ if len(top) != 3 || top[0] != 200 || top[1] != 200 || top[2] != 300 {
+ t.Errorf("top_int = %v, want [200 200 300]", top)
+ }
+ matrix, ok := chunk["position_int"].([][]int)
+ if !ok || len(matrix) != 3 {
+ t.Fatalf("position_int = %v, want 3 rows", chunk["position_int"])
+ }
+ if matrix[1][0] != 2 {
+ t.Errorf("expanded row page = %d, want 2", matrix[1][0])
+ }
+}
+
func TestProcessChunkPositions_NoPositions(t *testing.T) {
// _pdf_positions is pruned unconditionally, even on the early-return path
// where "positions" is absent, since the two fields are independent.
diff --git a/internal/parser/parser/pdf_positions.go b/internal/parser/parser/pdf_positions.go
index 6968684ee2..3ec7b0536f 100644
--- a/internal/parser/parser/pdf_positions.go
+++ b/internal/parser/parser/pdf_positions.go
@@ -8,9 +8,13 @@ import "encoding/json"
// supported.
//
// Supported shapes:
-// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64;
+// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64; a page cell
+// may be a scalar or a list (the raw layout.SectionsToJSON form, also
+// seen in external/historical payloads), ordered by its first page;
// - one flat 5-tuple: []float64, the legacy wire form some engines (TCADP)
-// still emit.
+// still emit;
+// - a flat sequence of multiple 5-tuples (TCADP/external producers), grouped
+// by five.
//
// A row is [page, left, right, top, bottom]. The page is 1-based.
@@ -52,27 +56,72 @@ func PDFPositionRows(raw any) [][]any {
}
// PDFPositionRowsFromJSON decodes a PDF position matrix carried as raw JSON.
-// It accepts the matrix form and the legacy flat 5-tuple and returns rows of
-// five floats. Rows shorter than five values are skipped.
+// It accepts the nested matrix form (a page cell may be a scalar or a list),
+// a single flat 5-tuple, and a flat sequence of multiple 5-tuples grouped by
+// five, and returns rows of five floats. Rows shorter than five values, rows
+// with non-numeric coordinates, and flat sequences whose length is not a
+// multiple of five (ambiguous tuple boundaries) are skipped.
func PDFPositionRowsFromJSON(raw json.RawMessage) [][]float64 {
if len(raw) == 0 {
return nil
}
- var matrix [][]float64
- if err := json.Unmarshal(raw, &matrix); err == nil {
- out := make([][]float64, 0, len(matrix))
- for _, row := range matrix {
- if len(row) >= 5 {
- out = append(out, row)
+ var rows [][]any
+ if err := json.Unmarshal(raw, &rows); err == nil {
+ out := make([][]float64, 0, len(rows))
+ for _, row := range rows {
+ if floats, ok := positionRowFloats(row); ok {
+ out = append(out, floats)
}
}
return out
}
- var flat []float64
- if err := json.Unmarshal(raw, &flat); err == nil && len(flat) >= 5 {
- return [][]float64{flat}
+ var flat []any
+ if err := json.Unmarshal(raw, &flat); err != nil || len(flat) < 5 || len(flat)%5 != 0 {
+ return nil
}
- return nil
+ out := make([][]float64, 0, len(flat)/5)
+ for i := 0; i < len(flat); i += 5 {
+ if floats, ok := positionRowFloats(flat[i : i+5]); ok {
+ out = append(out, floats)
+ }
+ }
+ return out
+}
+
+// positionRowFloats decodes one [page,left,right,top,bottom] row for numeric
+// consumers (ordering, crop-region checks). A list page cell is ordered by
+// its first page. The verbatim []any form is kept by the chunker merge for
+// consumers that need the original cell.
+func positionRowFloats(row []any) ([]float64, bool) {
+ if len(row) < 5 {
+ return nil, false
+ }
+ page, ok := positionPageFloat(row[0])
+ if !ok {
+ return nil, false
+ }
+ out := make([]float64, 5)
+ out[0] = page
+ for i := 1; i < 5; i++ {
+ v, ok := positionFloat(row[i])
+ if !ok {
+ return nil, false
+ }
+ out[i] = v
+ }
+ return out, true
+}
+
+// positionPageFloat decodes a page cell: a scalar number or a list whose
+// first entry is numeric.
+func positionPageFloat(raw any) (float64, bool) {
+ if v, ok := positionFloat(raw); ok {
+ return v, true
+ }
+ if pages, ok := raw.([]any); ok && len(pages) > 0 {
+ return positionPageFloat(pages[0])
+ }
+ return 0, false
}
func filterPositionRows(rows [][]any, keep func([]any) []any) [][]any {
diff --git a/internal/parser/parser/pdf_positions_test.go b/internal/parser/parser/pdf_positions_test.go
new file mode 100644
index 0000000000..d884bfda2d
--- /dev/null
+++ b/internal/parser/parser/pdf_positions_test.go
@@ -0,0 +1,38 @@
+package parser
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+)
+
+// TestPDFPositionRowsFromJSONShapes pins the wire shapes the chunker ordering
+// and crop-region checks rely on: scalar and list page cells, a single flat
+// 5-tuple, and a flat sequence of multiple 5-tuples grouped by five.
+// Malformed input reports no rows instead of misaligned keys.
+func TestPDFPositionRowsFromJSONShapes(t *testing.T) {
+ cases := []struct {
+ name string
+ raw string
+ want [][]float64
+ }{
+ {"scalar matrix", `[[1,10,100,20,80],[2,30,120,40,90]]`, [][]float64{{1, 10, 100, 20, 80}, {2, 30, 120, 40, 90}}},
+ {"list page cell", `[[[1],10,100,10,100]]`, [][]float64{{1, 10, 100, 10, 100}}},
+ {"flat single tuple", `[2,5,50,5,50]`, [][]float64{{2, 5, 50, 5, 50}}},
+ {"flat multi tuple", `[1,10,100,20,80,2,30,120,40,90]`, [][]float64{{1, 10, 100, 20, 80}, {2, 30, 120, 40, 90}}},
+ {"malformed flat", `[1,10,100,20,80,2,30]`, nil},
+ {"non numeric row skipped", `[[1,"bad",100,10,100],[2,5,50,5,50]]`, [][]float64{{2, 5, 50, 5, 50}}},
+ {"empty", ``, nil},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := PDFPositionRowsFromJSON(json.RawMessage(tc.raw))
+ if len(got) == 0 && len(tc.want) == 0 {
+ return
+ }
+ if !reflect.DeepEqual(got, tc.want) {
+ t.Fatalf("PDFPositionRowsFromJSON(%s) = %v, want %v", tc.raw, got, tc.want)
+ }
+ })
+ }
+}