RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(chunker): preserve list-form PDF positions in title chunkers (#21258)
jay7772193b314f3061c
diff --git a/internal/ingestion/component/chunker/crop_title_test.go b/internal/ingestion/component/chunker/crop_title_test.goindex 58256d756f..8f402ea953 100644--- a/internal/ingestion/component/chunker/crop_title_test.go+++ b/internal/ingestion/component/chunker/crop_title_test.go@@ -53,6 +53,23 @@ func TestCropTitleChunks_CropsPositionedChunk(t *testing.T) {}}+// TestCropTitleChunks_CropsImageWithPageList covers the image variant: a+// title-family image chunk whose page cell is a list must still be cropped.+func TestCropTitleChunks_CropsImageWithPageList(t *testing.T) {+ pos := [][]any{{[]any{1}, 10, 100, 10, 100}}+ in := []map[string]any{+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},+ }+ got := cropTitleChunks(context.Background(), mockCropEngine{}, in)+ if len(got) != 1 {+ t.Fatalf("len = %d, want 1", len(got))+ }+ img, _ := got[0]["image"].(string)+ if !strings.HasPrefix(img, "data:image/png;base64,") {+ t.Errorf("image = %q, want a cropped data URL", img)+ }+}+// TestCropTitleChunks_NilEnginePassthrough asserts the helper is a no-op// when no PDF engine is available (best-effort contract).func TestCropTitleChunks_NilEnginePassthrough(t *testing.T) {diff --git a/internal/ingestion/component/chunker/general.go b/internal/ingestion/component/chunker/general.goindex 9a63747458..146a3c7f00 100644--- a/internal/ingestion/component/chunker/general.go+++ b/internal/ingestion/component/chunker/general.go@@ -1273,7 +1273,7 @@ func mergeGeneralMetadata(dst *schema.ChunkDoc, src schema.ChunkDoc) {}func mergeGeneralPositions(first, second json.RawMessage) json.RawMessage {- if positions := mergePositionMatrix(first, second); len(positions) > 0 {+ if positions := mergePositionMatrix(first, second); positions != nil {encoded, err := json.Marshal(positions)if err == nil {return encodeddiff --git a/internal/ingestion/component/chunker/group.go b/internal/ingestion/component/chunker/group.goindex dec608e3e0..b35498985e 100644--- a/internal/ingestion/component/chunker/group.go+++ b/internal/ingestion/component/chunker/group.go@@ -420,22 +420,29 @@ func pdfPosRowLess(a, b []float64) bool {// [page,left,right,top,bottom]; duplicates (by the first five columns)// are dropped and rows are sorted by (page, top, left). Returns nil when// no source carries any usable coordinates.-func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {- var out [][]float64+//+// The page cell may be a scalar (deepdoc's normalized form) or a list (the+// raw layout.SectionsToJSON form, also seen in external/historical payloads);+// both are preserved verbatim so the downstream PositionsFromMatrix can decode+// either. A source may also be a single flat 5-tuple rather than a list of+// rows (TCADP and some external producers emit that shape).+//+// Rows stay []any because a page cell may itself be a list; the generic shape+// is what ChunkDoc.ToMap emits for such a matrix, so one representation flows+// through the chunker and the index boundary.+func mergePositionMatrix(sources ...json.RawMessage) [][]any {+ var out [][]anyseen := make(map[string]bool)for _, src := range sources {- if len(src) == 0 {- continue- }- var mat [][]float64- if err := json.Unmarshal(src, &mat); err != nil {+ rows, ok := decodePositionRows(src)+ if !ok {continue}- for _, row := range mat {+ for _, row := range rows {if len(row) < 5 {continue}- key := fmt.Sprintf("%v|%v|%v|%v|%v", row[0], row[1], row[2], row[3], row[4])+ key := fmt.Sprint(row[:5])if seen[key] {continue}@@ -446,12 +453,101 @@ func mergePositionMatrix(sources ...json.RawMessage) [][]float64 {if len(out) == 0 {return nil}- sort.Slice(out, func(i, j int) bool {- return pdfPosRowLess(out[i], out[j])+ sort.SliceStable(out, func(i, j int) bool {+ return positionRowLess(out[i], out[j])})return out}+// decodePositionRows accepts the nested matrix form, a single flat 5-tuple,+// and a flat sequence of multiple 5-tuples (TCADP/external producers emit a+// flat array). JSON-decoded numbers are float64, and a page cell may itself+// be a list, so the rows stay []any rather than being coerced to [][]float64.+//+// A flat sequence whose length is a multiple of five is grouped by five so+// every tuple survives; a flat length that is not a multiple of five has+// ambiguous tuple boundaries and is preserved verbatim as one row so the+// first tuple still reaches cropping (sorting abstains via firstPositionRow).+func decodePositionRows(src json.RawMessage) ([][]any, bool) {+ if len(src) == 0 {+ return nil, false+ }+ var rows [][]any+ if err := json.Unmarshal(src, &rows); err == nil {+ return rows, true+ }+ var flat []any+ if err := json.Unmarshal(src, &flat); err != nil || len(flat) < 5 {+ return nil, false+ }+ if len(flat)%5 != 0 {+ return [][]any{flat}, true+ }+ grouped := make([][]any, 0, len(flat)/5)+ for i := 0; i < len(flat); i += 5 {+ grouped = append(grouped, flat[i:i+5])+ }+ return grouped, true+}++// positionRowLess orders rows by (first page, top, left), matching+// pdfPosRowLess for scalar page numbers while tolerating a list page cell.+func positionRowLess(a, b []any) bool {+ ap, bp := positionRowPage(a), positionRowPage(b)+ if ap != bp {+ return ap < bp+ }+ at, bt := positionRowFloat(a, 3), positionRowFloat(b, 3)+ if at != bt {+ return at < bt+ }+ return positionRowFloat(a, 1) < positionRowFloat(b, 1)+}++// positionRowPage decodes the page cell for ordering only; the original cell+// is kept in the emitted row. A list page uses its first entry.+func positionRowPage(row []any) float64 {+ if len(row) == 0 {+ return 0+ }+ switch v := row[0].(type) {+ case float64:+ return v+ case int:+ return float64(v)+ case []any:+ if len(v) == 0 {+ return 0+ }+ return positionRowPage([]any{v[0]})+ default:+ return 0+ }+}++func positionRowFloat(row []any, index int) float64 {+ if index >= len(row) {+ return 0+ }+ value, _ := positionRowNumber(row[index])+ return value+}++// positionRowNumber decodes one numeric matrix element. JSON-decoded numbers+// are float64; Go-constructed callers may use int/int64.+func positionRowNumber(raw any) (float64, bool) {+ switch v := raw.(type) {+ case float64:+ return v, true+ case int:+ return float64(v), true+ case int64:+ return float64(v), true+ default:+ return 0, false+ }+}+// extractLineRecords converts the decoded upstream payload into records in the// same order the python BaseTitleChunker.extract_line_records uses: the field// named by output_format selects the payload (chunks / json / markdown / textdiff --git a/internal/ingestion/component/chunker/group_test.go b/internal/ingestion/component/chunker/group_test.goindex c14d4e8bbb..1da22158b6 100644--- a/internal/ingestion/component/chunker/group_test.go+++ b/internal/ingestion/component/chunker/group_test.go@@ -367,7 +367,7 @@ func TestGroupChunker_MergesPDFPositionsAndRemovesTags(t *testing.T) {t.Errorf("parser position tags leaked into chunk text: %q", text)}// (b) positions must be merged across both records.- pos, ok := ck["positions"].([][]float64)+ pos, ok := ck["positions"].([][]any)if !ok {t.Fatalf("positions missing or wrong type %T on merged group chunk", ck["positions"])}diff --git a/internal/ingestion/component/chunker/manual_test.go b/internal/ingestion/component/chunker/manual_test.goindex 10bf71c4e5..e896728fa1 100644--- a/internal/ingestion/component/chunker/manual_test.go+++ b/internal/ingestion/component/chunker/manual_test.go@@ -264,7 +264,7 @@ func TestManualChunker_MergesPositions(t *testing.T) {if strings.Contains(text, "@@") {t.Errorf("parser position tags leaked into chunk text: %q", text)}- pos, ok := ck["_pdf_positions"].([][]float64)+ pos, ok := ck["_pdf_positions"].([][]any)if !ok {t.Fatalf("_pdf_positions missing or wrong type %T on merged group chunk", ck["_pdf_positions"])}@@ -274,8 +274,8 @@ func TestManualChunker_MergesPositions(t *testing.T) {t.Fatalf("merged _pdf_positions = %d rows, want 2 (both records)", len(pos))}// Merged matrix must itself be sorted by (page, top, left) — the- // same key pdfPosRowLess uses when merging.- if pdfPosRowLess(pos[1], pos[0]) {+ // same key positionRowLess uses when merging.+ if positionRowLess(pos[1], pos[0]) {t.Errorf("merged _pdf_positions not sorted by (page, top, left): %v", pos)}}diff --git a/internal/ingestion/component/chunker/position_merge_test.go b/internal/ingestion/component/chunker/position_merge_test.gonew file mode 100644index 0000000000..88532829df--- /dev/null+++ b/internal/ingestion/component/chunker/position_merge_test.go@@ -0,0 +1,275 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package chunker++import (+ "encoding/json"+ "reflect"+ "testing"++ "ragflow/internal/ingestion/component/schema"+)++// TestMergePositionMatrixAcceptsPageLists pins the Title-family parity fix:+// positions produced by layout.SectionsToJSON keep the page as a list until+// normalization, and a source may be a single flat 5-tuple (TCADP). Both must+// survive the merge instead of being dropped, so the on-demand PDF crop can+// still run.+func TestMergePositionMatrixAcceptsPageLists(t *testing.T) {+ listPage := json.RawMessage(`[[[1],10,100,10,100]]`)+ flatRow := json.RawMessage(`[2,5,50,5,50]`)+ duplicate := json.RawMessage(`[[[1],10,100,10,100]]`)++ got := mergePositionMatrix(listPage, flatRow, duplicate)+ want := [][]any{+ {[]any{float64(1)}, float64(10), float64(100), float64(10), float64(100)},+ {float64(2), float64(5), float64(50), float64(5), float64(50)},+ }+ if !reflect.DeepEqual(got, want) {+ t.Fatalf("mergePositionMatrix = %#v, want %#v", got, want)+ }+}++// TestMergePositionMatrixPreservesNonNumericRows pins that a malformed row is+// kept verbatim instead of being silently coerced to zeros.+func TestMergePositionMatrixPreservesNonNumericRows(t *testing.T) {+ src := json.RawMessage(`[[1,"bad",100,10,100]]`)+ got := mergePositionMatrix(src)+ if len(got) != 1 {+ t.Fatalf("mergePositionMatrix = %#v, want the raw row", got)+ }+ raw, err := json.Marshal(got)+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ if string(raw) != `[[1,"bad",100,10,100]]` {+ t.Errorf("raw rows = %s, want the source unchanged", raw)+ }+}++// TestTitleFamilyAcceptsFlatRowPositions covers the TCADP shape: a source may+// carry one flat 5-tuple instead of a list of rows. The title merge must nest+// it so the emitted chunk is still [[page,left,right,top,bottom]].+func TestTitleFamilyAcceptsFlatRowPositions(t *testing.T) {+ pos := []float64{1, 10, 100, 10, 100}+ items := []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},+ }+ title, err := NewGroupTitleChunker(map[string]any{"levels": [][]string{{`^# `}}})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ out, err := title.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.pdf", "output_format": "json", "json": items,+ })+ if err != nil {+ t.Fatalf("Invoke: %v", err)+ }+ chunks, _ := out["chunks"].([]map[string]any)+ var image map[string]any+ for _, ck := range chunks {+ if ck["doc_type_kwd"] == "image" {+ image = ck+ }+ }+ if image == nil {+ t.Fatalf("no image chunk emitted: %#v", chunks)+ }+ raw, err := json.Marshal(image["positions"])+ if err != nil {+ t.Fatalf("marshal positions: %v", err)+ }+ if string(raw) != `[[1,10,100,10,100]]` {+ t.Errorf("positions = %s, want [[1,10,100,10,100]]", raw)+ }+}++// TestPositionOrderingAcceptsListPages pins the ordering paths to the same+// shape handling as the merge: a list page cell must be seen as positioned,+// otherwise Manual/General leave records in parser order.+func TestPositionOrderingAcceptsListPages(t *testing.T) {+ first := json.RawMessage(`[[[1],10,100,100,200]]`) // top 100+ second := json.RawMessage(`[[[1],10,100,300,400]]`) // top 300++ recs := []lineRecord{+ {text: "second", pdfPositions: second},+ {text: "first", pdfPositions: first},+ }+ if !hasPdfPositions(recs) {+ t.Fatal("hasPdfPositions = false for list-page positions")+ }+ if row, ok := firstPositionRow(recs[0]); !ok || row[0] != 1 || row[3] != 300 {+ t.Fatalf("firstPositionRow = %v, %v; want page 1 top 300", row, ok)+ }+ sortRecordsByPosition(recs)+ if recs[0].text != "first" || recs[1].text != "second" {+ t.Fatalf("sorted records = %q, %q; want first, second", recs[0].text, recs[1].text)+ }++ units := []schema.ChunkDoc{+ {Text: "second", PDFPositions: second},+ {Text: "first", PDFPositions: first},+ }+ if !hasPDFPositions(units) {+ t.Fatal("hasPDFPositions = false for list-page positions")+ }+ sorted := sortPDFUnits(units)+ if sorted[0].Text != "first" || sorted[1].Text != "second" {+ t.Fatalf("sortPDFUnits = %q, %q; want first, second", sorted[0].Text, sorted[1].Text)+ }+}++// TestMergePositionMatrixGroupsFlatMultiTuples pins the flat multi-tuple+// handling: a flat array carrying several 5-tuples (TCADP/external form) must+// be grouped by five instead of treated as one row, so no tuple is dropped+// and distinct sources sharing a first tuple are not deduplicated into one.+func TestMergePositionMatrixGroupsFlatMultiTuples(t *testing.T) {+ src := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)+ got := mergePositionMatrix(src)+ if len(got) != 2 {+ t.Fatalf("mergePositionMatrix flat multi-tuple = %#v, want 2 rows", got)+ }+ first, err := json.Marshal(got[0])+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ second, err := json.Marshal(got[1])+ if err != nil {+ t.Fatalf("marshal: %v", err)+ }+ if string(first) != `[1,10,100,20,80]` || string(second) != `[2,30,120,40,90]` {+ t.Fatalf("mergePositionMatrix = %s %s, want [1,10,100,20,80] [2,30,120,40,90]", first, second)+ }++ a := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)+ b := json.RawMessage(`[1,10,100,20,80,3,30,120,40,90]`)+ got = mergePositionMatrix(a, b)+ if len(got) != 3 {+ t.Fatalf("mergePositionMatrix distinct flat sources = %#v, want 3 rows", got)+ }+}++// TestPositionOrderingAcceptsFlatMultiTuples pins that a flat multi-tuple+// record participates in physical-order sorting by its first tuple, while a+// malformed flat sequence (ambiguous boundaries) reports unpositioned instead+// of sorting by the wrong keys.+func TestPositionOrderingAcceptsFlatMultiTuples(t *testing.T) {+ multi := json.RawMessage(`[2,30,120,40,90,1,10,100,20,80]`)+ single := json.RawMessage(`[[1,10,100,10,100]]`)++ if row, ok := firstPositionRow(lineRecord{pdfPositions: multi}); !ok || row[0] != 2 || row[3] != 40 {+ t.Fatalf("firstPositionRow flat multi-tuple = %v, %v; want page 2 top 40", row, ok)+ }++ recs := []lineRecord{+ {text: "multi", pdfPositions: multi},+ {text: "single", pdfPositions: single},+ }+ sortRecordsByPosition(recs)+ if recs[0].text != "single" || recs[1].text != "multi" {+ t.Fatalf("sorted records = %q, %q; want single, multi", recs[0].text, recs[1].text)+ }++ units := []schema.ChunkDoc{+ {Text: "multi", PDFPositions: multi},+ {Text: "single", PDFPositions: single},+ }+ sorted := sortPDFUnits(units)+ if sorted[0].Text != "single" || sorted[1].Text != "multi" {+ t.Fatalf("sortPDFUnits = %q, %q; want single, multi", sorted[0].Text, sorted[1].Text)+ }++ malformed := json.RawMessage(`[1,10,100,20,80,2,30]`)+ if _, ok := firstPositionRow(lineRecord{pdfPositions: malformed}); ok {+ t.Fatal("firstPositionRow malformed flat = ok, want false for ambiguous boundaries")+ }+}++// TestTitleFamilyKeepsListPagePositions is the end-to-end guard: an image+// record whose page cell is a list must keep its coordinates on the emitted+// chunk, matching General's pass-through behavior.+func TestTitleFamilyKeepsListPagePositions(t *testing.T) {+ pos := [][]any{{[]any{1}, 10, 100, 10, 100}}+ items := []map[string]any{+ {"text": "before", "doc_type_kwd": "text"},+ {"text": "Figure 1: caption", "doc_type_kwd": "image", "positions": pos, "_pdf_positions": pos},+ }+ levels := [][]string{{`^# `}}++ title, err := NewGroupTitleChunker(map[string]any{"levels": levels})+ if err != nil {+ t.Fatalf("NewGroupTitleChunker: %v", err)+ }+ out, err := title.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.pdf", "output_format": "json", "json": items,+ })+ if err != nil {+ t.Fatalf("Title Invoke: %v", err)+ }+ titleChunks, _ := out["chunks"].([]map[string]any)+ var titleImage map[string]any+ for _, ck := range titleChunks {+ if ck["doc_type_kwd"] == "image" {+ titleImage = ck+ }+ }+ if titleImage == nil {+ t.Fatalf("Title emitted no image chunk: %#v", titleChunks)+ }+ assertListPagePositions(t, "title", titleImage)++ general, err := NewGeneralChunker(map[string]any{"chunk_token_size": 512})+ if err != nil {+ t.Fatalf("NewGeneralChunker: %v", err)+ }+ gout, err := general.Invoke(t.Context(), nil, map[string]any{+ "name": "doc.pdf", "file_type": "pdf", "output_format": "json", "json": items,+ })+ if err != nil {+ t.Fatalf("General Invoke: %v", err)+ }+ generalChunks, _ := gout["chunks"].([]map[string]any)+ var generalImage map[string]any+ for _, ck := range generalChunks {+ if ck["doc_type_kwd"] == "image" {+ generalImage = ck+ }+ }+ if generalImage == nil {+ t.Fatalf("General emitted no image chunk: %#v", generalChunks)+ }+ assertListPagePositions(t, "general", generalImage)+}++func assertListPagePositions(t *testing.T, name string, ck map[string]any) {+ t.Helper()+ want := []any{[]any{[]any{float64(1)}, float64(10), float64(100), float64(10), float64(100)}}+ for _, key := range []string{"positions", "_pdf_positions"} {+ raw, err := json.Marshal(ck[key])+ if err != nil {+ t.Fatalf("%s %s: marshal: %v", name, key, err)+ }+ var got any+ if err := json.Unmarshal(raw, &got); err != nil {+ t.Fatalf("%s %s: unmarshal: %v", name, key, err)+ }+ if !reflect.DeepEqual(got, want) {+ t.Errorf("%s %s = %s, want [[[1],10,100,10,100]]", name, key, raw)+ }+ }+}diff --git a/internal/ingestion/component/chunker/title_structured_media_test.go b/internal/ingestion/component/chunker/title_structured_media_test.goindex 34a2529f98..2db586ed10 100644--- a/internal/ingestion/component/chunker/title_structured_media_test.go+++ b/internal/ingestion/component/chunker/title_structured_media_test.go@@ -136,7 +136,7 @@ func TestTitleFamilyRendersStructuredTableAtomic(t *testing.T) {if _, ok := ck["table"]; ok {t.Errorf("structured TableData must not leak onto the chunk: %#v", ck["table"])}- if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]float64{{1, 10, 200, 10, 40}}) {+ if got := ck["_pdf_positions"]; !reflect.DeepEqual(got, [][]any{{float64(1), float64(10), float64(200), float64(10), float64(40)}}) {t.Errorf("_pdf_positions = %#v, want the source matrix", got)}for _, other := range chunks {@@ -345,7 +345,10 @@ func TestTitleFamilyPreservesSheetIdentity(t *testing.T) {if _, ok := ck["_pdf_positions"]; ok {t.Errorf("spreadsheet selections leaked into _pdf_positions: %#v", ck["_pdf_positions"])}- if got := ck["positions"]; !reflect.DeepEqual(got, [][]float64{{1, 1, 1, 1, 2}, {1, 2, 2, 1, 2}}) {+ if got := ck["positions"]; !reflect.DeepEqual(got, [][]any{+ {float64(1), float64(1), float64(1), float64(1), float64(2)},+ {float64(1), float64(2), float64(2), float64(1), float64(2)},+ }) {t.Errorf("positions = %#v, want the source matrix", got)}}diff --git a/internal/ingestion/task/indexdoc/position.go b/internal/ingestion/task/indexdoc/position.goindex c68bb8d074..55799dfd8d 100644--- a/internal/ingestion/task/indexdoc/position.go+++ b/internal/ingestion/task/indexdoc/position.go@@ -34,6 +34,131 @@ func positionsToIntMatrix(positions []float64) [][]int {return matrix}+// positionRowsFromAny normalizes the generic []any shape produced by+// ChunkDoc.ToMap (JSON decoding into `any`) into [][]any. It accepts a list of+// rows (each []any/[]float64), a single flat 5-tuple, or a flat sequence of+// multiple 5-tuples (TCADP/external producers).+//+// Element 1 distinguishes the shapes: a row list has another row (or an+// invalid entry) there, while a flat sequence has a numeric coordinate. A flat+// sequence is then grouped by five; a length that is not a multiple of five is+// malformed, matching positionsToIntMatrix.+func positionRowsFromAny(v []any) ([][]any, bool) {+ if len(v) == 0 {+ return nil, false+ }+ if len(v) == 1 {+ if row, ok := positionRowFromAny(v[0]); ok {+ return [][]any{row}, true+ }+ }+ if len(v) >= 2 {+ _, secondIsRow := positionRowFromAny(v[1])+ if secondIsRow || v[1] == nil {+ _, firstIsRow := positionRowFromAny(v[0])+ if firstIsRow || v[0] == nil {+ rows := make([][]any, 0, len(v))+ for _, e := range v {+ if row, ok := positionRowFromAny(e); ok {+ rows = append(rows, row)+ }+ }+ return rows, len(rows) > 0+ }+ }+ }+ if len(v)%5 != 0 {+ return nil, false+ }+ rows := make([][]any, 0, len(v)/5)+ for i := 0; i < len(v); i += 5 {+ rows = append(rows, v[i:i+5])+ }+ return rows, len(rows) > 0+}++func positionRowFromAny(e any) ([]any, bool) {+ switch row := e.(type) {+ case []any:+ return row, true+ case []float64:+ out := make([]any, len(row))+ for i, value := range row {+ out[i] = value+ }+ return out, true+ default:+ return nil, false+ }+}++// flattenMixedPositions expands a [][]any position matrix into the flat+// [page,left,right,top,bottom,...] form addPDFPositions expects. This is the+// shape left by the title-family merge when a page cell is a list (or rows are+// otherwise non-float); each page in a list becomes its own scalar tuple so+// page_num_int/top_int can be built without discarding the whole matrix.+func flattenMixedPositions(rows [][]any) ([]float64, bool) {+ flat := make([]float64, 0, len(rows)*5)+ for _, row := range rows {+ if len(row) < 5 {+ continue+ }+ left, lok := positionNumber(row[1])+ right, rok := positionNumber(row[2])+ top, tok := positionNumber(row[3])+ bottom, bok := positionNumber(row[4])+ if !lok || !rok || !tok || !bok {+ continue+ }+ pages, ok := positionPages(row[0])+ if !ok {+ continue+ }+ for _, page := range pages {+ flat = append(flat, page, left, right, top, bottom)+ }+ }+ return flat, len(flat) > 0+}++func positionPages(raw any) ([]float64, bool) {+ switch v := raw.(type) {+ case []any:+ pages := make([]float64, 0, len(v))+ for _, e := range v {+ page, ok := positionNumber(e)+ if !ok {+ return nil, false+ }+ pages = append(pages, page)+ }+ return pages, len(pages) > 0+ default:+ page, ok := positionNumber(v)+ if !ok {+ return nil, false+ }+ return []float64{page}, true+ }+}++func positionNumber(raw any) (float64, bool) {+ switch v := raw.(type) {+ case float64:+ return v, true+ case float32:+ return float64(v), true+ case int:+ return float64(v), true+ case int32:+ return float64(v), true+ case int64:+ return float64(v), true+ default:+ return 0, false+ }+}+// addPDFPositions stores PDF layout boxes: positions is a flat []float64// grouped by five as [page, left, right, top, bottom] and is written to// page_num_int / top_int / position_int. Page numbers are ALREADY 1-indexed ondiff --git a/internal/ingestion/task/indexdoc/process.go b/internal/ingestion/task/indexdoc/process.goindex 1415fe7476..e18bab93b2 100644--- a/internal/ingestion/task/indexdoc/process.go+++ b/internal/ingestion/task/indexdoc/process.go@@ -232,6 +232,23 @@ func processChunkPositions(ck map[string]any, spreadsheet bool) {flat = append(flat, group...)}storePositions(ck, flat, spreadsheet)+ case [][]any:+ if flat, ok := flattenMixedPositions(v); ok {+ storePositions(ck, flat, spreadsheet)+ } else {+ common.Warn(fmt.Sprintf("chunk positions %T contained no usable rows; discarding", poss))+ }+ case []any:+ rows, ok := positionRowsFromAny(v)+ if !ok {+ common.Warn(fmt.Sprintf("chunk positions %T contained no usable rows; discarding", poss))+ break+ }+ if flat, ok := flattenMixedPositions(rows); ok {+ storePositions(ck, flat, spreadsheet)+ } else {+ common.Warn("chunk positions contained no usable rows; discarding")+ }default:common.Warn(fmt.Sprintf("chunk positions unexpected type %T; discarding", poss))}diff --git a/internal/ingestion/task/indexdoc/process_test.go b/internal/ingestion/task/indexdoc/process_test.goindex 31635a7cf8..a909ad7b19 100644--- a/internal/ingestion/task/indexdoc/process_test.go+++ b/internal/ingestion/task/indexdoc/process_test.go@@ -1,6 +1,7 @@package indexdocimport (+ "encoding/json""reflect""testing""time"@@ -456,6 +457,135 @@ func TestProcessChunkPositions_2DFloat64(t *testing.T) {}}+// TestProcessChunkPositions_ToMapMixedMatrix pins the real production type+// boundary: ChunkDoc.ToMap() decodes a list-page matrix through the generic+// JSON path, so the indexer receives []any (not [][]any). Every valid scalar+// and list-page row must still reach page_num_int/top_int/position_int.+func TestProcessChunkPositions_ToMapMixedMatrix(t *testing.T) {+ raw := json.RawMessage(`[[[1,2],10,100,20,80],[3,30,120,40,90]]`)+ doc := schema.ChunkDoc{+ Text: "Figure 1: caption",+ DocType: "image",+ Positions: raw,+ PDFPositions: raw,+ }+ ck := doc.ToMap()+ if _, ok := ck["positions"].([]any); !ok {+ t.Fatalf("production positions type = %T, want []any", ck["positions"])+ }++ processChunkPositions(ck, false)++ pageNum, ok := ck["page_num_int"].([]int)+ if !ok {+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])+ }+ if !reflect.DeepEqual(pageNum, []int{1, 2, 3}) {+ t.Errorf("page_num_int = %v, want [1 2 3]", pageNum)+ }+ top, ok := ck["top_int"].([]int)+ if !ok {+ t.Fatalf("top_int = %#v, want []int", ck["top_int"])+ }+ if !reflect.DeepEqual(top, []int{20, 20, 40}) {+ t.Errorf("top_int = %v, want [20 20 40]", top)+ }+ matrix, ok := ck["position_int"].([][]int)+ if !ok || len(matrix) != 3 {+ t.Fatalf("position_int = %#v, want 3 rows", ck["position_int"])+ }+ if !reflect.DeepEqual(matrix[1], []int{2, 10, 100, 20, 80}) {+ t.Errorf("expanded row = %v, want [2 10 100 20 80]", matrix[1])+ }+}++// TestProcessChunkPositions_FlatMultiTuple pins the flat multi-row shape:+// ChunkDoc.ToMap produces a flat []any of numbers, and every five-tuple must+// reach the index fields, not just the first.+func TestProcessChunkPositions_FlatMultiTuple(t *testing.T) {+ raw := json.RawMessage(`[1,10,100,20,80,2,30,120,40,90]`)+ doc := schema.ChunkDoc{+ Text: "two rows",+ Positions: raw,+ PDFPositions: raw,+ }+ ck := doc.ToMap()+ processChunkPositions(ck, false)++ pageNum, ok := ck["page_num_int"].([]int)+ if !ok {+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])+ }+ if !reflect.DeepEqual(pageNum, []int{1, 2}) {+ t.Errorf("page_num_int = %v, want [1 2]", pageNum)+ }+ matrix, ok := ck["position_int"].([][]int)+ if !ok || len(matrix) != 2 {+ t.Fatalf("position_int = %#v, want two rows", ck["position_int"])+ }+ if !reflect.DeepEqual(matrix[1], []int{2, 30, 120, 40, 90}) {+ t.Errorf("second row = %v, want [2 30 120 40 90]", matrix[1])+ }+}++// TestProcessChunkPositions_SkipsInvalidRows pins the partial-loss guard: a+// null row must not discard the valid rows around it.+func TestProcessChunkPositions_SkipsInvalidRows(t *testing.T) {+ raw := json.RawMessage(`[[1,10,100,20,80],null,[2,30,120,40,90]]`)+ doc := schema.ChunkDoc{+ Text: "Figure 1: caption",+ DocType: "image",+ Positions: raw,+ PDFPositions: raw,+ }+ ck := doc.ToMap()+ processChunkPositions(ck, false)++ pageNum, ok := ck["page_num_int"].([]int)+ if !ok {+ t.Fatalf("page_num_int = %#v, want []int", ck["page_num_int"])+ }+ if !reflect.DeepEqual(pageNum, []int{1, 2}) {+ t.Errorf("page_num_int = %v, want [1 2]", pageNum)+ }+ matrix, ok := ck["position_int"].([][]int)+ if !ok || len(matrix) != 2 {+ t.Fatalf("position_int = %#v, want the two valid rows", ck["position_int"])+ }+}++// TestProcessChunkPositions_MixedPageList covers the matrix shape left by the+// title-family merge when a row spans pages: a list page expands into one+// scalar tuple per page instead of discarding the whole matrix.+func TestProcessChunkPositions_MixedPageList(t *testing.T) {+ chunk := map[string]any{+ "positions": [][]any{+ {[]any{1, 2}, 100, 50, 200, 150},+ {3, 200, 60, 300, 250},+ },+ }+ processChunkPositions(chunk, false)++ if _, exists := chunk["positions"]; exists {+ t.Fatal("positions key must be removed")+ }+ pageNum := chunk["page_num_int"].([]int)+ if len(pageNum) != 3 || pageNum[0] != 1 || pageNum[1] != 2 || pageNum[2] != 3 {+ t.Errorf("page_num_int = %v, want [1 2 3]", pageNum)+ }+ top := chunk["top_int"].([]int)+ if len(top) != 3 || top[0] != 200 || top[1] != 200 || top[2] != 300 {+ t.Errorf("top_int = %v, want [200 200 300]", top)+ }+ matrix, ok := chunk["position_int"].([][]int)+ if !ok || len(matrix) != 3 {+ t.Fatalf("position_int = %v, want 3 rows", chunk["position_int"])+ }+ if matrix[1][0] != 2 {+ t.Errorf("expanded row page = %d, want 2", matrix[1][0])+ }+}+func TestProcessChunkPositions_NoPositions(t *testing.T) {// _pdf_positions is pruned unconditionally, even on the early-return path// where "positions" is absent, since the two fields are independent.diff --git a/internal/parser/parser/pdf_positions.go b/internal/parser/parser/pdf_positions.goindex 6968684ee2..3ec7b0536f 100644--- a/internal/parser/parser/pdf_positions.go+++ b/internal/parser/parser/pdf_positions.go@@ -8,9 +8,13 @@ import "encoding/json"// supported.//// Supported shapes:-// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64;+// - a matrix of 5-tuples: [][]any, []any of rows, [][]float64; a page cell+// may be a scalar or a list (the raw layout.SectionsToJSON form, also+// seen in external/historical payloads), ordered by its first page;// - one flat 5-tuple: []float64, the legacy wire form some engines (TCADP)-// still emit.+// still emit;+// - a flat sequence of multiple 5-tuples (TCADP/external producers), grouped+// by five.//// A row is [page, left, right, top, bottom]. The page is 1-based.@@ -52,27 +56,72 @@ func PDFPositionRows(raw any) [][]any {}// PDFPositionRowsFromJSON decodes a PDF position matrix carried as raw JSON.-// It accepts the matrix form and the legacy flat 5-tuple and returns rows of-// five floats. Rows shorter than five values are skipped.+// It accepts the nested matrix form (a page cell may be a scalar or a list),+// a single flat 5-tuple, and a flat sequence of multiple 5-tuples grouped by+// five, and returns rows of five floats. Rows shorter than five values, rows+// with non-numeric coordinates, and flat sequences whose length is not a+// multiple of five (ambiguous tuple boundaries) are skipped.func PDFPositionRowsFromJSON(raw json.RawMessage) [][]float64 {if len(raw) == 0 {return nil}- var matrix [][]float64- if err := json.Unmarshal(raw, &matrix); err == nil {- out := make([][]float64, 0, len(matrix))- for _, row := range matrix {- if len(row) >= 5 {- out = append(out, row)+ var rows [][]any+ if err := json.Unmarshal(raw, &rows); err == nil {+ out := make([][]float64, 0, len(rows))+ for _, row := range rows {+ if floats, ok := positionRowFloats(row); ok {+ out = append(out, floats)}}return out}- var flat []float64- if err := json.Unmarshal(raw, &flat); err == nil && len(flat) >= 5 {- return [][]float64{flat}+ var flat []any+ if err := json.Unmarshal(raw, &flat); err != nil || len(flat) < 5 || len(flat)%5 != 0 {+ return nil}- return nil+ out := make([][]float64, 0, len(flat)/5)+ for i := 0; i < len(flat); i += 5 {+ if floats, ok := positionRowFloats(flat[i : i+5]); ok {+ out = append(out, floats)+ }+ }+ return out+}++// positionRowFloats decodes one [page,left,right,top,bottom] row for numeric+// consumers (ordering, crop-region checks). A list page cell is ordered by+// its first page. The verbatim []any form is kept by the chunker merge for+// consumers that need the original cell.+func positionRowFloats(row []any) ([]float64, bool) {+ if len(row) < 5 {+ return nil, false+ }+ page, ok := positionPageFloat(row[0])+ if !ok {+ return nil, false+ }+ out := make([]float64, 5)+ out[0] = page+ for i := 1; i < 5; i++ {+ v, ok := positionFloat(row[i])+ if !ok {+ return nil, false+ }+ out[i] = v+ }+ return out, true+}++// positionPageFloat decodes a page cell: a scalar number or a list whose+// first entry is numeric.+func positionPageFloat(raw any) (float64, bool) {+ if v, ok := positionFloat(raw); ok {+ return v, true+ }+ if pages, ok := raw.([]any); ok && len(pages) > 0 {+ return positionPageFloat(pages[0])+ }+ return 0, false}func filterPositionRows(rows [][]any, keep func([]any) []any) [][]any {diff --git a/internal/parser/parser/pdf_positions_test.go b/internal/parser/parser/pdf_positions_test.gonew file mode 100644index 0000000000..d884bfda2d--- /dev/null+++ b/internal/parser/parser/pdf_positions_test.go@@ -0,0 +1,38 @@+package parser++import (+ "encoding/json"+ "reflect"+ "testing"+)++// TestPDFPositionRowsFromJSONShapes pins the wire shapes the chunker ordering+// and crop-region checks rely on: scalar and list page cells, a single flat+// 5-tuple, and a flat sequence of multiple 5-tuples grouped by five.+// Malformed input reports no rows instead of misaligned keys.+func TestPDFPositionRowsFromJSONShapes(t *testing.T) {+ cases := []struct {+ name string+ raw string+ want [][]float64+ }{+ {"scalar matrix", `[[1,10,100,20,80],[2,30,120,40,90]]`, [][]float64{{1, 10, 100, 20, 80}, {2, 30, 120, 40, 90}}},+ {"list page cell", `[[[1],10,100,10,100]]`, [][]float64{{1, 10, 100, 10, 100}}},+ {"flat single tuple", `[2,5,50,5,50]`, [][]float64{{2, 5, 50, 5, 50}}},+ {"flat multi tuple", `[1,10,100,20,80,2,30,120,40,90]`, [][]float64{{1, 10, 100, 20, 80}, {2, 30, 120, 40, 90}}},+ {"malformed flat", `[1,10,100,20,80,2,30]`, nil},+ {"non numeric row skipped", `[[1,"bad",100,10,100],[2,5,50,5,50]]`, [][]float64{{2, 5, 50, 5, 50}}},+ {"empty", ``, nil},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got := PDFPositionRowsFromJSON(json.RawMessage(tc.raw))+ if len(got) == 0 && len(tc.want) == 0 {+ return+ }+ if !reflect.DeepEqual(got, tc.want) {+ t.Fatalf("PDFPositionRowsFromJSON(%s) = %v, want %v", tc.raw, got, tc.want)+ }+ })+ }+}