RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): carry OpenDataLoader PDF positions through to parse items (#21450)
CCLUCKY2eae6eb63b04
diff --git a/internal/parser/parser/pdf_parser_opendataloader.go b/internal/parser/parser/pdf_parser_opendataloader.goindex cdb448cff9..daa1084b41 100644--- a/internal/parser/parser/pdf_parser_opendataloader.go+++ b/internal/parser/parser/pdf_parser_opendataloader.go@@ -5,10 +5,12 @@ import ("encoding/json""fmt""io"+ "math""mime/multipart""net/http""ragflow/internal/common""ragflow/internal/deepdoc/parser/pdf/table"+ "sort""strings")@@ -62,9 +64,20 @@ func parsePDFWithOpenDataLoader(ctx context.Context, filename string, data []bytreturn ParseResult{Err: fmt.Errorf("parser: OpenDataLoader decode: %w", err)}}if payload.JSONDoc != nil {- items := openDataLoaderItems(payload.JSONDoc)+ sizes, pageCount, geometryErr := openDataLoaderPageGeometry(ctx, data, openDataLoaderPageIndexes(payload.JSONDoc))+ items, boxWarnings := openDataLoaderItemsWithGeometry(payload.JSONDoc, sizes)if len(items) > 0 {- return pdfItemsToResult(filename, items, openDataLoaderPageCount(payload.JSONDoc))+ if pageCount <= 0 {+ // No pdfium geometry (nocgo, or a document it cannot load):+ // fall back to the pages the service reported.+ pageCount = openDataLoaderReportedPageCount(payload.JSONDoc)+ }+ res := pdfItemsToResult(filename, items, pageCount)+ if geometryErr != nil {+ res.Warnings = append(res.Warnings, "OpenDataLoader PDF geometry unavailable: "+geometryErr.Error())+ }+ res.Warnings = append(res.Warnings, boxWarnings...)+ return res}}if strings.TrimSpace(payload.MDText) != "" {@@ -102,15 +115,28 @@ func openDataLoaderMultipart(filename string, data []byte, parser *PDFParser) (ireturn strings.NewReader(body.String()), writer.FormDataContentType(), nil}-func openDataLoaderItems(root any) []map[string]any {+// openDataLoaderItemsWithGeometry flattens every typed element into a parse+// item. pageSizes maps a 0-based page index to the PDF's page size in points;+// when it is nil (nocgo, or a document pdfium cannot load) the items keep+// their text and the reported page number, but carry no positions. The second+// return value reports the pages whose boxes did not fit that page size.+func openDataLoaderItemsWithGeometry(root any, pageSizes map[int][2]float64) ([]map[string]any, []string) {items := []map[string]any{}+ // One warning per page: a page whose boxes disagree with the point space+ // is a coordinate-space mismatch worth surfacing, but repeating it per+ // element would bury the rest of the parse result in noise.+ reported := map[int]string{}var walk func(any)walk = func(node any) {switch v := node.(type) {case map[string]any:- if item := openDataLoaderNodeToItem(v); item != nil {+ item, page, warning := openDataLoaderNodeToItem(v, pageSizes)+ if item != nil {items = append(items, item)}+ if warning != "" && reported[page] == "" {+ reported[page] = warning+ }for _, child := range v {walk(child)}@@ -121,14 +147,23 @@ func openDataLoaderItems(root any) []map[string]any {}}walk(root)- return items+ pages := make([]int, 0, len(reported))+ for page := range reported {+ pages = append(pages, page)+ }+ sort.Ints(pages)+ warnings := make([]string, 0, len(pages))+ for _, page := range pages {+ warnings = append(warnings, reported[page])+ }+ return items, warnings}-func openDataLoaderNodeToItem(el map[string]any) map[string]any {+func openDataLoaderNodeToItem(el map[string]any, pageSizes map[int][2]float64) (map[string]any, int, string) {rawType, _ := el["type"].(string)t := strings.ToLower(strings.TrimSpace(rawType))if t == "" {- return nil+ return nil, 0, ""}text := strings.TrimSpace(stringValue(el["content"]))if text == "" {@@ -149,31 +184,158 @@ func openDataLoaderNodeToItem(el map[string]any) map[string]any {text = html}if text == "" {- return nil+ return nil, 0, ""}item := map[string]any{"text": text, "doc_type_kwd": pdfTableDocType(text), "layout": "table"}projectPDFTable(item)- return item+ return openDataLoaderWithPositions(item, el, pageSizes)case "image", "picture", "figure":if text == "" {text = "[Image]"}- return map[string]any{"text": text, "doc_type_kwd": "image", "layout": "figure"}+ return openDataLoaderWithPositions(map[string]any{"text": text, "doc_type_kwd": "image", "layout": "figure"}, el, pageSizes)case "formula", "equation":if text == "" {- return nil+ return nil, 0, ""}- return map[string]any{"text": text, "doc_type_kwd": "text", "layout": "equation"}+ return openDataLoaderWithPositions(map[string]any{"text": text, "doc_type_kwd": "text", "layout": "equation"}, el, pageSizes)default:if text == "" {- return nil+ return nil, 0, ""}layout := "text"if t == "title" || t == "heading" {layout = "title"}- return map[string]any{"text": text, "doc_type_kwd": "text", "layout": layout}+ return openDataLoaderWithPositions(map[string]any{"text": text, "doc_type_kwd": "text", "layout": layout}, el, pageSizes)+ }+}++func openDataLoaderWithPositions(item map[string]any, el map[string]any, pageSizes map[int][2]float64) (map[string]any, int, string) {+ page, ok := openDataLoaderPageNumber(el)+ if !ok {+ return item, 0, ""+ }+ // The page identity survives without geometry (nocgo, or a missing box),+ // matching the PaddleOCR adapter: downstream chunking and croppers can+ // still attribute the item to its page.+ item["page_number"] = page+ positions, warning := openDataLoaderPositions(page, el, pageSizes)+ if len(positions) > 0 {+ item["positions"], item["_pdf_positions"] = positions, positions+ }+ return item, page, warning+}++// openDataLoaderBleedTolerance is how far past the page edge a box may reach,+// as a fraction of that page dimension. Printed content bleeds slightly; a box+// further out than this is not a bleed but a disagreement about the coordinate+// space, so its positions are dropped and reported rather than clamped.+const openDataLoaderBleedTolerance = 0.1++// openDataLoaderPositions converts one element's geometry into RAGFlow's+// canonical [page, left, right, top, bottom] row. OpenDataLoader reports+// [left, bottom, right, top] in PDF points with the origin at the bottom-left,+// while positions are stored in screen space with the origin at the top-left,+// so the vertical edges are flipped against the page height. Without the page+// size the flip cannot be computed, and a wrong-y highlight is worse than none.+// The flip assumes both sides describe the same (post-rotation) page box, which+// holds for the page sizes pdfium reports; the point space itself is a service+// contract this adapter cannot read, so a box that cannot belong to the page is+// treated as evidence of a different space and reported through the second+// return value.+func openDataLoaderPositions(page int, el map[string]any, pageSizes map[int][2]float64) ([][]any, string) {+ size, ok := pageSizes[page-1]+ if !ok || size[0] <= 0 || size[1] <= 0 {+ return nil, ""+ }+ box, ok := openDataLoaderBox(el)+ if !ok {+ return nil, ""+ }+ left, bottom, right, top := box[0], box[1], box[2], box[3]+ if right <= left || top <= bottom {+ return nil, ""+ }+ overflowRight, overflowBottom := math.Max(0, math.Max(-left, right-size[0])), math.Max(0, math.Max(-bottom, top-size[1]))+ if overflowRight > size[0]*openDataLoaderBleedTolerance || overflowBottom > size[1]*openDataLoaderBleedTolerance {+ return nil, fmt.Sprintf("OpenDataLoader page %d: element box [%g %g %g %g] does not fit page size %gx%g; positions omitted because the service coordinates do not match the PDF point space", page, left, bottom, right, top, size[0], size[1])+ }+ left = math.Max(0, math.Min(size[0], left))+ right = math.Max(0, math.Min(size[0], right))+ screenTop := math.Max(0, math.Min(size[1], size[1]-top))+ screenBottom := math.Max(0, math.Min(size[1], size[1]-bottom))+ if right <= left || screenBottom <= screenTop {+ return nil, ""+ }+ return [][]any{{float64(page), left, right, screenTop, screenBottom}}, ""+}++// openDataLoaderReportedPageCount is the page-count fallback when pdfium is+// unavailable: prefer the root's page total, then the highest page number any+// element reports (a sparse document must not be counted by distinct pages),+// and finally one, because items exist.+func openDataLoaderReportedPageCount(root any) int {+ if top, ok := root.(map[string]any); ok {+ for _, key := range []string{"number of pages", "number_of_pages", "page count", "page_count"} {+ if count := int(numberValue(top[key])); count > 0 {+ return count+ }+ }}+ count := 0+ for page := range collectPDFPageNumbers(root) {+ if page > count {+ count = page+ }+ }+ if count == 0 {+ count = 1+ }+ return count+}++// openDataLoaderPageIndexes lists the zero-based pages the service reported+// content for, so the geometry lookup only opens those pages.+func openDataLoaderPageIndexes(root any) []int {+ pages := collectPDFPageNumbers(root)+ indexes := make([]int, 0, len(pages))+ for page := range pages {+ indexes = append(indexes, page-1)+ }+ sort.Ints(indexes)+ return indexes+}++func openDataLoaderPageNumber(el map[string]any) (int, bool) {+ for _, key := range []string{"page number", "page_number", "page"} {+ if raw, ok := el[key]; ok {+ n := numberValue(raw)+ if page := int(n); n > 0 && float64(page) == n {+ return page, true+ }+ }+ }+ return 0, false+}++func openDataLoaderBox(el map[string]any) ([4]float64, bool) {+ var box [4]float64+ for _, key := range []string{"bounding box", "bounding_box", "bbox"} {+ coords, ok := el[key].([]any)+ if !ok || len(coords) < 4 {+ continue+ }+ for i := 0; i < 4; i++ {+ v, ok := numericAny(coords[i])+ if !ok {+ return box, false+ }+ box[i] = v+ }+ return box, true+ }+ return box, false}func openDataLoaderCellRows(raw any) [][]string {@@ -209,17 +371,6 @@ func openDataLoaderCellRows(raw any) [][]string {return ordered}-func openDataLoaderPageCount(root any) int {- pages := collectPDFPageNumbers(root)- if len(pages) > 0 {- return len(pages)- }- if root != nil {- return 1- }- return 0-}-func stringValue(v any) string {s, _ := v.(string)return sdiff --git a/internal/parser/parser/pdf_parser_opendataloader_cgo.go b/internal/parser/parser/pdf_parser_opendataloader_cgo.gonew file mode 100644index 0000000000..7613da2433--- /dev/null+++ b/internal/parser/parser/pdf_parser_opendataloader_cgo.go@@ -0,0 +1,17 @@+//go:build cgo++package parser++import (+ "context"++ "ragflow/internal/deepdoc/parser/pdf/pdfium"+)++// openDataLoaderPageGeometry returns the requested pages' sizes in PDF points+// and the document page count, which the adapter needs to flip+// OpenDataLoader's bottom-left coordinates into the screen-space rows+// positions use. One document load serves both values.+func openDataLoaderPageGeometry(ctx context.Context, data []byte, indexes []int) (map[int][2]float64, int, error) {+ return pdfium.PageSizes(ctx, data, indexes)+}diff --git a/internal/parser/parser/pdf_parser_opendataloader_cgo_test.go b/internal/parser/parser/pdf_parser_opendataloader_cgo_test.gonew file mode 100644index 0000000000..4b45e77edc--- /dev/null+++ b/internal/parser/parser/pdf_parser_opendataloader_cgo_test.go@@ -0,0 +1,67 @@+//go:build cgo++package parser++import (+ "io"+ "math"+ "net/http"+ "net/http/httptest"+ "os"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/pdfium"+)++func TestPDFParser_ParseWithResult_OpenDataLoaderPositions(t *testing.T) {+ withSSRFBypass(t)+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")+ if err != nil {+ t.Fatal(err)+ }+ sizes, pageCount, err := pdfium.PageSizes(t.Context(), data, []int{0})+ if err != nil || pageCount == 0 {+ t.Fatalf("PageSizes: sizes=%v count=%d err=%v", sizes, pageCount, err)+ }+ pageHeight := sizes[0][1]++ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if _, err := io.Copy(io.Discard, r.Body); err != nil {+ t.Errorf("drain body: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"json_doc":{"type":"paragraph","content":"ODL Body","page number":1,"bounding box":[72,650.5,540,700.2]}}`))+ }))+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "OpenDataLoader",+ "opendataloader_apiserver": server.URL,+ })+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got, want := res.File["page_count"], pageCount; got != want {+ t.Errorf("page_count = %v, want %d (pdfium)", got, want)+ }+ if len(res.Warnings) != 0 {+ t.Errorf("warnings = %v, want none when geometry is available", res.Warnings)+ }+ item := res.JSON[0].ToMap()+ rows, ok := item["_pdf_positions"].([][]any)+ if !ok || len(rows) != 1 {+ t.Fatalf("_pdf_positions = %#v, want one row", item["_pdf_positions"])+ }+ if page, ok := numericAny(rows[0][0]); !ok || page != 1 {+ t.Fatalf("page = %#v, want 1 (positions normalize the page to int)", rows[0][0])+ }+ want := []float64{72, 540, pageHeight - 700.2, pageHeight - 650.5}+ for i := range want {+ got, ok := rows[0][i+1].(float64)+ if !ok || math.Abs(got-want[i]) > 1e-6 {+ t.Fatalf("_pdf_positions = %#v, want %v", rows, want)+ }+ }+}diff --git a/internal/parser/parser/pdf_parser_opendataloader_nocgo.go b/internal/parser/parser/pdf_parser_opendataloader_nocgo.gonew file mode 100644index 0000000000..efb54bdd33--- /dev/null+++ b/internal/parser/parser/pdf_parser_opendataloader_nocgo.go@@ -0,0 +1,15 @@+//go:build !cgo++package parser++import (+ "context"+ "fmt"+)++func openDataLoaderPageGeometry(ctx context.Context, _ []byte, _ []int) (map[int][2]float64, int, error) {+ if err := ctx.Err(); err != nil {+ return nil, 0, err+ }+ return nil, 0, fmt.Errorf("PDF page geometry requires cgo")+}diff --git a/internal/parser/parser/pdf_parser_opendataloader_nocgo_test.go b/internal/parser/parser/pdf_parser_opendataloader_nocgo_test.gonew file mode 100644index 0000000000..9e5ba09d43--- /dev/null+++ b/internal/parser/parser/pdf_parser_opendataloader_nocgo_test.go@@ -0,0 +1,76 @@+//go:build !cgo++package parser++import (+ "io"+ "net/http"+ "net/http/httptest"+ "os"+ "strings"+ "testing"+)++func TestPDFParser_ParseWithResult_OpenDataLoaderWithoutGeometry(t *testing.T) {+ withSSRFBypass(t)+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")+ if err != nil {+ t.Fatal(err)+ }+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if _, err := io.Copy(io.Discard, r.Body); err != nil {+ t.Errorf("drain body: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"json_doc":{"type":"paragraph","content":"ODL Body","page number":1,"bounding box":[72,650.5,540,700.2]}}`))+ }))+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "OpenDataLoader",+ "opendataloader_apiserver": server.URL,+ })+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ item := res.JSON[0].ToMap()+ if item["text"] != "ODL Body" {+ t.Errorf("text = %v, want ODL Body", item["text"])+ }+ if item["_pdf_positions"] != nil {+ t.Errorf("_pdf_positions = %#v, want none without page geometry", item["_pdf_positions"])+ }+ if page, ok := numericAny(item["page_number"]); !ok || page != 1 {+ t.Errorf("page_number = %#v, want 1 (page identity survives without geometry)", item["page_number"])+ }+ if got, want := res.File["page_count"], 1; got != want {+ t.Errorf("page_count = %v, want %d (pages reported by the service)", got, want)+ }+ if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "requires cgo") {+ t.Errorf("warnings = %v, want one geometry warning", res.Warnings)+ }++ // A response without page keys still yields items; the document must not+ // report zero pages because nothing pins the count down.+ pageLess := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if _, err := io.Copy(io.Discard, r.Body); err != nil {+ t.Errorf("drain body: %v", err)+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"json_doc":{"type":"document","number of pages":3,"children":[{"type":"paragraph","content":"No page"}]}}`))+ }))+ defer pageLess.Close()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "OpenDataLoader",+ "opendataloader_apiserver": pageLess.URL,+ })+ pageLessRes := pdf.ParseWithResult(t.Context(), "sample.pdf", data)+ if pageLessRes.Err != nil {+ t.Fatalf("ParseWithResult (page-less): %v", pageLessRes.Err)+ }+ if got, want := pageLessRes.File["page_count"], 3; got != want {+ t.Errorf("page_count = %v, want %d from the service's page total", got, want)+ }+}diff --git a/internal/parser/parser/pdf_parser_opendataloader_positions_test.go b/internal/parser/parser/pdf_parser_opendataloader_positions_test.gonew file mode 100644index 0000000000..4a2e22fae0--- /dev/null+++ b/internal/parser/parser/pdf_parser_opendataloader_positions_test.go@@ -0,0 +1,205 @@+package parser++import (+ "math"+ "strings"+ "testing"+)++// OpenDataLoader reports [left, bottom, right, top] in PDF points (origin+// bottom-left); RAGFlow stores [page, left, right, top, bottom] in screen space+// (origin top-left). These tests pin the conversion that keeps citations+// highlightable (issue #21216).+func TestOpenDataLoaderItems_ScreenSpacePositions(t *testing.T) {+ root := map[string]any{+ "type": "table of contents",+ "page number": 1.0,+ "bounding box": []any{72.0, 650.5, 540.0, 700.2},+ "children": []any{+ map[string]any{+ "type": "paragraph",+ "content": "body",+ "page number": 2.0,+ "bbox": []any{100.0, 50.0, 200.0, 80.0},+ },+ },+ }+ sizes := map[int][2]float64{0: {612, 792}, 1: {612, 792}}+ items, _ := openDataLoaderItemsWithGeometry(root, sizes)+ if len(items) != 1 {+ t.Fatalf("items = %d, want 1 (unknown type without text is skipped): %v", len(items), items)+ }+ got := items[0]["_pdf_positions"].([][]any)+ want := []any{2.0, 100.0, 200.0, 712.0, 742.0}+ if len(got) != 1 || len(got[0]) != 5 {+ t.Fatalf("positions = %#v, want one 5-column row", got)+ }+ for i := range want {+ if math.Abs(got[0][i].(float64)-want[i].(float64)) > 1e-6 {+ t.Fatalf("positions = %#v, want %#v", got, want)+ }+ }+ legacy, ok := items[0]["positions"].([][]any)+ if !ok || len(legacy) != 1 || len(legacy[0]) != 5 {+ t.Fatalf("positions alias = %#v, want the same one-row matrix", items[0]["positions"])+ }+ for i := range want {+ if math.Abs(legacy[0][i].(float64)-want[i].(float64)) > 1e-6 {+ t.Fatalf("positions alias = %#v, want %#v", legacy, want)+ }+ }+}++func TestOpenDataLoaderItems_AllElementTypesCarryPositions(t *testing.T) {+ root := map[string]any{+ "type": "document", "page number": 1.0, "bounding box": []any{0.0, 0.0, 612.0, 792.0},+ "children": []any{+ map[string]any{"type": "heading", "content": "Title", "page number": 1.0, "bounding box": []any{72.0, 700.0, 400.0, 730.0}},+ map[string]any{"type": "table", "html": "<table><tr><td>a</td></tr></table>", "page number": 1.0, "bounding box": []any{72.0, 400.0, 500.0, 600.0}},+ map[string]any{"type": "image", "page number": 1.0, "bounding box": []any{72.0, 100.0, 300.0, 300.0}},+ map[string]any{"type": "formula", "content": "a+b", "page number": 1.0, "bounding box": []any{72.0, 50.0, 200.0, 80.0}},+ },+ }+ sizes := map[int][2]float64{0: {612, 792}}+ items, _ := openDataLoaderItemsWithGeometry(root, sizes)+ if len(items) != 4 {+ t.Fatalf("items = %d, want 4: %v", len(items), items)+ }+ for i, item := range items {+ rows, ok := item["_pdf_positions"].([][]any)+ if !ok || len(rows) != 1 || rows[0][0] != 1.0 {+ t.Errorf("item %d (%v) positions = %#v, want one page-1 row", i, item["layout"], item["_pdf_positions"])+ }+ }+}++func TestOpenDataLoaderItems_NoGeometryNoPositions(t *testing.T) {+ root := map[string]any{"type": "paragraph", "content": "body", "page number": 1.0, "bounding box": []any{72.0, 650.5, 540.0, 700.2}}+ if items, _ := openDataLoaderItemsWithGeometry(root, nil); len(items) != 1 || items[0]["_pdf_positions"] != nil {+ t.Fatalf("items = %v, want one item without positions when page geometry is unavailable", items)+ } else if items[0]["page_number"] != 1 {+ t.Errorf("page_number = %#v, want 1 (page identity survives without geometry)", items[0]["page_number"])+ }+ // A page the geometry map does not cover behaves the same way.+ sizes := map[int][2]float64{2: {612, 792}}+ if items, _ := openDataLoaderItemsWithGeometry(root, sizes); len(items) != 1 || items[0]["_pdf_positions"] != nil {+ t.Fatalf("items = %v, want no positions for an uncovered page", items)+ }+}++func TestOpenDataLoaderItems_BadBoxesAreDropped(t *testing.T) {+ for _, el := range []map[string]any{+ {"type": "paragraph", "content": "no page", "bounding box": []any{1.0, 2.0, 3.0, 4.0}},+ {"type": "paragraph", "content": "no box", "page number": 1.0},+ {"type": "paragraph", "content": "short box", "page number": 1.0, "bounding box": []any{1.0, 2.0}},+ {"type": "paragraph", "content": "zero width", "page number": 1.0, "bounding box": []any{5.0, 2.0, 5.0, 4.0}},+ {"type": "paragraph", "content": "inverted", "page number": 1.0, "bounding box": []any{200.0, 300.0, 100.0, 100.0}},+ {"type": "paragraph", "content": "page zero", "page number": 0.0, "bounding box": []any{1.0, 2.0, 3.0, 4.0}},+ } {+ items, _ := openDataLoaderItemsWithGeometry(el, map[int][2]float64{0: {612, 792}})+ if len(items) != 1 {+ t.Fatalf("%v: items = %d, want 1", el["content"], len(items))+ }+ if items[0]["_pdf_positions"] != nil {+ t.Errorf("%v: positions = %#v, want none", el["content"], items[0]["_pdf_positions"])+ }+ }+}++// TestOpenDataLoaderItems_BoxClampedToPage keeps a slightly bleeding box from+// producing a highlight outside the page rectangle.+func TestOpenDataLoaderItems_BoxClampedToPage(t *testing.T) {+ el := map[string]any{+ "type": "paragraph", "content": "bleed", "page number": 1.0,+ "bounding box": []any{-20.0, 700.0, 620.0, 780.0},+ }+ items, warnings := openDataLoaderItemsWithGeometry(el, map[int][2]float64{0: {612, 792}})+ if len(warnings) != 0 {+ t.Fatalf("warnings = %v, want none for a bleed inside the tolerance", warnings)+ }+ rows, ok := items[0]["_pdf_positions"].([][]any)+ if !ok || len(rows) != 1 {+ t.Fatalf("positions = %#v, want one row", items[0]["_pdf_positions"])+ }+ want := []float64{0, 612, 12, 92}+ for i, v := range want {+ if got, _ := numericAny(rows[0][i+1]); math.Abs(got-v) > 1e-6 {+ t.Fatalf("positions = %#v, want the clamped page box %v", rows, want)+ }+ }+}++// TestOpenDataLoaderItems_ForeignCoordinateSpaceIsReported covers the failure+// the clamp used to hide: boxes in another space (image pixels, for example)+// land outside the page, and clamping them produced a full-page highlight that+// looked valid. The positions are dropped and the mismatch is surfaced instead.+func TestOpenDataLoaderItems_ForeignCoordinateSpaceIsReported(t *testing.T) {+ root := []any{+ // 150 DPI pixels on a 612x792 point page.+ map[string]any{"type": "paragraph", "content": "a", "page number": 1.0, "bounding box": []any{150.0, 1351.0, 550.0, 1400.0}},+ map[string]any{"type": "paragraph", "content": "b", "page number": 1.0, "bounding box": []any{0.0, 0.0, 1275.0, 1650.0}},+ }+ items, warnings := openDataLoaderItemsWithGeometry(root, map[int][2]float64{0: {612, 792}})+ if len(items) != 2 {+ t.Fatalf("items = %d, want 2 (text survives without positions)", len(items))+ }+ for i, item := range items {+ if item["_pdf_positions"] != nil {+ t.Errorf("item %d positions = %#v, want none", i, item["_pdf_positions"])+ }+ if item["page_number"] != 1 {+ t.Errorf("item %d page_number = %#v, want 1", i, item["page_number"])+ }+ }+ // One warning per page, naming the offending box and the page size.+ if len(warnings) != 1 {+ t.Fatalf("warnings = %q, want one per page", warnings)+ }+ for _, want := range []string{"page 1", "612x792", "coordinate"} {+ if !strings.Contains(warnings[0], want) {+ t.Fatalf("warning = %q, want it to mention %q", warnings[0], want)+ }+ }+}++// TestOpenDataLoaderItems_MixedPagesWarnPerPage checks that a second page with+// its own mismatch gets its own warning rather than being folded away.+func TestOpenDataLoaderItems_MixedPagesWarnPerPage(t *testing.T) {+ root := []any{+ map[string]any{"type": "paragraph", "content": "ok", "page number": 1.0, "bounding box": []any{72.0, 100.0, 540.0, 140.0}},+ map[string]any{"type": "paragraph", "content": "bad", "page number": 3.0, "bounding box": []any{0.0, 0.0, 2000.0, 2000.0}},+ }+ sizes := map[int][2]float64{0: {612, 792}, 2: {612, 792}}+ items, warnings := openDataLoaderItemsWithGeometry(root, sizes)+ if len(warnings) != 1 || !strings.Contains(warnings[0], "page 3") {+ t.Fatalf("warnings = %q, want one warning for page 3", warnings)+ }+ if items[0]["_pdf_positions"] == nil {+ t.Errorf("page 1 positions = nil, want the valid box kept")+ }+}++// TestOpenDataLoaderReportedPageCount pins the geometry-free fallback: the+// root's total wins; otherwise the HIGHEST page number, not the number of+// distinct pages (a sparse document would otherwise undercount); otherwise 1.+func TestOpenDataLoaderReportedPageCount(t *testing.T) {+ cases := []struct {+ name string+ root any+ want int+ }{+ {"root total", map[string]any{"number of pages": 7, "children": []any{+ map[string]any{"type": "paragraph", "content": "a", "page number": 1.0}}}, 7},+ {"highest page", map[string]any{"children": []any{+ map[string]any{"type": "paragraph", "content": "a", "page number": 3.0},+ map[string]any{"type": "paragraph", "content": "b", "page number": 1.0}}}, 3},+ {"no page keys", map[string]any{"type": "paragraph", "content": "a"}, 1},+ }+ for _, c := range cases {+ t.Run(c.name, func(t *testing.T) {+ if got := openDataLoaderReportedPageCount(c.root); got != c.want {+ t.Errorf("page count = %d, want %d", got, c.want)+ }+ })+ }+}diff --git a/internal/parser/parser/pdf_parser_opendataloader_test.go b/internal/parser/parser/pdf_parser_opendataloader_test.goindex 481f57906c..e7c51693ce 100644--- a/internal/parser/parser/pdf_parser_opendataloader_test.go+++ b/internal/parser/parser/pdf_parser_opendataloader_test.go@@ -136,7 +136,7 @@ func TestOpenDataLoaderItems_TableCellsFallback(t *testing.T) {map[string]any{"row": 0, "content": "b"},},}- items := openDataLoaderItems(root)+ items, _ := openDataLoaderItemsWithGeometry(root, nil)if len(items) != 1 {t.Fatalf("items len = %d, want 1", len(items))}@@ -164,7 +164,7 @@ func TestOpenDataLoaderItems_TableCellsFallbackSparseRows(t *testing.T) {map[string]any{"row": 5, "content": "z"},},}- items := openDataLoaderItems(root)+ items, _ := openDataLoaderItemsWithGeometry(root, nil)if len(items) != 1 {t.Fatalf("items len = %d, want 1", len(items))}diff --git a/internal/parser/parser/pdf_parser_remote_common.go b/internal/parser/parser/pdf_parser_remote_common.goindex 5b39ffadac..62158f8f38 100644--- a/internal/parser/parser/pdf_parser_remote_common.go+++ b/internal/parser/parser/pdf_parser_remote_common.go@@ -50,7 +50,7 @@ func collectPDFPageNumbers(raw any) map[int]struct{} {walk = func(node any) {switch v := node.(type) {case map[string]any:- for _, key := range []string{"page_number", "page_num", "page_no", "page_index", "page_idx", "page"} {+ for _, key := range []string{"page number", "page_number", "page_num", "page_no", "page_index", "page_idx", "page"} {if page := int(numberValue(v[key])); page > 0 {pages[page] = struct{}{}}diff --git a/internal/parser/parser/pdf_parser_table_contract_test.go b/internal/parser/parser/pdf_parser_table_contract_test.goindex 9a3d222df4..b8501c8d4f 100644--- a/internal/parser/parser/pdf_parser_table_contract_test.go+++ b/internal/parser/parser/pdf_parser_table_contract_test.go@@ -49,16 +49,20 @@ func TestPDFRemoteParsers_TableItemsCarryTableMarkup(t *testing.T) {wantDocType string}{{"opendataloader html", func() []map[string]any {- return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})+ items, _ := openDataLoaderItemsWithGeometry(map[string]any{"type": "table", "html": tableHTML}, nil)+ return items}, "table"},{"opendataloader content only", func() []map[string]any {- return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText})+ items, _ := openDataLoaderItemsWithGeometry(map[string]any{"type": "table", "content": pipeText}, nil)+ return items}, "text"},{"opendataloader cells only", func() []map[string]any {- return openDataLoaderItems(map[string]any{"type": "table", "cells": cells})+ items, _ := openDataLoaderItemsWithGeometry(map[string]any{"type": "table", "cells": cells}, nil)+ return items}, "table"},{"opendataloader blank cells keep content", func() []map[string]any {- return openDataLoaderItems(map[string]any{"type": "table", "content": pipeText, "cells": blankCells})+ items, _ := openDataLoaderItemsWithGeometry(map[string]any{"type": "table", "content": pipeText, "cells": blankCells}, nil)+ return items}, "text"},{"tcadp html content", func() []map[string]any {return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})diff --git a/internal/parser/parser/pdf_table_no_html_text_test.go b/internal/parser/parser/pdf_table_no_html_text_test.goindex d88deedf55..ee4c88f270 100644--- a/internal/parser/parser/pdf_table_no_html_text_test.go+++ b/internal/parser/parser/pdf_table_no_html_text_test.go@@ -27,7 +27,8 @@ func TestPDFTableItemsOmitHTMLText(t *testing.T) {return []map[string]any{item}}},{"opendataloader html", func() []map[string]any {- return openDataLoaderItems(map[string]any{"type": "table", "html": tableHTML})+ items, _ := openDataLoaderItemsWithGeometry(map[string]any{"type": "table", "html": tableHTML}, nil)+ return items}},{"tcadp html content", func() []map[string]any {return tcadpAnyToItems(map[string]any{"type": "table", "content": tableHTML})