RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): emit page positions for SoMark JSON blocks (#21324)
Taranum Wasu863304fe780c
diff --git a/internal/parser/parser/pdf_parser_somark.go b/internal/parser/parser/pdf_parser_somark.goindex 1008d35e82..cd9b5a9ee0 100644--- a/internal/parser/parser/pdf_parser_somark.go+++ b/internal/parser/parser/pdf_parser_somark.go@@ -6,11 +6,14 @@ import ("encoding/json""fmt""io"+ "math""mime/multipart""net/http""net/url""ragflow/internal/common""strings"++ pdfreader "github.com/ledongthuc/pdf")// parsePDFWithSoMark submits a PDF and converts the task result into parser@@ -38,7 +41,7 @@ func parsePDFWithSoMark(ctx context.Context, filename string, data []byte, parseif err != nil {return ParseResult{Err: err}}- items, pageCount, warnings := soMarkItems(ctx, result, parser.SoMarkKeepHeaderFooter)+ items, pageCount, warnings := soMarkItems(ctx, result, parser.SoMarkKeepHeaderFooter, soMarkPagePointSizes(data))if err := ctx.Err(); err != nil {return ParseResult{Err: err}}@@ -160,21 +163,52 @@ func soMarkPoll(ctx context.Context, baseURL, taskID, apiKey string) (map[stringreturn result, nil}-func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bool) ([]map[string]any, int, []string) {+func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bool, pageSizes [][2]float64) ([]map[string]any, int, []string) {outputs, _ := result["outputs"].(map[string]any)jsonPayload, _ := outputs["json"].(map[string]any)pages, _ := jsonPayload["pages"].([]any)items := make([]map[string]any, 0)state := &htmlWalkState{}var warnings []string++ // Positions reference real document pages: prefer the source PDF's page+ // count when it is known, falling back to the number of pages the+ // service returned.+ pageCount := len(pages)+ if len(pageSizes) > 0 {+ pageCount = len(pageSizes)+ }++ // Pixel dimensions of each page, keyed by the same resolved page number+ // positions use, so a block whose page_idx points at another page scales+ // against THAT page's pixel space instead of the enclosing page's.+ pixByPage := make(map[int][2]float64, len(pages))for pageIndex, pageRaw := range pages {page, ok := pageRaw.(map[string]any)if !ok {continue}+ // SoMark pages carry a 0-based page_num; positions and page_number+ // are 1-based.pageNumber := pageIndex + 1- if n := int(numberValue(page["page_number"])); n > 0 {- pageNumber = n+ if n, ok := soMarkPageIndex(page["page_num"]); ok {+ pageNumber = n + 1+ }+ if w, h := soMarkPagePixelSize(page["page_size"]); w > 0 && h > 0 {+ pixByPage[pageNumber] = [2]float64{w, h}+ }+ }++ for pageIndex, pageRaw := range pages {+ page, ok := pageRaw.(map[string]any)+ if !ok {+ continue+ }+ // SoMark pages carry a 0-based page_num; positions and page_number+ // are 1-based.+ pageNumber := pageIndex + 1+ if n, ok := soMarkPageIndex(page["page_num"]); ok {+ pageNumber = n + 1}blocks, _ := page["blocks"].([]any)for _, blockRaw := range blocks {@@ -186,11 +220,165 @@ func soMarkItems(ctx context.Context, result map[string]any, keepHeaderFooter bowarnings = append(warnings, blockWarnings...)for _, item := range blockItems {item["page_number"] = pageNumber+ // A block may carry its own 0-based page_idx; it wins over the+ // enclosing page value, which stays as the fallback. An index+ // beyond the known page range (the source PDF's pages when+ // readable, else the returned pages) falls back too: emitting+ // it would point at a page that does not exist and inflate+ // page_count downstream.+ blockPage := pageNumber+ pageBound := len(pages)+ if len(pageSizes) > 0 {+ pageBound = len(pageSizes)+ }+ if n, ok := soMarkPageIndex(block["page_idx"]); ok && n < pageBound {+ blockPage = n + 1+ }+ if len(pageSizes) > 0 && blockPage > len(pageSizes) {+ // The page is not in the source document; a position for+ // it cannot be correct, so the item goes out without one.+ items = append(items, item)+ continue+ }+ src := pixByPage[blockPage]+ if pos := soMarkBlockPosition(block, blockPage, src[0], src[1], pageSizes); pos != nil {+ item["positions"] = [][]any{pos}+ item["_pdf_positions"] = [][]any{pos}+ }items = append(items, item)}}}- return items, len(pages), append(state.warnings(), warnings...)+ return items, pageCount, append(state.warnings(), warnings...)+}++// soMarkBlockPosition converts a SoMark block bbox ([x0, top, x1, bottom]) and+// its 1-based page number into the [page, left, right, top, bottom] position+// row used by _pdf_positions.+// soMarkPageIndex validates a SoMark page index before use: JSON numbers+// arrive as float64, so reject fractional, negative, non-finite, and+// out-of-int-range values instead of truncating them into a wrong page.+func soMarkPageIndex(raw any) (int, bool) {+ f, ok := numericAny(raw)+ if !ok || math.IsNaN(f) || math.IsInf(f, 0) || f < 0 || f != math.Trunc(f) || f > float64(math.MaxInt32) {+ return 0, false+ }+ return int(f), true+}++func soMarkBlockPosition(block map[string]any, pageNumber int, srcW, srcH float64, pageSizes [][2]float64) []any {+ bbox, _ := block["bbox"].([]any)+ if len(bbox) != 4 {+ return nil+ }+ var v [4]float64+ for i, raw := range bbox {+ n, ok := numericAny(raw)+ if !ok || math.IsNaN(n) || math.IsInf(n, 0) {+ return nil+ }+ v[i] = n+ }+ x0, top, x1, bott := v[0], v[1], v[2], v[3]+ if x0 > x1 {+ x0, x1 = x1, x0+ }+ if top > bott {+ top, bott = bott, top+ }+ // SoMark reports bboxes in its own pixel space (page["page_size"]), while+ // RAGFlow positions are PDF points. Rescale into the page's point size and+ // clamp to the page, mirroring the Python converter's pdfplumber-at-72dpi+ // mapping. When either side is unknown the raw box is kept, matching the+ // Python fallback for pages it could not render.+ idx := pageNumber - 1+ if srcW > 0 && srcH > 0 && idx >= 0 && idx < len(pageSizes) {+ dst := pageSizes[idx]+ if dst[0] > 0 && dst[1] > 0 {+ x0 = min(max(x0/srcW*dst[0], 0), dst[0])+ x1 = min(max(x1/srcW*dst[0], 0), dst[0])+ top = min(max(top/srcH*dst[1], 0), dst[1])+ bott = min(max(bott/srcH*dst[1], 0), dst[1])+ }+ }+ return []any{float64(pageNumber), x0, x1, top, bott}+}++// soMarkPagePixelSize reads a page's SoMark pixel dimensions ({w, h}); a zero+// value means unknown.+func soMarkPagePixelSize(raw any) (float64, float64) {+ m, _ := raw.(map[string]any)+ w, _ := numericAny(m["w"])+ h, _ := numericAny(m["h"])+ return w, h+}++// soMarkPagePointSizes reads each page's [width, height] in PDF points from+// the source document (CropBox, falling back to MediaBox), the space RAGFlow+// positions and crops use. A nil result or a zero entry means the page size+// is unknown and positions stay in the service's pixel space, matching the+// Python converter's fallback when a page cannot be rendered.+func soMarkPagePointSizes(data []byte) [][2]float64 {+ r, err := pdfreader.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ return nil+ }+ numPage := r.NumPage()+ // A corrupt or malicious catalog /Count can be negative (panics make) or+ // enormous (allocates/iterates without bound). Treat it as unreadable and+ // fall back to the caller's pixel-space scaling, mirroring the+ // maxPDFPageCount guard deepdoc/pdf/plaintext_nocgo.go applies to the+ // same library.+ if numPage < 0 || numPage > maxSoMarkPageCount {+ return nil+ }+ sizes := make([][2]float64, 0, numPage)+ for p := 1; p <= numPage; p++ {+ box := soMarkInheritedPageValue(r.Page(p), "CropBox")+ if box.IsNull() || box.Len() != 4 {+ box = soMarkInheritedPageValue(r.Page(p), "MediaBox")+ }+ if box.IsNull() || box.Len() != 4 {+ sizes = append(sizes, [2]float64{})+ continue+ }+ x0, y0 := box.Index(0).Float64(), box.Index(1).Float64()+ x1, y1 := box.Index(2).Float64(), box.Index(3).Float64()+ w, h := math.Abs(x1-x0), math.Abs(y1-y0)+ // A quarter-turn /Rotate (also inheritable) swaps the dimensions of+ // the rendered page SoMark sees; scaling and clamping must use the+ // rotated size.+ if rot := soMarkInheritedPageValue(r.Page(p), "Rotate"); !rot.IsNull() {+ switch deg := ((int(rot.Float64()) % 360) + 360) % 360; deg {+ case 90, 270:+ w, h = h, w+ }+ }+ sizes = append(sizes, [2]float64{w, h})+ }+ return sizes+}++// maxSoMarkPageCount mirrors deepdoc/pdf/plaintext_nocgo.go: one million+// pages is far beyond any real document, so larger claimed counts are+// malformed.+const maxSoMarkPageCount = 1_000_000++// maxSoMarkParentDepth bounds the /Parent walk: ledongthuc/pdf detects no+// reference cycles, so a self-referencing /Parent would loop forever.+const maxSoMarkParentDepth = 32++// soMarkInheritedPageValue resolves a page dictionary key through the Parent+// chain, the inheritance the PDF spec defines for page boxes.+func soMarkInheritedPageValue(page pdfreader.Page, key string) pdfreader.Value {+ depth := 0+ for v := page.V; !v.IsNull() && depth < maxSoMarkParentDepth; v = v.Key("Parent") {+ if r := v.Key(key); !r.IsNull() {+ return r+ }+ depth+++ }+ return pdfreader.Value{}}func soMarkBlockToItems(ctx context.Context, block map[string]any, keepHeaderFooter bool, state *htmlWalkState) ([]map[string]any, []string) {diff --git a/internal/parser/parser/pdf_parser_somark_test.go b/internal/parser/parser/pdf_parser_somark_test.goindex 9362055f33..f9b898c5f3 100644--- a/internal/parser/parser/pdf_parser_somark_test.go+++ b/internal/parser/parser/pdf_parser_somark_test.go@@ -1,13 +1,21 @@package parserimport (+ "bytes""context"+ "fmt""io"+ "math""mime/multipart""net/http""net/http/httptest"+ "os"+ "reflect""strings""testing"+ "time"++ "github.com/signintech/gopdf")func TestPDFParser_ParseWithResult_SoMarkJSONIntegration(t *testing.T) {@@ -115,3 +123,379 @@ func TestSoMarkSubmitMultipartShape(t *testing.T) {t.Fatalf("api_key = %q, want key", got)}}++func TestSoMarkItemsEmitPagePositions(t *testing.T) {+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{+ map[string]any{"page_num": float64(0), "blocks": []any{+ map[string]any{"type": "text", "content": "first", "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ }},+ map[string]any{"page_num": float64(2), "blocks": []any{+ map[string]any{"type": "text", "content": "swapped", "bbox": []any{200.0, 90.0, 100.0, 50.0}},+ map[string]any{"type": "text", "content": "no bbox"},+ }},+ }}}}+ items, pageCount, _ := soMarkItems(context.Background(), result, false, nil)+ if pageCount != 2 || len(items) != 3 {+ t.Fatalf("pageCount=%d items=%d, want 2/3", pageCount, len(items))+ }+ want := [][]any{+ {float64(1), 10.0, 110.0, 20.0, 40.0},+ {float64(3), 100.0, 200.0, 50.0, 90.0},+ }+ for i, w := range want {+ for _, key := range []string{"positions", "_pdf_positions"} {+ rows, _ := items[i][key].([][]any)+ if len(rows) != 1 || !reflect.DeepEqual(rows[0], w) {+ t.Fatalf("item %d %s = %#v, want %#v", i, key, items[i][key], w)+ }+ }+ }+ if _, ok := items[2]["positions"]; ok {+ t.Fatalf("block without bbox must not carry positions: %#v", items[2])+ }+}++func TestSoMarkItemsPreferBlockPageIdx(t *testing.T) {+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{+ map[string]any{"page_num": float64(0), "blocks": []any{+ map[string]any{"type": "text", "content": "own page", "page_idx": float64(1), "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ map[string]any{"type": "text", "content": "page fallback", "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ }},+ map[string]any{"page_num": float64(1), "blocks": []any{}},+ }}}}+ items, _, _ := soMarkItems(context.Background(), result, false, nil)+ if len(items) != 2 {+ t.Fatalf("items=%d, want 2", len(items))+ }+ rows, _ := items[0]["_pdf_positions"].([][]any)+ if len(rows) != 1 || rows[0][0] != float64(2) {+ t.Fatalf("block page_idx must win: %#v", items[0]["_pdf_positions"])+ }+ rows, _ = items[1]["_pdf_positions"].([][]any)+ if len(rows) != 1 || rows[0][0] != float64(1) {+ t.Fatalf("missing page_idx must fall back to page_num: %#v", items[1]["_pdf_positions"])+ }+}++func TestSoMarkItemsRejectInvalidPageIdx(t *testing.T) {+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{+ map[string]any{"page_num": float64(2), "blocks": []any{+ map[string]any{"type": "text", "content": "fractional", "page_idx": float64(4.5), "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ map[string]any{"type": "text", "content": "overflow", "page_idx": float64(1e30), "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ map[string]any{"type": "text", "content": "valid", "page_idx": float64(7), "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ }},+ }}}}+ items, _, _ := soMarkItems(context.Background(), result, false, nil)+ if len(items) != 3 {+ t.Fatalf("items=%d, want 3", len(items))+ }+ // Fractional, overflowing AND merely out-of-range page_idx values all+ // fall back to the enclosing page: emitting page 8 of a 1-page result+ // would point at a page that does not exist and inflate page_count.+ for i, want := range []float64{3, 3, 3} {+ rows, _ := items[i]["_pdf_positions"].([][]any)+ if len(rows) != 1 || rows[0][0] != want {+ t.Fatalf("item %d page = %#v, want %v (invalid or out-of-range page_idx must fall back to page_num)", i, rows, want)+ }+ }+}++func TestPDFParser_ParseWithResult_SoMarkJSONCarriesPositions(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/parse/async":+ _, _ = io.Copy(io.Discard, r.Body)+ _, _ = w.Write([]byte(`{"code":0,"data":{"task_id":"task-1"}}`))+ case "/parse/async_check":+ _, _ = w.Write([]byte(`{"code":0,"data":{"status":"SUCCESS","result":{"outputs":{"json":{"pages":[{"page_num":1,"blocks":[{"type":"text","content":"hello","bbox":[5,6,50,16]}]}]}}}}}`))+ default:+ http.NotFound(w, r)+ }+ }))+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "SoMark",+ "output_format": "json",+ "somark_base_url": server.URL,+ "somark_api_key": "k",+ })+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if len(res.JSON) != 1 {+ t.Fatalf("JSON len = %d, want 1", len(res.JSON))+ }+ rows, ok := ExtractPDFPositions(res.JSON[0].ToMap())+ if !ok || len(rows) != 1 || len(rows[0]) != 5 {+ t.Fatalf("positions = %#v ok=%v", rows, ok)+ }+ for i, w := range []float64{2, 5, 50, 6, 16} {+ if got, _ := numericAny(rows[0][i]); got != w {+ t.Fatalf("positions[0][%d] = %v, want %v (row %#v)", i, rows[0][i], w, rows[0])+ }+ }+}++// makeTestPDF synthesizes a one-page PDF with the given point size so tests+// can verify pixel-to-point rescaling against a real page box.+func makeTestPDF(t *testing.T, widthPt, heightPt float64) []byte {+ t.Helper()+ var buf bytes.Buffer+ pdf := &gopdf.GoPdf{}+ pdf.Start(gopdf.Config{PageSize: gopdf.Rect{W: widthPt, H: heightPt}})+ pdf.AddPage()+ if _, err := pdf.WriteTo(&buf); err != nil {+ t.Fatalf("synthesize PDF: %v", err)+ }+ return buf.Bytes()+}++func TestSoMarkPagePointSizesReadsPageBox(t *testing.T) {+ data := makeTestPDF(t, 595.28, 841.89)+ sizes := soMarkPagePointSizes(data)+ if len(sizes) != 1 {+ t.Fatalf("sizes = %v, want one page", sizes)+ }+ if math.Abs(sizes[0][0]-595.28) > 1 || math.Abs(sizes[0][1]-841.89) > 1 {+ t.Fatalf("sizes[0] = %v, want ~[595.28 841.89]", sizes[0])+ }+ if got := soMarkPagePointSizes([]byte("not a pdf")); got != nil {+ t.Fatalf("unreadable PDF must yield nil (no rescale), got %v", got)+ }+}++func TestSoMarkPositionsRescaledFromPixelsToPoints(t *testing.T) {+ // SoMark reports bbox in its rendered pixel space (page_size {w,h});+ // RAGFlow positions are PDF points. An A4 page (~595x842 pt) rendered by+ // the service at ~200 dpi is 1654x2339 px, so raw pixel boxes are ~2.78x+ // too large without rescaling.+ data := makeTestPDF(t, 595.28, 841.89)+ sizes := soMarkPagePointSizes(data)+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{+ "pages": []any{+ map[string]any{+ "page_num": float64(0),+ "page_size": map[string]any{"w": float64(1654), "h": float64(2339)},+ "blocks": []any{+ map[string]any{"type": "text", "content": "scaled", "bbox": []any{165.4, 233.9, 827.0, 467.8}},+ },+ },+ },+ }}}+ items, _, _ := soMarkItems(context.Background(), result, false, sizes)+ pos := items[0]["_pdf_positions"].([][]any)[0]+ wantX0 := 165.4 / 1654 * 595.28+ wantX1 := 827.0 / 1654 * 595.28+ wantTop := 233.9 / 2339 * 841.89+ wantBott := 467.8 / 2339 * 841.89+ if pos[0] != float64(1) {+ t.Fatalf("page = %v, want 1", pos[0])+ }+ for i, want := range []float64{wantX0, wantX1, wantTop, wantBott} {+ if got := pos[i+1].(float64); math.Abs(got-want) > 0.5 {+ t.Fatalf("pos[%d] = %v, want ~%v (unscaled pixel value would be ~2.8x larger)", i+1, got, want)+ }+ }+}++func TestSoMarkPositionsClampedToPage(t *testing.T) {+ data := makeTestPDF(t, 595.28, 841.89)+ sizes := soMarkPagePointSizes(data)+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{+ "pages": []any{+ map[string]any{+ "page_num": float64(0),+ "page_size": map[string]any{"w": float64(1654), "h": float64(2339)},+ "blocks": []any{+ map[string]any{"type": "text", "content": "overflow", "bbox": []any{-50.0, -10.0, 1700.0, 2400.0}},+ },+ },+ },+ }}}+ items, _, _ := soMarkItems(context.Background(), result, false, sizes)+ pos := items[0]["_pdf_positions"].([][]any)[0]+ for i, v := range []float64{0, 595.28, 0, 841.89} {+ if got := pos[i+1].(float64); math.Abs(got-v) > 1 {+ t.Fatalf("pos[%d] = %v, want clamped ~%v", i+1, got, v)+ }+ }+}++func TestSoMarkPositionsRawWhenScaleUnknown(t *testing.T) {+ // No page_size in the payload (or unknown page dims): keep the raw box,+ // matching the Python converter's fallback when a page cannot be rendered.+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{+ "pages": []any{+ map[string]any{+ "page_num": float64(0),+ "blocks": []any{+ map[string]any{"type": "text", "content": "raw", "bbox": []any{10.0, 20.0, 110.0, 40.0}},+ },+ },+ },+ }}}+ items, _, _ := soMarkItems(context.Background(), result, false, nil)+ pos := items[0]["_pdf_positions"].([][]any)[0]+ for i, v := range []float64{10, 110, 20, 40} {+ if got := pos[i+1].(float64); got != v {+ t.Fatalf("pos[%d] = %v, want raw %v", i+1, got, v)+ }+ }+}++func TestSoMarkPositionsCrossPageScaling(t *testing.T) {+ // A block whose page_idx points at another page must scale against THAT+ // page's pixel space AND point size. Fixture pages: 595.5x842 pt (page 1)+ // and 300x400 pt (page 2); the block lives in page 1's entry but claims+ // page_idx 1 (page 2), rendered at 827x1169 px.+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")+ if err != nil {+ t.Skipf("fixture: %v", err)+ }+ sizes := soMarkPagePointSizes(data)+ if len(sizes) != 3 {+ t.Fatalf("sizes = %v, want three pages", sizes)+ }+ result := map[string]any{"outputs": map[string]any{"json": map[string]any{+ "pages": []any{+ map[string]any{+ "page_num": float64(0),+ "page_size": map[string]any{"w": float64(1654), "h": float64(2339)},+ "blocks": []any{+ map[string]any{"type": "text", "content": "cross", "page_idx": float64(1),+ "bbox": []any{413.5, 292.25, 620.25, 438.375}},+ },+ },+ map[string]any{+ "page_num": float64(1),+ "page_size": map[string]any{"w": float64(827), "h": float64(1169)},+ "blocks": []any{},+ },+ },+ }}}+ items, _, _ := soMarkItems(context.Background(), result, false, sizes)+ pos := items[0]["_pdf_positions"].([][]any)[0]+ if pos[0] != float64(2) {+ t.Fatalf("page = %v, want 2", pos[0])+ }+ // Hand-computed against page 2 (300x400 pt at 827x1169 px):+ // x0 = 413.5/827*300 = 150, x1 = 620.25/827*300 = 225,+ // top = 292.25/1169*400 = 100, bott = 438.375/1169*400 = 150.+ for i, want := range []float64{150, 225, 100, 150} {+ if got := pos[i+1].(float64); math.Abs(got-want) > 0.5 {+ t.Fatalf("pos[%d] = %v, want ~%v (mixing the enclosing page's 1654px width would give ~75/112.5)", i+1, got, want)+ }+ }+}++func TestSoMarkPagePointSizesHonorRotate(t *testing.T) {+ // Fixture page 3 declares MediaBox [0 0 400 300] with /Rotate 90: the+ // rendered page is 300 wide x 400 high, and positions must scale and+ // clamp against the rotated size.+ data, err := os.ReadFile("testdata/paddleocr_pages.pdf")+ if err != nil {+ t.Skipf("fixture: %v", err)+ }+ sizes := soMarkPagePointSizes(data)+ if len(sizes) != 3 {+ t.Fatalf("sizes = %v, want three pages", sizes)+ }+ if math.Abs(sizes[0][0]-595.5) > 1 || math.Abs(sizes[0][1]-842) > 1 {+ t.Fatalf("page 1 = %v, want ~[595.5 842]", sizes[0])+ }+ if math.Abs(sizes[1][0]-300) > 1 || math.Abs(sizes[1][1]-400) > 1 {+ t.Fatalf("page 2 = %v, want ~[300 400]", sizes[1])+ }+ if math.Abs(sizes[2][0]-300) > 1 || math.Abs(sizes[2][1]-400) > 1 {+ t.Fatalf("page 3 = %v, want rotated ~[300 400], not raw MediaBox [400 300]", sizes[2])+ }+}++func TestSoMarkBlockPositionRejectsNonFinite(t *testing.T) {+ for _, bbox := range [][]any{+ {1.0, math.NaN(), 110.0, 40.0},+ {1.0, 20.0, math.Inf(1), 40.0},+ {1.0, 20.0, 110.0, math.Inf(-1)},+ } {+ if pos := soMarkBlockPosition(map[string]any{"bbox": bbox}, 0, 100, 100, nil); pos != nil {+ t.Fatalf("non-finite bbox %v must drop the position, got %#v", bbox, pos)+ }+ }+}++// buildSoMarkPDFWithCount emits a minimal but structurally valid PDF whose+// /Pages /Count is set to the supplied (possibly malformed) value, like+// deepdoc/pdf's buildPDFWithCount. Offsets are computed so the xref table is+// self-consistent and ledongthuc/pdf can open it.+func buildSoMarkPDFWithCount(t *testing.T, count int) []byte {+ t.Helper()+ var buf bytes.Buffer+ buf.WriteString("%PDF-1.4\n")+ off1 := buf.Len()+ fmt.Fprintf(&buf, "1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")+ off2 := buf.Len()+ fmt.Fprintf(&buf, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count %d >>\nendobj\n", count)+ off3 := buf.Len()+ fmt.Fprintf(&buf, "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>\nendobj\n")+ xrefStart := buf.Len()+ buf.WriteString("xref\n0 4\n")+ buf.WriteString("0000000000 65535 f \n")+ fmt.Fprintf(&buf, "%010d 00000 n \n", off1)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off2)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off3)+ fmt.Fprintf(&buf, "trailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xrefStart)+ return buf.Bytes()+}++func TestSoMarkPagePointSizesRejectsMalformedCount(t *testing.T) {+ // A negative or absurd catalog /Count must fall back to the caller's+ // pixel-space scaling instead of panicking make or allocating/iterating+ // without bound.+ for _, count := range []int{-1, 1 << 30} {+ if got := soMarkPagePointSizes(buildSoMarkPDFWithCount(t, count)); got != nil {+ t.Fatalf("/Count %d: got %v, want nil fallback", count, got)+ }+ }+ // A sane count still reads page boxes.+ if got := soMarkPagePointSizes(buildSoMarkPDFWithCount(t, 1)); len(got) != 1 {+ t.Fatalf("/Count 1: got %v, want one page", got)+ }+}++func TestSoMarkPagePointSizesStopsAtParentCycle(t *testing.T) {+ // ledongthuc/pdf detects no reference cycles: a page whose /Parent points+ // at itself must not hang the inheritance walk.+ var buf bytes.Buffer+ buf.WriteString("%PDF-1.4\n")+ off1 := buf.Len()+ fmt.Fprintf(&buf, "1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")+ off2 := buf.Len()+ fmt.Fprintf(&buf, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count 1 >>\nendobj\n")+ off3 := buf.Len()+ fmt.Fprintf(&buf, "3 0 obj\n<< /Type /Page /Parent 3 0 R /Resources << >> >>\nendobj\n")+ xrefStart := buf.Len()+ buf.WriteString("xref\n0 4\n")+ buf.WriteString("0000000000 65535 f \n")+ fmt.Fprintf(&buf, "%010d 00000 n \n", off1)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off2)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off3)+ fmt.Fprintf(&buf, "trailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xrefStart)++ done := make(chan [][2]float64, 1)+ go func() { done <- soMarkPagePointSizes(buf.Bytes()) }()+ select {+ case sizes := <-done:+ // No box is resolvable through the cycle: the page falls back to an+ // empty size (the caller's pixel-space scaling), as an unreadable+ // box already does.+ if len(sizes) != 1 || sizes[0] != [2]float64{} {+ t.Fatalf("cyclic /Parent page: got %v, want one empty size", sizes)+ }+ case <-time.After(10 * time.Second):+ t.Fatal("soMarkPagePointSizes hung on a self-referencing /Parent")+ }+}diff --git a/internal/parser/parser/table_regression_test.go b/internal/parser/parser/table_regression_test.goindex 8440123900..6320ea815a 100644--- a/internal/parser/parser/table_regression_test.go+++ b/internal/parser/parser/table_regression_test.go@@ -88,7 +88,7 @@ func TestTableRegressionSoMarkTableImageIDs(t *testing.T) {markup := "<table><tr><td>Logo</td><td><img src=\"data:image/png;base64,aGVsbG8=\"></td></tr></table>"items, _, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{map[string]any{"type": "table", "content": markup}, map[string]any{"type": "table", "content": markup},- }}}}}}, false)+ }}}}}}, false, nil)seen := map[string]bool{}for _, item := range items {@@ -110,7 +110,7 @@ func TestTableRegressionSoMarkDocumentImageBudget(t *testing.T) {for i := range blocks {blocks[i] = map[string]any{"type": "table", "content": markup}}- items, _, warnings := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": blocks}}}}}, false)+ items, _, warnings := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": blocks}}}}}, false, nil)images := 0for _, item := range items {if item["doc_type_kwd"] == "image" && item["image"] != nil {@@ -338,7 +338,7 @@ func TestSoMarkPreservesImageResolutionWarnings(t *testing.T) {items, _, warnings := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{map[string]any{"type": "table", "content": source},map[string]any{"type": "table", "content": "| A | B |\n|---|---|\n| x | y |"},- }}}}}}, false)+ }}}}}}, false, nil)if len(items) != 2 {t.Fatalf("items=%#v", items)}diff --git a/internal/parser/parser/table_structure_test.go b/internal/parser/parser/table_structure_test.goindex 745eeffd95..b15e880430 100644--- a/internal/parser/parser/table_structure_test.go+++ b/internal/parser/parser/table_structure_test.go@@ -172,7 +172,7 @@ func TestPDFMarkdownPreservesEmptyHeaderTable(t *testing.T) {func TestSoMarkPreservesMarkdownTableAndNotes(t *testing.T) {items, pages, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{map[string]any{"type": "table", "content": "Before.\n\n| | |\n|---|---|\n| Owner | RET |\n\nAfter."},- }}}}}}, false)+ }}}}}}, false, nil)if pages != 1 || len(items) != 3 {t.Fatalf("pages=%d items=%#v", pages, items)}@@ -189,7 +189,7 @@ func TestSoMarkPreservesPageOnSplitTableItems(t *testing.T) {items, _, _ := soMarkItems(t.Context(), map[string]any{"outputs": map[string]any{"json": map[string]any{"pages": []any{map[string]any{"blocks": []any{}},map[string]any{"blocks": []any{map[string]any{"type": "table", "content": "Before.\n\n| A | B |\n|---|---|\n| Owner | RET |\n\nAfter."}}},- }}}}, false)+ }}}}, false, nil)if len(items) != 3 {t.Fatalf("items=%#v", items)}