Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

perf(pdf): read page directives once per document (#21950)

jay77721bb5437e69389
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/cropbox.go b/internal/deepdoc/parser/pdf/pdfoxide/cropbox.go
deleted file mode 100644
index 92bf38f30c..0000000000
--- a/internal/deepdoc/parser/pdf/pdfoxide/cropbox.go
+++ /dev/null
@@ -1,109 +0,0 @@
-package pdfoxide
-
-import "strconv"
-
-// parseCropBoxFromRaw scans raw PDF bytes for /CropBox entries and
-// returns the array [x0, y0, x1, y1] for the given page index (0-based).
-// The second return value is false if no /CropBox was found.
-//
-// Algorithm: sequential scan of "/CropBox [...]" patterns — same approach
-// as parsePageRotationFromRaw. Works for all common PDF generators.
-func parseCropBoxFromRaw(data []byte, pageIdx int) ([4]float64, bool) {
- type cb [4]float64
- var boxes []cb
- rest := data
- for {
- idx := indexAfter(rest, "/CropBox")
- if idx < 0 {
- break
- }
- rest = rest[idx:]
- // Skip whitespace, expect '['
- for len(rest) > 0 && isSpace(rest[0]) {
- rest = rest[1:]
- }
- if len(rest) == 0 || rest[0] != '[' {
- continue
- }
- rest = rest[1:]
- // Parse 4 float values inside [...]
- var vals [4]float64
- ok := true
- for i := 0; i < 4; i++ {
- for len(rest) > 0 && isSpace(rest[0]) {
- rest = rest[1:]
- }
- v, n := parseFloat(rest)
- if n == 0 {
- ok = false
- break
- }
- vals[i] = v
- rest = rest[n:]
- }
- if !ok {
- continue
- }
- boxes = append(boxes, cb(vals))
- }
- if pageIdx < len(boxes) {
- return boxes[pageIdx], true
- }
- return [4]float64{}, false
-}
-
-// indexAfter finds the byte position right after the first occurrence of s in
-// data. Returns -1 if not found.
-func indexAfter(data []byte, s string) int {
- for i := 0; i < len(data)-len(s); i++ {
- match := true
- for j := 0; j < len(s); j++ {
- if data[i+j] != s[j] {
- match = false
- break
- }
- }
- if match {
- return i + len(s)
- }
- }
- return -1
-}
-
-func isSpace(b byte) bool {
- return b == ' ' || b == '\t' || b == '\n' || b == '\r'
-}
-
-// parseFloat parses a decimal number from the beginning of s.
-// Returns the value and the number of bytes consumed (0 on failure).
-func parseFloat(s []byte) (float64, int) {
- i := 0
- for i < len(s) && isSpace(s[i]) {
- i++
- }
- j := i
- // Scan: optional sign, digits, optional decimal point + digits
- if j < len(s) && (s[j] == '+' || s[j] == '-') {
- j++
- }
- hasDigit := false
- for j < len(s) && s[j] >= '0' && s[j] <= '9' {
- j++
- hasDigit = true
- }
- if j < len(s) && s[j] == '.' {
- j++
- for j < len(s) && s[j] >= '0' && s[j] <= '9' {
- j++
- hasDigit = true
- }
- }
- if !hasDigit || j == i {
- return 0, 0
- }
- v, err := strconv.ParseFloat(string(s[i:j]), 64)
- if err != nil {
- return 0, 0
- }
- return v, j
-}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go b/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go
deleted file mode 100644
index 803a87c983..0000000000
--- a/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package pdfoxide
-
-import (
- "math"
- "testing"
-)
-
-func TestParseCropBoxFromRaw(t *testing.T) {
- eps := 1e-6
-
- tests := []struct {
- name string
- raw string
- pageIdx int
- want [4]float64
- ok bool
- }{
- {
- name: "standard A4 portrait",
- raw: "/CropBox [0 0 595.28 841.89]",
- want: [4]float64{0, 0, 595.28, 841.89},
- ok: true,
- },
- {
- name: "non-zero origin",
- raw: "/CropBox [30 20 575 832]",
- want: [4]float64{30, 20, 575, 832},
- ok: true,
- },
- {
- name: "with extra whitespace",
- raw: "/CropBox [ 0.5 10.25 595.3 842.0 ]",
- want: [4]float64{0.5, 10.25, 595.3, 842.0},
- ok: true,
- },
- {
- name: "no spaces inside brackets",
- raw: "/CropBox[0 0 595 842]",
- want: [4]float64{0, 0, 595, 842},
- ok: true,
- },
- {
- name: "page index 1 picks second CropBox",
- raw: "/CropBox [0 0 1 1] /Rotate 90 /CropBox [2 2 3 3]",
- pageIdx: 1,
- want: [4]float64{2, 2, 3, 3},
- ok: true,
- },
- {
- name: "page index out of range",
- raw: "/CropBox [0 0 1 1]",
- pageIdx: 5,
- want: [4]float64{},
- ok: false,
- },
- {
- name: "no cropbox",
- raw: "/MediaBox [0 0 595 842] /Rotate 90",
- want: [4]float64{},
- ok: false,
- },
- {
- name: "empty input",
- raw: "",
- want: [4]float64{},
- ok: false,
- },
- {
- name: "incomplete array — fewer than 4 values",
- raw: "/CropBox [0 0 595]",
- want: [4]float64{},
- ok: false,
- },
- {
- name: "negative values",
- raw: "/CropBox [-10 -20 595 842]",
- want: [4]float64{-10, -20, 595, 842},
- ok: true,
- },
- {
- name: "real pypdf output format (multiple spaces, decimals)",
- raw: "/Type /Page /MediaBox [0 0 595.2756 841.8898] /CropBox [30.0 20.0 575.0 832.0] /Rotate 90",
- want: [4]float64{30.0, 20.0, 575.0, 832.0},
- ok: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got, ok := parseCropBoxFromRaw([]byte(tt.raw), tt.pageIdx)
- if ok != tt.ok {
- t.Fatalf("ok=%v want %v", ok, tt.ok)
- }
- if !ok {
- return
- }
- for i := 0; i < 4; i++ {
- if math.Abs(got[i]-tt.want[i]) > eps {
- t.Errorf("[%d]: got %.4f, want %.4f", i, got[i], tt.want[i])
- }
- }
- })
- }
-}
-
-func TestParseFloat(t *testing.T) {
- tests := []struct {
- s string
- want float64
- n int
- }{
- {"0", 0, 1},
- {"595.28", 595.28, 6},
- {" 42", 42, 4},
- {"-10.5", -10.5, 5},
- {"+3.14", 3.14, 5},
- {"123abc", 123, 3},
- {"abc", 0, 0},
- {"", 0, 0},
- {".5", 0.5, 2},
- }
- for _, tt := range tests {
- v, n := parseFloat([]byte(tt.s))
- if n != tt.n || math.Abs(v-tt.want) > 1e-6 {
- t.Errorf("parseFloat(%q) = (%.4f, %d), want (%.4f, %d)",
- tt.s, v, n, tt.want, tt.n)
- }
- }
-}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/directives.go b/internal/deepdoc/parser/pdf/pdfoxide/directives.go
new file mode 100644
index 0000000000..a466f4e5cf
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pdfoxide/directives.go
@@ -0,0 +1,134 @@
+package pdfoxide
+
+import (
+ "bytes"
+ "strconv"
+)
+
+// pageDirectives carries the two page attributes the oxide extraction path
+// needs but pdf_oxide does not expose in post-rotation space.
+type pageDirectives struct {
+ rotate int
+ cropBox [4]float64
+ hasCrop bool
+}
+
+// scanPageDirectives collects /Rotate and /CropBox for every page in a single
+// pass over the raw PDF bytes. Both values are keyed the way the per-page
+// scanners keyed them — the nth occurrence carrying a parseable value is page
+// n — so callers observe identical results. A non-numeric /Rotate or a
+// malformed /CropBox is skipped without consuming a page index.
+func scanPageDirectives(data []byte) map[int]pageDirectives {
+ out := make(map[int]pageDirectives)
+ rotPage, cropPage := 0, 0
+ for i := 0; i < len(data); i++ {
+ // The explicit bound keeps the previous scanners' tail rule: a key
+ // starting in the last len(key)-1 bytes is not matched. bytes.HasPrefix
+ // alone would accept a key that ends exactly at EOF.
+ if i+len("/Rotate") < len(data) && bytes.HasPrefix(data[i:], []byte("/Rotate")) {
+ val, next, ok := parseRotationValueAt(data, i+len("/Rotate"))
+ if ok {
+ d := out[rotPage]
+ d.rotate = val
+ out[rotPage] = d
+ rotPage++
+ }
+ i = next - 1
+ continue
+ }
+ if i+len("/CropBox") < len(data) && bytes.HasPrefix(data[i:], []byte("/CropBox")) {
+ box, next, ok := parseCropBoxAt(data, i+len("/CropBox"))
+ if ok {
+ d := out[cropPage]
+ d.cropBox = box
+ d.hasCrop = true
+ out[cropPage] = d
+ cropPage++
+ }
+ i = next - 1
+ continue
+ }
+ }
+ return out
+}
+
+// parseRotationValueAt parses the integer that follows a /Rotate key at pos.
+// It returns the value, the position just past it, and whether a value was
+// found. Whitespace between the key and the value is tolerated.
+func parseRotationValueAt(data []byte, pos int) (int, int, bool) {
+ for pos < len(data) && isSpace(data[pos]) {
+ pos++
+ }
+ val, digits := 0, 0
+ for pos < len(data) && data[pos] >= '0' && data[pos] <= '9' {
+ val = val*10 + int(data[pos]-'0')
+ pos++
+ digits++
+ }
+ if digits == 0 {
+ return 0, pos, false
+ }
+ return val, pos, true
+}
+
+// parseCropBoxAt parses "[x0 y0 x1 y1]" after a /CropBox key at pos. It returns
+// the box, the position just past it, and whether a complete box was found.
+func parseCropBoxAt(data []byte, pos int) ([4]float64, int, bool) {
+ var box [4]float64
+ for pos < len(data) && isSpace(data[pos]) {
+ pos++
+ }
+ if pos >= len(data) || data[pos] != '[' {
+ return box, pos, false
+ }
+ pos++
+ for i := 0; i < 4; i++ {
+ // parseFloat skips leading whitespace itself and reports how far it
+ // consumed, so no separate skip is needed here.
+ v, n := parseFloat(data[pos:])
+ if n == 0 {
+ return box, pos, false
+ }
+ box[i] = v
+ pos += n
+ }
+ return box, pos, true
+}
+
+func isSpace(b byte) bool {
+ return b == ' ' || b == '\t' || b == '\n' || b == '\r'
+}
+
+// parseFloat parses a decimal number from the beginning of s.
+// Returns the value and the number of bytes consumed (0 on failure).
+func parseFloat(s []byte) (float64, int) {
+ i := 0
+ for i < len(s) && isSpace(s[i]) {
+ i++
+ }
+ j := i
+ // Scan: optional sign, digits, optional decimal point + digits
+ if j < len(s) && (s[j] == '+' || s[j] == '-') {
+ j++
+ }
+ hasDigit := false
+ for j < len(s) && s[j] >= '0' && s[j] <= '9' {
+ j++
+ hasDigit = true
+ }
+ if j < len(s) && s[j] == '.' {
+ j++
+ for j < len(s) && s[j] >= '0' && s[j] <= '9' {
+ j++
+ hasDigit = true
+ }
+ }
+ if !hasDigit || j == i {
+ return 0, 0
+ }
+ v, err := strconv.ParseFloat(string(s[i:j]), 64)
+ if err != nil {
+ return 0, 0
+ }
+ return v, j
+}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/directives_test.go b/internal/deepdoc/parser/pdf/pdfoxide/directives_test.go
new file mode 100644
index 0000000000..a559235658
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pdfoxide/directives_test.go
@@ -0,0 +1,331 @@
+package pdfoxide
+
+import (
+ "math"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// referenceRotationScan is the per-page scanner scanPageDirectives replaced.
+// It is kept here verbatim so the merged scan can be proven equivalent on
+// every input, including malformed ones.
+func referenceRotationScan(data []byte, pageIdx int) int {
+ var rotations []int
+ rest := data
+ for {
+ idx := -1
+ for i := 0; i < len(rest)-7; i++ {
+ if rest[i] == '/' && rest[i+1] == 'R' && rest[i+2] == 'o' &&
+ rest[i+3] == 't' && rest[i+4] == 'a' && rest[i+5] == 't' &&
+ rest[i+6] == 'e' {
+ idx = i
+ break
+ }
+ }
+ if idx < 0 {
+ break
+ }
+ rest = rest[idx+7:]
+ for len(rest) > 0 && (rest[0] == ' ' || rest[0] == '\t' || rest[0] == '\n' || rest[0] == '\r') {
+ rest = rest[1:]
+ }
+ if len(rest) == 0 {
+ break
+ }
+ val := 0
+ i := 0
+ for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
+ val = val*10 + int(rest[i]-'0')
+ i++
+ }
+ if i > 0 {
+ rotations = append(rotations, val)
+ }
+ rest = rest[i:]
+ }
+ if pageIdx < len(rotations) {
+ return rotations[pageIdx]
+ }
+ return 0
+}
+
+// referenceIndexAfter is the byte search the previous CropBox scanner used.
+func referenceIndexAfter(data []byte, s string) int {
+ for i := 0; i < len(data)-len(s); i++ {
+ match := true
+ for j := 0; j < len(s); j++ {
+ if data[i+j] != s[j] {
+ match = false
+ break
+ }
+ }
+ if match {
+ return i + len(s)
+ }
+ }
+ return -1
+}
+
+// referenceCropBoxScan is the per-page /CropBox scanner scanPageDirectives
+// replaced.
+func referenceCropBoxScan(data []byte, pageIdx int) ([4]float64, bool) {
+ type cb [4]float64
+ var boxes []cb
+ rest := data
+ for {
+ idx := referenceIndexAfter(rest, "/CropBox")
+ if idx < 0 {
+ break
+ }
+ rest = rest[idx:]
+ for len(rest) > 0 && isSpace(rest[0]) {
+ rest = rest[1:]
+ }
+ if len(rest) == 0 || rest[0] != '[' {
+ continue
+ }
+ rest = rest[1:]
+ var vals [4]float64
+ ok := true
+ for i := 0; i < 4; i++ {
+ for len(rest) > 0 && isSpace(rest[0]) {
+ rest = rest[1:]
+ }
+ v, n := parseFloat(rest)
+ if n == 0 {
+ ok = false
+ break
+ }
+ vals[i] = v
+ rest = rest[n:]
+ }
+ if !ok {
+ continue
+ }
+ boxes = append(boxes, cb(vals))
+ }
+ if pageIdx < len(boxes) {
+ return boxes[pageIdx], true
+ }
+ return [4]float64{}, false
+}
+
+// comparedPages bounds how many leading page indexes each comparison checks.
+// It exceeds the page count of the shipped fixtures so real PDFs are covered
+// end to end.
+const comparedPages = 32
+
+// compareScanners asserts that the merged scan reports exactly what the two
+// former per-page scanners reported for every page index in range.
+func compareScanners(t *testing.T, data []byte, label string) {
+ t.Helper()
+ got := scanPageDirectives(data)
+ for page := 0; page < comparedPages; page++ {
+ wantRot := referenceRotationScan(data, page)
+ if d := got[page]; d.rotate != wantRot {
+ t.Fatalf("%s: page %d rotate = %d, want %d", label, page, d.rotate, wantRot)
+ }
+ wantBox, wantOK := referenceCropBoxScan(data, page)
+ d := got[page]
+ if d.hasCrop != wantOK {
+ t.Fatalf("%s: page %d hasCrop = %v, want %v", label, page, d.hasCrop, wantOK)
+ }
+ if wantOK {
+ for i := 0; i < 4; i++ {
+ if d.cropBox[i] != wantBox[i] {
+ t.Fatalf("%s: page %d cropBox[%d] = %v, want %v", label, page, i, d.cropBox[i], wantBox[i])
+ }
+ }
+ }
+ }
+}
+
+func TestScanPageDirectivesMatchesPreviousScanners(t *testing.T) {
+ cases := []struct {
+ name string
+ raw string
+ }{
+ {"empty", ""},
+ {"no directives", "/MediaBox [0 0 595 842] /Type /Page"},
+ {"rotate only", "/Rotate 90"},
+ {"cropbox only", "/CropBox [0 0 595 842]"},
+ {"no space after key", "/CropBox[0 0 595 842]/Rotate 180"},
+ {"extra whitespace", "/CropBox [ 0.5 10.25 595.3 842.0 ]"},
+ {"negative cropbox", "/CropBox [-10 -20 595 842]"},
+ {"interleaved pages", "/Rotate 90 /CropBox [0 0 1 1] /Rotate 270 /CropBox [2 2 3 3]"},
+ {"cropbox before rotate", "/CropBox [5 5 6 6] /Rotate 90"},
+ {"non-numeric rotate is skipped", "/Rotate </Rotate 90"},
+ {"negative rotate is skipped", "/Rotate -90 /Rotate 90"},
+ {"malformed cropbox is skipped", "/CropBox [0 0 595] /CropBox [0 0 1 1]"},
+ {"cropbox without array", "/CropBox /Rotate 90 /CropBox [0 0 1 1]"},
+ {"trailing rotate", "/Rotate 90<<>>"},
+ {"keyword inside value", "/CropBox [0 0 595 842] /Rotate 90 endobj /Rotate 180"},
+ {"media box only", "/MediaBox [0 0 612 792] endobj /MediaBox [0 0 612 792]"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ compareScanners(t, []byte(tc.raw), tc.name)
+ })
+ }
+}
+
+func TestScanPageDirectivesRandomizedEquivalence(t *testing.T) {
+ tokens := []string{
+ "/Rotate", " 90", " 0", " 270", " -45", " </Rotate", " abc",
+ "/CropBox", " [0 0 1 1]", " [ 10.5 -2 30 40 ]", " [0 0 595]",
+ " ", "\n", "/MediaBox", "[0 0 9 9]", "endobj", "/Type /Page",
+ }
+ rng := rand.New(rand.NewSource(20261104))
+ for iter := 0; iter < 300; iter++ {
+ var buf []byte
+ for n := rng.Intn(40); n > 0; n-- {
+ buf = append(buf, tokens[rng.Intn(len(tokens))]...)
+ }
+ compareScanners(t, buf, "random")
+ }
+}
+
+// TestScanPageDirectivesOnRealPDFs sweeps every PDF shipped in testdata so the
+// merged scan is compared against both former scanners on real byte streams,
+// not only on synthetic ones.
+func TestScanPageDirectivesOnRealPDFs(t *testing.T) {
+ paths, err := filepath.Glob(filepath.Join("..", "testdata", "real_pdfs", "*.pdf"))
+ if err != nil {
+ t.Fatalf("glob: %v", err)
+ }
+ if len(paths) == 0 {
+ t.Skip("no real PDFs in ../testdata/real_pdfs")
+ }
+ for _, path := range paths {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read %s: %v", path, err)
+ }
+ compareScanners(t, data, filepath.Base(path))
+ }
+}
+
+func TestScanPageDirectivesCropBox(t *testing.T) {
+ eps := 1e-6
+
+ tests := []struct {
+ name string
+ raw string
+ pageIdx int
+ want [4]float64
+ ok bool
+ }{
+ {
+ name: "standard A4 portrait",
+ raw: "/CropBox [0 0 595.28 841.89]",
+ want: [4]float64{0, 0, 595.28, 841.89},
+ ok: true,
+ },
+ {
+ name: "non-zero origin",
+ raw: "/CropBox [30 20 575 832]",
+ want: [4]float64{30, 20, 575, 832},
+ ok: true,
+ },
+ {
+ name: "with extra whitespace",
+ raw: "/CropBox [ 0.5 10.25 595.3 842.0 ]",
+ want: [4]float64{0.5, 10.25, 595.3, 842.0},
+ ok: true,
+ },
+ {
+ name: "no spaces inside brackets",
+ raw: "/CropBox[0 0 595 842]",
+ want: [4]float64{0, 0, 595, 842},
+ ok: true,
+ },
+ {
+ name: "page index 1 picks second CropBox",
+ raw: "/CropBox [0 0 1 1] /Rotate 90 /CropBox [2 2 3 3]",
+ pageIdx: 1,
+ want: [4]float64{2, 2, 3, 3},
+ ok: true,
+ },
+ {
+ name: "page index out of range",
+ raw: "/CropBox [0 0 1 1]",
+ pageIdx: 5,
+ want: [4]float64{},
+ ok: false,
+ },
+ {
+ name: "no cropbox",
+ raw: "/MediaBox [0 0 595 842] /Rotate 90",
+ want: [4]float64{},
+ ok: false,
+ },
+ {
+ name: "empty input",
+ raw: "",
+ want: [4]float64{},
+ ok: false,
+ },
+ {
+ name: "incomplete array — fewer than 4 values",
+ raw: "/CropBox [0 0 595]",
+ want: [4]float64{},
+ ok: false,
+ },
+ {
+ name: "negative values",
+ raw: "/CropBox [-10 -20 595 842]",
+ want: [4]float64{-10, -20, 595, 842},
+ ok: true,
+ },
+ {
+ name: "real pypdf output format (multiple spaces, decimals)",
+ raw: "/Type /Page /MediaBox [0 0 595.2756 841.8898] /CropBox [30.0 20.0 575.0 832.0] /Rotate 90",
+ want: [4]float64{30.0, 20.0, 575.0, 832.0},
+ ok: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ d := scanPageDirectives([]byte(tt.raw))[tt.pageIdx]
+ got, ok := d.cropBox, d.hasCrop
+ if ok != tt.ok {
+ t.Fatalf("ok=%v want %v", ok, tt.ok)
+ }
+ if !ok {
+ return
+ }
+ for i := 0; i < 4; i++ {
+ if math.Abs(got[i]-tt.want[i]) > eps {
+ t.Errorf("[%d]: got %.4f, want %.4f", i, got[i], tt.want[i])
+ }
+ }
+ })
+ }
+}
+
+func TestParseFloat(t *testing.T) {
+ tests := []struct {
+ s string
+ want float64
+ n int
+ }{
+ {"0", 0, 1},
+ {"595.28", 595.28, 6},
+ {" 42", 42, 4},
+ {"-10.5", -10.5, 5},
+ {"+3.14", 3.14, 5},
+ {"123abc", 123, 3},
+ {"abc", 0, 0},
+ {"", 0, 0},
+ {".5", 0.5, 2},
+ }
+ for _, tt := range tests {
+ v, n := parseFloat([]byte(tt.s))
+ if n != tt.n || math.Abs(v-tt.want) > 1e-6 {
+ t.Errorf("parseFloat(%q) = (%.4f, %d), want (%.4f, %d)",
+ tt.s, v, n, tt.want, tt.n)
+ }
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.go b/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.go
new file mode 100644
index 0000000000..3941e32aee
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.go
@@ -0,0 +1,253 @@
+//go:build cgo
+
+package pdfoxide
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "ragflow/internal/deepdoc/parser/pdf/pdfium"
+)
+
+// pdfFixture is one PDF used by the table-driven tests below.
+type pdfFixture struct {
+ name string
+ data []byte
+}
+
+// TestPageDimsCacheMatchesPerPageLookup proves the document-wide size pass
+// reproduces exactly what the former per-page pdfium call returned. The
+// generated fixture keeps the test meaningful in a clean checkout; every real
+// PDF shipped next to the parser is swept as well when present.
+func TestPageDimsCacheMatchesPerPageLookup(t *testing.T) {
+ const pages = 3
+ generated := buildTestPDF(t, pages, 90)
+
+ paths, err := filepath.Glob(filepath.Join("..", "testdata", "real_pdfs", "*.pdf"))
+ if err != nil {
+ t.Fatalf("glob: %v", err)
+ }
+
+ cases := []pdfFixture{{name: "generated", data: generated}}
+ for _, path := range paths {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read %s: %v", path, err)
+ }
+ cases = append(cases, pdfFixture{name: filepath.Base(path), data: data})
+ }
+ if len(paths) == 0 {
+ t.Log("no real PDFs in ../testdata/real_pdfs; generated fixture only")
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ eng, err := NewEngine(tc.data)
+ if err != nil {
+ t.Fatalf("NewEngine: %v", err)
+ }
+ defer eng.Close()
+
+ pageCount, err := eng.PageCount()
+ if err != nil {
+ t.Fatalf("PageCount: %v", err)
+ }
+ if pageCount == 0 {
+ t.Fatalf("PageCount = 0")
+ }
+
+ for page := 0; page < pageCount; page++ {
+ wantW, wantH, wantErr := pdfium.PageSize(tc.data, page)
+ gotD, gotOK := eng.pageSize(page)
+ if wantErr != nil {
+ if gotOK {
+ t.Fatalf("page %d: cached size despite pdfium error %v", page, wantErr)
+ }
+ continue
+ }
+ if !gotOK {
+ t.Fatalf("page %d: size missing from cache", page)
+ }
+ if gotD[0] != wantW || gotD[1] != wantH {
+ t.Fatalf("page %d: cached %.4fx%.4f, want %.4fx%.4f", page, gotD[0], gotD[1], wantW, wantH)
+ }
+ }
+ })
+ }
+}
+
+// TestPageSizeBatchPartialFailure pins the failure contract: a page missing
+// from the batch is retried on its own, the retry is cached, and the batch is
+// never re-run because some page failed.
+func TestPageSizeBatchPartialFailure(t *testing.T) {
+ eng, err := NewEngine(buildTestPDF(t, 2, 0))
+ if err != nil {
+ t.Fatalf("NewEngine: %v", err)
+ }
+ defer eng.Close()
+
+ var batchCalls, singleCalls int
+ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {
+ batchCalls++
+ // Page 1 is unreadable in this pass; page 0 succeeded.
+ return map[int][2]float64{0: {200, 100}}, 2, errors.New("page 1 unreadable")
+ }
+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {
+ singleCalls++
+ return 200, 100, nil
+ }
+
+ if d, ok := eng.pageSize(0); !ok || d != [2]float64{200, 100} {
+ t.Fatalf("page 0: got %v ok=%v, want cached batch size", d, ok)
+ }
+ if singleCalls != 0 {
+ t.Fatalf("page 0 must come from the batch, single-page lookups = %d", singleCalls)
+ }
+
+ if d, ok := eng.pageSize(1); !ok || d != [2]float64{200, 100} {
+ t.Fatalf("page 1: got %v ok=%v, want single-page retry result", d, ok)
+ }
+ if singleCalls != 1 {
+ t.Fatalf("page 1 must be retried once, got %d lookups", singleCalls)
+ }
+
+ if _, ok := eng.pageSize(1); !ok || singleCalls != 1 {
+ t.Fatalf("page 1 retry must be cached, lookups = %d", singleCalls)
+ }
+ if _, ok := eng.pageSize(0); !ok || batchCalls != 1 {
+ t.Fatalf("partial batch must not run again, batch calls = %d", batchCalls)
+ }
+}
+
+// TestPageSizeFallbackWhenBothLookupsFail keeps the original fallback chain:
+// only when the batch and the single-page retry both fail does PageSize answer
+// with pdf_oxide's raw dimensions.
+func TestPageSizeFallbackWhenBothLookupsFail(t *testing.T) {
+ data := buildTestPDF(t, 1, 0)
+ eng, err := NewEngine(data)
+ if err != nil {
+ t.Fatalf("NewEngine: %v", err)
+ }
+ defer eng.Close()
+
+ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {
+ return nil, 1, errors.New("batch failed")
+ }
+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {
+ return 0, 0, errors.New("single page failed")
+ }
+
+ wantW, wantH, err := eng.doc.PageSize(0)
+ if err != nil {
+ t.Fatalf("oxide PageSize: %v", err)
+ }
+ gotW, gotH, err := eng.PageSize(0)
+ if err != nil {
+ t.Fatalf("PageSize: %v", err)
+ }
+ if gotW != wantW || gotH != wantH {
+ t.Fatalf("PageSize = %.4fx%.4f, want oxide fallback %.4fx%.4f", gotW, gotH, wantW, wantH)
+ }
+}
+
+// TestExtractCharsUsesSinglePageRetry covers the regression the batch cache
+// could introduce: a page missing from the batch must still get its pdfium
+// size (and therefore correct rotation) instead of silently falling back to
+// the unrotated pdf_oxide dimensions.
+func TestExtractCharsUsesSinglePageRetry(t *testing.T) {
+ const rotate = 90
+ data := buildTestPDF(t, 2, rotate)
+ wantW, wantH, err := pdfium.PageSize(data, 1)
+ if err != nil {
+ t.Fatalf("pdfium.PageSize: %v", err)
+ }
+
+ eng, err := NewEngine(data)
+ if err != nil {
+ t.Fatalf("NewEngine: %v", err)
+ }
+ defer eng.Close()
+
+ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {
+ return map[int][2]float64{0: {200, 100}}, 2, errors.New("page 1 unreadable")
+ }
+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {
+ return wantW, wantH, nil
+ }
+
+ w, h, err := eng.PageSize(1)
+ if err != nil {
+ t.Fatalf("PageSize: %v", err)
+ }
+ if w != wantW || h != wantH {
+ t.Fatalf("PageSize = %.2fx%.2f, want %.2fx%.2f", w, h, wantW, wantH)
+ }
+
+ chars, err := eng.ExtractChars(1)
+ if err != nil {
+ t.Fatalf("ExtractChars: %v", err)
+ }
+ assertCharsInsidePage(t, chars, w, h)
+}
+
+// TestExtractCharsRotatedPageCoordinates is the clean-checkout coordinate
+// check the previous manual tier owned: a rotated page must yield characters
+// inside the post-rotation page box.
+func TestExtractCharsRotatedPageCoordinates(t *testing.T) {
+ cases := []struct {
+ rotate int
+ wantW float64
+ wantH float64
+ }{
+ {0, 200, 100},
+ {90, 100, 200},
+ {180, 200, 100},
+ {270, 100, 200},
+ }
+ const eps = 0.01
+ for _, tc := range cases {
+ t.Run(fmt.Sprintf("rotate-%d", tc.rotate), func(t *testing.T) {
+ data := buildTestPDF(t, 1, tc.rotate)
+ eng, err := NewEngine(data)
+ if err != nil {
+ t.Fatalf("NewEngine: %v", err)
+ }
+ defer eng.Close()
+
+ w, h, err := eng.PageSize(0)
+ if err != nil {
+ t.Fatalf("PageSize: %v", err)
+ }
+ if math.Abs(w-tc.wantW) > eps || math.Abs(h-tc.wantH) > eps {
+ t.Fatalf("PageSize = %.2fx%.2f, want %.2fx%.2f", w, h, tc.wantW, tc.wantH)
+ }
+
+ chars, err := eng.ExtractChars(0)
+ if err != nil {
+ t.Fatalf("ExtractChars: %v", err)
+ }
+ assertCharsInsidePage(t, chars, w, h)
+ })
+ }
+}
+
+// assertCharsInsidePage fails when any extracted character falls outside the
+// post-rotation page box.
+func assertCharsInsidePage(t *testing.T, chars []Char, w, h float64) {
+ t.Helper()
+ if len(chars) == 0 {
+ t.Fatalf("no characters extracted")
+ }
+ const eps = 0.01
+ for _, c := range chars {
+ if c.X0 < -eps || c.X1 > w+eps || c.Top < -eps || c.Bottom > h+eps {
+ t.Fatalf("char %q out of post-rotation box %.1fx%.1f: x=[%.2f,%.2f] y=[%.2f,%.2f]",
+ c.Text, w, h, c.X0, c.X1, c.Top, c.Bottom)
+ }
+ }
+}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.go b/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.go
new file mode 100644
index 0000000000..2a6098a22a
--- /dev/null
+++ b/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.go
@@ -0,0 +1,54 @@
+//go:build cgo
+
+package pdfoxide
+
+import (
+ "bytes"
+ "fmt"
+ "testing"
+)
+
+// buildTestPDF returns a minimal but valid PDF with `pages` identical pages at
+// the given /Rotate. The bytes are generated in the test so the PDF-parsing
+// tests run in a clean checkout, without a tracked binary fixture.
+func buildTestPDF(t *testing.T, pages, rotate int) []byte {
+ t.Helper()
+ if pages < 1 {
+ pages = 1
+ }
+
+ var buf bytes.Buffer
+ var offsets []int
+ writeObj := func(s string) {
+ offsets = append(offsets, buf.Len())
+ buf.WriteString(s)
+ }
+
+ buf.WriteString("%PDF-1.4\n")
+ writeObj("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")
+
+ kids := ""
+ for p := 0; p < pages; p++ {
+ kids += fmt.Sprintf("%d 0 R ", 3+p)
+ }
+ fontObj := 3 + pages
+ contentObj := 4 + pages
+ writeObj(fmt.Sprintf("2 0 obj\n<< /Type /Pages /Kids [%s] /Count %d >>\nendobj\n", kids, pages))
+ for p := 0; p < pages; p++ {
+ writeObj(fmt.Sprintf(
+ "%d 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 100] /Rotate %d "+
+ "/Resources << /Font << /F1 %d 0 R >> >> /Contents %d 0 R >>\nendobj\n",
+ 3+p, rotate, fontObj, contentObj))
+ }
+ writeObj(fmt.Sprintf("%d 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n", fontObj))
+ stream := "BT /F1 24 Tf 20 40 Td (Hello) Tj ET"
+ writeObj(fmt.Sprintf("%d 0 obj\n<< /Length %d >>\nstream\n%s\nendstream\nendobj\n", contentObj, len(stream), stream))
+
+ xref := buf.Len()
+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)
+ for _, off := range offsets {
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)
+ }
+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)
+ return buf.Bytes()
+}
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go b/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go
index 1fa7911db7..c294ec3d02 100644
--- a/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go
+++ b/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go
@@ -3,8 +3,10 @@
package pdfoxide
import (
+ "context"
"image"
"math"
+ "sync"
"ragflow/internal/deepdoc/parser/pdf/pdfium"
)
@@ -20,9 +22,29 @@ type Char struct {
}
// Engine wraps pdf_oxide to extract chars and render pages.
+//
+// rawData and doc belong to exactly one document (NewEngine opens one and
+// Close releases it), so the per-document lookups below are cached on the
+// engine: the byte-level /Rotate and /CropBox scan runs once, and pdfium's
+// page sizes are read in one pass instead of one full document reload per
+// page.
type Engine struct {
doc *Document
rawData []byte
+
+ mu sync.Mutex
+ directives map[int]pageDirectives
+ // pageDims caches the page sizes pdfium has reported so far; dimsAttempted
+ // records that the document-wide pass already ran, so a page missing from
+ // the map is retried on its own rather than by re-reading every page.
+ pageDims map[int][2]float64
+ dimsAttempted bool
+
+ // pageSizesBatch and pageSizeSingle are the pdfium lookups this engine
+ // uses. They are fields rather than package variables so a test can
+ // simulate a partial batch on one engine without touching global state.
+ pageSizesBatch func(ctx context.Context, pdf []byte, pages []int) (map[int][2]float64, int, error)
+ pageSizeSingle func(pdf []byte, page int) (float64, float64, error)
}
// NewEngine opens a PDF from bytes and returns an Engine.
@@ -31,11 +53,91 @@ func NewEngine(pdfBytes []byte) (*Engine, error) {
if err != nil {
return nil, err
}
- return &Engine{doc: doc, rawData: pdfBytes}, nil
+ return &Engine{
+ doc: doc,
+ rawData: pdfBytes,
+ pageSizesBatch: pdfium.PageSizes,
+ pageSizeSingle: pdfium.PageSize,
+ }, nil
}
func (e *Engine) RawData() []byte { return e.rawData }
+// loadPageDims fills pageDims with every page size pdfium can report from one
+// batch pass. The pass runs once: successful pages are cached even when other
+// pages fail, so a missing page is retried individually by pageSize instead of
+// re-reading the whole document on every call. A failed pass is not retried in
+// bulk; only the pages that missed out are looked up again.
+//
+// The pass has no caller context to honour: ExtractChars carries none, and
+// threading one through the PDFEngine interface is a separate change.
+func (e *Engine) loadPageDims() {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ if e.dimsAttempted {
+ return
+ }
+ e.dimsAttempted = true
+ n, err := e.doc.PageCount()
+ if err != nil || n <= 0 {
+ return
+ }
+ idxs := make([]int, n)
+ for i := range idxs {
+ idxs[i] = i
+ }
+ sizes, _, _ := e.pageSizesBatch(context.Background(), e.rawData, idxs)
+ if len(sizes) == 0 {
+ return
+ }
+ if e.pageDims == nil {
+ e.pageDims = make(map[int][2]float64, len(sizes))
+ }
+ for page, size := range sizes {
+ e.pageDims[page] = size
+ }
+}
+
+// pageSize returns a page's post-rotation size. It prefers the document-wide
+// batch, then retries that one page directly, and reports false only when both
+// fail. The retry result is cached so a page pdfium cannot size costs at most
+// one extra lookup per document. Both ExtractChars and PageSize use this helper
+// so the fallback rule cannot diverge between them.
+func (e *Engine) pageSize(pageNum int) ([2]float64, bool) {
+ e.loadPageDims()
+
+ e.mu.Lock()
+ d, ok := e.pageDims[pageNum]
+ e.mu.Unlock()
+ if ok {
+ return d, true
+ }
+
+ w, h, err := e.pageSizeSingle(e.rawData, pageNum)
+ if err != nil {
+ return [2]float64{}, false
+ }
+ d = [2]float64{w, h}
+ e.mu.Lock()
+ if e.pageDims == nil {
+ e.pageDims = make(map[int][2]float64, 1)
+ }
+ e.pageDims[pageNum] = d
+ e.mu.Unlock()
+ return d, true
+}
+
+// pageDirective returns the cached /Rotate and /CropBox of one page, scanning
+// the raw bytes on first use.
+func (e *Engine) pageDirective(pageNum int) pageDirectives {
+ e.mu.Lock()
+ defer e.mu.Unlock()
+ if e.directives == nil {
+ e.directives = scanPageDirectives(e.rawData)
+ }
+ return e.directives[pageNum]
+}
+
func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {
chars, err := e.doc.GetDedupePageChars(pageNum, 0.5)
if err != nil {
@@ -52,15 +154,16 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {
// If dimensions are swapped, the page has implicit rotation
// (inherited /Rotate or ContentBox rotation).
rawW, rawH, _ := e.doc.PageSize(pageNum)
- effW, effH, pdfErr := pdfium.PageSize(e.rawData, pageNum)
- if pdfErr != nil {
- effW, effH = rawW, rawH
+ effW, effH := rawW, rawH
+ if d, ok := e.pageSize(pageNum); ok {
+ effW, effH = d[0], d[1]
}
dimSwapped := rawW > 0 && rawH > 0 && effW > 0 && effH > 0 &&
math.Abs(rawW-effH) < 1 && math.Abs(rawH-effW) < 1
- rawRot := parsePageRotationFromRaw(e.rawData, pageNum)
+ dir := e.pageDirective(pageNum)
+ rawRot := dir.rotate
needsRotate := false
rotation90 := false
@@ -86,7 +189,7 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {
// CropBox correction — shift origin if CropBox differs from MediaBox.
var cropDX, cropDY float64
- realCrop, hasCrop := parseCropBoxFromRaw(e.rawData, pageNum)
+ realCrop, hasCrop := dir.cropBox, dir.hasCrop
if hasCrop {
cropH := realCrop[3] - realCrop[1]
oxideCropH := rawH
@@ -173,51 +276,6 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {
return result, nil
}
-// parsePageRotationFromRaw scans raw PDF bytes for /Rotate entries.
-// Returns the rotation value for the given page index, or 0 if not found.
-// NOTE: This only finds /Rotate defined directly on page objects.
-// Inherited /Rotate (from parent Pages dict) is not detected here but
-// is caught by the dimension-comparison fallback in ExtractChars.
-func parsePageRotationFromRaw(data []byte, pageIdx int) int {
- var rotations []int
- rest := data
- for {
- idx := -1
- for i := 0; i < len(rest)-7; i++ {
- if rest[i] == '/' && rest[i+1] == 'R' && rest[i+2] == 'o' &&
- rest[i+3] == 't' && rest[i+4] == 'a' && rest[i+5] == 't' &&
- rest[i+6] == 'e' {
- idx = i
- break
- }
- }
- if idx < 0 {
- break
- }
- rest = rest[idx+7:]
- for len(rest) > 0 && (rest[0] == ' ' || rest[0] == '\t' || rest[0] == '\n' || rest[0] == '\r') {
- rest = rest[1:]
- }
- if len(rest) == 0 {
- break
- }
- val := 0
- i := 0
- for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {
- val = val*10 + int(rest[i]-'0')
- i++
- }
- if i > 0 {
- rotations = append(rotations, val)
- }
- rest = rest[i:]
- }
- if pageIdx < len(rotations) {
- return rotations[pageIdx]
- }
- return 0
-}
-
// RenderPageImage uses pdfium for page rendering — pdfium correctly
// applies /Rotate so the output matches character coordinates and DLA.
// There is no pdf_oxide fallback because pdf_oxide does not apply
@@ -236,13 +294,14 @@ func (e *Engine) RenderPage(pageNum int, dpi float64) ([]byte, error) {
// PageSize returns the effective page dimensions via pdfium, which
// correctly applies /Rotate. pdf_oxide's own PageSize returns raw
-// (unrotated) dimensions.
+// (unrotated) dimensions. Sizes come from the document-wide cache, with a
+// single-page pdfium retry for a page the batch could not size; only when
+// both fail does it fall back to pdf_oxide, exactly as before.
func (e *Engine) PageSize(pageNum int) (float64, float64, error) {
- w, h, err := pdfium.PageSize(e.rawData, pageNum)
- if err != nil {
- return e.doc.PageSize(pageNum)
+ if d, ok := e.pageSize(pageNum); ok {
+ return d[0], d[1], nil
}
- return w, h, nil
+ return e.doc.PageSize(pageNum)
}
func (e *Engine) PageCount() (int, error) { return e.doc.PageCount() }
func (e *Engine) Close() error { e.doc.Close(); return nil }