RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
perf(pdf): read page directives once per document (#21950)
jay77721bb5437e69389
diff --git a/internal/deepdoc/parser/pdf/pdfoxide/cropbox.go b/internal/deepdoc/parser/pdf/pdfoxide/cropbox.godeleted file mode 100644index 92bf38f30c..0000000000--- a/internal/deepdoc/parser/pdf/pdfoxide/cropbox.go+++ /dev/null@@ -1,109 +0,0 @@-package pdfoxide--import "strconv"--// parseCropBoxFromRaw scans raw PDF bytes for /CropBox entries and-// returns the array [x0, y0, x1, y1] for the given page index (0-based).-// The second return value is false if no /CropBox was found.-//-// Algorithm: sequential scan of "/CropBox [...]" patterns — same approach-// as parsePageRotationFromRaw. Works for all common PDF generators.-func parseCropBoxFromRaw(data []byte, pageIdx int) ([4]float64, bool) {- type cb [4]float64- var boxes []cb- rest := data- for {- idx := indexAfter(rest, "/CropBox")- if idx < 0 {- break- }- rest = rest[idx:]- // Skip whitespace, expect '['- for len(rest) > 0 && isSpace(rest[0]) {- rest = rest[1:]- }- if len(rest) == 0 || rest[0] != '[' {- continue- }- rest = rest[1:]- // Parse 4 float values inside [...]- var vals [4]float64- ok := true- for i := 0; i < 4; i++ {- for len(rest) > 0 && isSpace(rest[0]) {- rest = rest[1:]- }- v, n := parseFloat(rest)- if n == 0 {- ok = false- break- }- vals[i] = v- rest = rest[n:]- }- if !ok {- continue- }- boxes = append(boxes, cb(vals))- }- if pageIdx < len(boxes) {- return boxes[pageIdx], true- }- return [4]float64{}, false-}--// indexAfter finds the byte position right after the first occurrence of s in-// data. Returns -1 if not found.-func indexAfter(data []byte, s string) int {- for i := 0; i < len(data)-len(s); i++ {- match := true- for j := 0; j < len(s); j++ {- if data[i+j] != s[j] {- match = false- break- }- }- if match {- return i + len(s)- }- }- return -1-}--func isSpace(b byte) bool {- return b == ' ' || b == '\t' || b == '\n' || b == '\r'-}--// parseFloat parses a decimal number from the beginning of s.-// Returns the value and the number of bytes consumed (0 on failure).-func parseFloat(s []byte) (float64, int) {- i := 0- for i < len(s) && isSpace(s[i]) {- i++- }- j := i- // Scan: optional sign, digits, optional decimal point + digits- if j < len(s) && (s[j] == '+' || s[j] == '-') {- j++- }- hasDigit := false- for j < len(s) && s[j] >= '0' && s[j] <= '9' {- j++- hasDigit = true- }- if j < len(s) && s[j] == '.' {- j++- for j < len(s) && s[j] >= '0' && s[j] <= '9' {- j++- hasDigit = true- }- }- if !hasDigit || j == i {- return 0, 0- }- v, err := strconv.ParseFloat(string(s[i:j]), 64)- if err != nil {- return 0, 0- }- return v, j-}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go b/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.godeleted file mode 100644index 803a87c983..0000000000--- a/internal/deepdoc/parser/pdf/pdfoxide/cropbox_test.go+++ /dev/null@@ -1,128 +0,0 @@-package pdfoxide--import (- "math"- "testing"-)--func TestParseCropBoxFromRaw(t *testing.T) {- eps := 1e-6-- tests := []struct {- name string- raw string- pageIdx int- want [4]float64- ok bool- }{- {- name: "standard A4 portrait",- raw: "/CropBox [0 0 595.28 841.89]",- want: [4]float64{0, 0, 595.28, 841.89},- ok: true,- },- {- name: "non-zero origin",- raw: "/CropBox [30 20 575 832]",- want: [4]float64{30, 20, 575, 832},- ok: true,- },- {- name: "with extra whitespace",- raw: "/CropBox [ 0.5 10.25 595.3 842.0 ]",- want: [4]float64{0.5, 10.25, 595.3, 842.0},- ok: true,- },- {- name: "no spaces inside brackets",- raw: "/CropBox[0 0 595 842]",- want: [4]float64{0, 0, 595, 842},- ok: true,- },- {- name: "page index 1 picks second CropBox",- raw: "/CropBox [0 0 1 1] /Rotate 90 /CropBox [2 2 3 3]",- pageIdx: 1,- want: [4]float64{2, 2, 3, 3},- ok: true,- },- {- name: "page index out of range",- raw: "/CropBox [0 0 1 1]",- pageIdx: 5,- want: [4]float64{},- ok: false,- },- {- name: "no cropbox",- raw: "/MediaBox [0 0 595 842] /Rotate 90",- want: [4]float64{},- ok: false,- },- {- name: "empty input",- raw: "",- want: [4]float64{},- ok: false,- },- {- name: "incomplete array — fewer than 4 values",- raw: "/CropBox [0 0 595]",- want: [4]float64{},- ok: false,- },- {- name: "negative values",- raw: "/CropBox [-10 -20 595 842]",- want: [4]float64{-10, -20, 595, 842},- ok: true,- },- {- name: "real pypdf output format (multiple spaces, decimals)",- raw: "/Type /Page /MediaBox [0 0 595.2756 841.8898] /CropBox [30.0 20.0 575.0 832.0] /Rotate 90",- want: [4]float64{30.0, 20.0, 575.0, 832.0},- ok: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got, ok := parseCropBoxFromRaw([]byte(tt.raw), tt.pageIdx)- if ok != tt.ok {- t.Fatalf("ok=%v want %v", ok, tt.ok)- }- if !ok {- return- }- for i := 0; i < 4; i++ {- if math.Abs(got[i]-tt.want[i]) > eps {- t.Errorf("[%d]: got %.4f, want %.4f", i, got[i], tt.want[i])- }- }- })- }-}--func TestParseFloat(t *testing.T) {- tests := []struct {- s string- want float64- n int- }{- {"0", 0, 1},- {"595.28", 595.28, 6},- {" 42", 42, 4},- {"-10.5", -10.5, 5},- {"+3.14", 3.14, 5},- {"123abc", 123, 3},- {"abc", 0, 0},- {"", 0, 0},- {".5", 0.5, 2},- }- for _, tt := range tests {- v, n := parseFloat([]byte(tt.s))- if n != tt.n || math.Abs(v-tt.want) > 1e-6 {- t.Errorf("parseFloat(%q) = (%.4f, %d), want (%.4f, %d)",- tt.s, v, n, tt.want, tt.n)- }- }-}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/directives.go b/internal/deepdoc/parser/pdf/pdfoxide/directives.gonew file mode 100644index 0000000000..a466f4e5cf--- /dev/null+++ b/internal/deepdoc/parser/pdf/pdfoxide/directives.go@@ -0,0 +1,134 @@+package pdfoxide++import (+ "bytes"+ "strconv"+)++// pageDirectives carries the two page attributes the oxide extraction path+// needs but pdf_oxide does not expose in post-rotation space.+type pageDirectives struct {+ rotate int+ cropBox [4]float64+ hasCrop bool+}++// scanPageDirectives collects /Rotate and /CropBox for every page in a single+// pass over the raw PDF bytes. Both values are keyed the way the per-page+// scanners keyed them — the nth occurrence carrying a parseable value is page+// n — so callers observe identical results. A non-numeric /Rotate or a+// malformed /CropBox is skipped without consuming a page index.+func scanPageDirectives(data []byte) map[int]pageDirectives {+ out := make(map[int]pageDirectives)+ rotPage, cropPage := 0, 0+ for i := 0; i < len(data); i++ {+ // The explicit bound keeps the previous scanners' tail rule: a key+ // starting in the last len(key)-1 bytes is not matched. bytes.HasPrefix+ // alone would accept a key that ends exactly at EOF.+ if i+len("/Rotate") < len(data) && bytes.HasPrefix(data[i:], []byte("/Rotate")) {+ val, next, ok := parseRotationValueAt(data, i+len("/Rotate"))+ if ok {+ d := out[rotPage]+ d.rotate = val+ out[rotPage] = d+ rotPage+++ }+ i = next - 1+ continue+ }+ if i+len("/CropBox") < len(data) && bytes.HasPrefix(data[i:], []byte("/CropBox")) {+ box, next, ok := parseCropBoxAt(data, i+len("/CropBox"))+ if ok {+ d := out[cropPage]+ d.cropBox = box+ d.hasCrop = true+ out[cropPage] = d+ cropPage+++ }+ i = next - 1+ continue+ }+ }+ return out+}++// parseRotationValueAt parses the integer that follows a /Rotate key at pos.+// It returns the value, the position just past it, and whether a value was+// found. Whitespace between the key and the value is tolerated.+func parseRotationValueAt(data []byte, pos int) (int, int, bool) {+ for pos < len(data) && isSpace(data[pos]) {+ pos+++ }+ val, digits := 0, 0+ for pos < len(data) && data[pos] >= '0' && data[pos] <= '9' {+ val = val*10 + int(data[pos]-'0')+ pos+++ digits+++ }+ if digits == 0 {+ return 0, pos, false+ }+ return val, pos, true+}++// parseCropBoxAt parses "[x0 y0 x1 y1]" after a /CropBox key at pos. It returns+// the box, the position just past it, and whether a complete box was found.+func parseCropBoxAt(data []byte, pos int) ([4]float64, int, bool) {+ var box [4]float64+ for pos < len(data) && isSpace(data[pos]) {+ pos+++ }+ if pos >= len(data) || data[pos] != '[' {+ return box, pos, false+ }+ pos+++ for i := 0; i < 4; i++ {+ // parseFloat skips leading whitespace itself and reports how far it+ // consumed, so no separate skip is needed here.+ v, n := parseFloat(data[pos:])+ if n == 0 {+ return box, pos, false+ }+ box[i] = v+ pos += n+ }+ return box, pos, true+}++func isSpace(b byte) bool {+ return b == ' ' || b == '\t' || b == '\n' || b == '\r'+}++// parseFloat parses a decimal number from the beginning of s.+// Returns the value and the number of bytes consumed (0 on failure).+func parseFloat(s []byte) (float64, int) {+ i := 0+ for i < len(s) && isSpace(s[i]) {+ i+++ }+ j := i+ // Scan: optional sign, digits, optional decimal point + digits+ if j < len(s) && (s[j] == '+' || s[j] == '-') {+ j+++ }+ hasDigit := false+ for j < len(s) && s[j] >= '0' && s[j] <= '9' {+ j+++ hasDigit = true+ }+ if j < len(s) && s[j] == '.' {+ j+++ for j < len(s) && s[j] >= '0' && s[j] <= '9' {+ j+++ hasDigit = true+ }+ }+ if !hasDigit || j == i {+ return 0, 0+ }+ v, err := strconv.ParseFloat(string(s[i:j]), 64)+ if err != nil {+ return 0, 0+ }+ return v, j+}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/directives_test.go b/internal/deepdoc/parser/pdf/pdfoxide/directives_test.gonew file mode 100644index 0000000000..a559235658--- /dev/null+++ b/internal/deepdoc/parser/pdf/pdfoxide/directives_test.go@@ -0,0 +1,331 @@+package pdfoxide++import (+ "math"+ "math/rand"+ "os"+ "path/filepath"+ "testing"+)++// referenceRotationScan is the per-page scanner scanPageDirectives replaced.+// It is kept here verbatim so the merged scan can be proven equivalent on+// every input, including malformed ones.+func referenceRotationScan(data []byte, pageIdx int) int {+ var rotations []int+ rest := data+ for {+ idx := -1+ for i := 0; i < len(rest)-7; i++ {+ if rest[i] == '/' && rest[i+1] == 'R' && rest[i+2] == 'o' &&+ rest[i+3] == 't' && rest[i+4] == 'a' && rest[i+5] == 't' &&+ rest[i+6] == 'e' {+ idx = i+ break+ }+ }+ if idx < 0 {+ break+ }+ rest = rest[idx+7:]+ for len(rest) > 0 && (rest[0] == ' ' || rest[0] == '\t' || rest[0] == '\n' || rest[0] == '\r') {+ rest = rest[1:]+ }+ if len(rest) == 0 {+ break+ }+ val := 0+ i := 0+ for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {+ val = val*10 + int(rest[i]-'0')+ i+++ }+ if i > 0 {+ rotations = append(rotations, val)+ }+ rest = rest[i:]+ }+ if pageIdx < len(rotations) {+ return rotations[pageIdx]+ }+ return 0+}++// referenceIndexAfter is the byte search the previous CropBox scanner used.+func referenceIndexAfter(data []byte, s string) int {+ for i := 0; i < len(data)-len(s); i++ {+ match := true+ for j := 0; j < len(s); j++ {+ if data[i+j] != s[j] {+ match = false+ break+ }+ }+ if match {+ return i + len(s)+ }+ }+ return -1+}++// referenceCropBoxScan is the per-page /CropBox scanner scanPageDirectives+// replaced.+func referenceCropBoxScan(data []byte, pageIdx int) ([4]float64, bool) {+ type cb [4]float64+ var boxes []cb+ rest := data+ for {+ idx := referenceIndexAfter(rest, "/CropBox")+ if idx < 0 {+ break+ }+ rest = rest[idx:]+ for len(rest) > 0 && isSpace(rest[0]) {+ rest = rest[1:]+ }+ if len(rest) == 0 || rest[0] != '[' {+ continue+ }+ rest = rest[1:]+ var vals [4]float64+ ok := true+ for i := 0; i < 4; i++ {+ for len(rest) > 0 && isSpace(rest[0]) {+ rest = rest[1:]+ }+ v, n := parseFloat(rest)+ if n == 0 {+ ok = false+ break+ }+ vals[i] = v+ rest = rest[n:]+ }+ if !ok {+ continue+ }+ boxes = append(boxes, cb(vals))+ }+ if pageIdx < len(boxes) {+ return boxes[pageIdx], true+ }+ return [4]float64{}, false+}++// comparedPages bounds how many leading page indexes each comparison checks.+// It exceeds the page count of the shipped fixtures so real PDFs are covered+// end to end.+const comparedPages = 32++// compareScanners asserts that the merged scan reports exactly what the two+// former per-page scanners reported for every page index in range.+func compareScanners(t *testing.T, data []byte, label string) {+ t.Helper()+ got := scanPageDirectives(data)+ for page := 0; page < comparedPages; page++ {+ wantRot := referenceRotationScan(data, page)+ if d := got[page]; d.rotate != wantRot {+ t.Fatalf("%s: page %d rotate = %d, want %d", label, page, d.rotate, wantRot)+ }+ wantBox, wantOK := referenceCropBoxScan(data, page)+ d := got[page]+ if d.hasCrop != wantOK {+ t.Fatalf("%s: page %d hasCrop = %v, want %v", label, page, d.hasCrop, wantOK)+ }+ if wantOK {+ for i := 0; i < 4; i++ {+ if d.cropBox[i] != wantBox[i] {+ t.Fatalf("%s: page %d cropBox[%d] = %v, want %v", label, page, i, d.cropBox[i], wantBox[i])+ }+ }+ }+ }+}++func TestScanPageDirectivesMatchesPreviousScanners(t *testing.T) {+ cases := []struct {+ name string+ raw string+ }{+ {"empty", ""},+ {"no directives", "/MediaBox [0 0 595 842] /Type /Page"},+ {"rotate only", "/Rotate 90"},+ {"cropbox only", "/CropBox [0 0 595 842]"},+ {"no space after key", "/CropBox[0 0 595 842]/Rotate 180"},+ {"extra whitespace", "/CropBox [ 0.5 10.25 595.3 842.0 ]"},+ {"negative cropbox", "/CropBox [-10 -20 595 842]"},+ {"interleaved pages", "/Rotate 90 /CropBox [0 0 1 1] /Rotate 270 /CropBox [2 2 3 3]"},+ {"cropbox before rotate", "/CropBox [5 5 6 6] /Rotate 90"},+ {"non-numeric rotate is skipped", "/Rotate </Rotate 90"},+ {"negative rotate is skipped", "/Rotate -90 /Rotate 90"},+ {"malformed cropbox is skipped", "/CropBox [0 0 595] /CropBox [0 0 1 1]"},+ {"cropbox without array", "/CropBox /Rotate 90 /CropBox [0 0 1 1]"},+ {"trailing rotate", "/Rotate 90<<>>"},+ {"keyword inside value", "/CropBox [0 0 595 842] /Rotate 90 endobj /Rotate 180"},+ {"media box only", "/MediaBox [0 0 612 792] endobj /MediaBox [0 0 612 792]"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ compareScanners(t, []byte(tc.raw), tc.name)+ })+ }+}++func TestScanPageDirectivesRandomizedEquivalence(t *testing.T) {+ tokens := []string{+ "/Rotate", " 90", " 0", " 270", " -45", " </Rotate", " abc",+ "/CropBox", " [0 0 1 1]", " [ 10.5 -2 30 40 ]", " [0 0 595]",+ " ", "\n", "/MediaBox", "[0 0 9 9]", "endobj", "/Type /Page",+ }+ rng := rand.New(rand.NewSource(20261104))+ for iter := 0; iter < 300; iter++ {+ var buf []byte+ for n := rng.Intn(40); n > 0; n-- {+ buf = append(buf, tokens[rng.Intn(len(tokens))]...)+ }+ compareScanners(t, buf, "random")+ }+}++// TestScanPageDirectivesOnRealPDFs sweeps every PDF shipped in testdata so the+// merged scan is compared against both former scanners on real byte streams,+// not only on synthetic ones.+func TestScanPageDirectivesOnRealPDFs(t *testing.T) {+ paths, err := filepath.Glob(filepath.Join("..", "testdata", "real_pdfs", "*.pdf"))+ if err != nil {+ t.Fatalf("glob: %v", err)+ }+ if len(paths) == 0 {+ t.Skip("no real PDFs in ../testdata/real_pdfs")+ }+ for _, path := range paths {+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("read %s: %v", path, err)+ }+ compareScanners(t, data, filepath.Base(path))+ }+}++func TestScanPageDirectivesCropBox(t *testing.T) {+ eps := 1e-6++ tests := []struct {+ name string+ raw string+ pageIdx int+ want [4]float64+ ok bool+ }{+ {+ name: "standard A4 portrait",+ raw: "/CropBox [0 0 595.28 841.89]",+ want: [4]float64{0, 0, 595.28, 841.89},+ ok: true,+ },+ {+ name: "non-zero origin",+ raw: "/CropBox [30 20 575 832]",+ want: [4]float64{30, 20, 575, 832},+ ok: true,+ },+ {+ name: "with extra whitespace",+ raw: "/CropBox [ 0.5 10.25 595.3 842.0 ]",+ want: [4]float64{0.5, 10.25, 595.3, 842.0},+ ok: true,+ },+ {+ name: "no spaces inside brackets",+ raw: "/CropBox[0 0 595 842]",+ want: [4]float64{0, 0, 595, 842},+ ok: true,+ },+ {+ name: "page index 1 picks second CropBox",+ raw: "/CropBox [0 0 1 1] /Rotate 90 /CropBox [2 2 3 3]",+ pageIdx: 1,+ want: [4]float64{2, 2, 3, 3},+ ok: true,+ },+ {+ name: "page index out of range",+ raw: "/CropBox [0 0 1 1]",+ pageIdx: 5,+ want: [4]float64{},+ ok: false,+ },+ {+ name: "no cropbox",+ raw: "/MediaBox [0 0 595 842] /Rotate 90",+ want: [4]float64{},+ ok: false,+ },+ {+ name: "empty input",+ raw: "",+ want: [4]float64{},+ ok: false,+ },+ {+ name: "incomplete array — fewer than 4 values",+ raw: "/CropBox [0 0 595]",+ want: [4]float64{},+ ok: false,+ },+ {+ name: "negative values",+ raw: "/CropBox [-10 -20 595 842]",+ want: [4]float64{-10, -20, 595, 842},+ ok: true,+ },+ {+ name: "real pypdf output format (multiple spaces, decimals)",+ raw: "/Type /Page /MediaBox [0 0 595.2756 841.8898] /CropBox [30.0 20.0 575.0 832.0] /Rotate 90",+ want: [4]float64{30.0, 20.0, 575.0, 832.0},+ ok: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ d := scanPageDirectives([]byte(tt.raw))[tt.pageIdx]+ got, ok := d.cropBox, d.hasCrop+ if ok != tt.ok {+ t.Fatalf("ok=%v want %v", ok, tt.ok)+ }+ if !ok {+ return+ }+ for i := 0; i < 4; i++ {+ if math.Abs(got[i]-tt.want[i]) > eps {+ t.Errorf("[%d]: got %.4f, want %.4f", i, got[i], tt.want[i])+ }+ }+ })+ }+}++func TestParseFloat(t *testing.T) {+ tests := []struct {+ s string+ want float64+ n int+ }{+ {"0", 0, 1},+ {"595.28", 595.28, 6},+ {" 42", 42, 4},+ {"-10.5", -10.5, 5},+ {"+3.14", 3.14, 5},+ {"123abc", 123, 3},+ {"abc", 0, 0},+ {"", 0, 0},+ {".5", 0.5, 2},+ }+ for _, tt := range tests {+ v, n := parseFloat([]byte(tt.s))+ if n != tt.n || math.Abs(v-tt.want) > 1e-6 {+ t.Errorf("parseFloat(%q) = (%.4f, %d), want (%.4f, %d)",+ tt.s, v, n, tt.want, tt.n)+ }+ }+}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.go b/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.gonew file mode 100644index 0000000000..3941e32aee--- /dev/null+++ b/internal/deepdoc/parser/pdf/pdfoxide/page_dims_cache_test.go@@ -0,0 +1,253 @@+//go:build cgo++package pdfoxide++import (+ "context"+ "errors"+ "fmt"+ "math"+ "os"+ "path/filepath"+ "testing"++ "ragflow/internal/deepdoc/parser/pdf/pdfium"+)++// pdfFixture is one PDF used by the table-driven tests below.+type pdfFixture struct {+ name string+ data []byte+}++// TestPageDimsCacheMatchesPerPageLookup proves the document-wide size pass+// reproduces exactly what the former per-page pdfium call returned. The+// generated fixture keeps the test meaningful in a clean checkout; every real+// PDF shipped next to the parser is swept as well when present.+func TestPageDimsCacheMatchesPerPageLookup(t *testing.T) {+ const pages = 3+ generated := buildTestPDF(t, pages, 90)++ paths, err := filepath.Glob(filepath.Join("..", "testdata", "real_pdfs", "*.pdf"))+ if err != nil {+ t.Fatalf("glob: %v", err)+ }++ cases := []pdfFixture{{name: "generated", data: generated}}+ for _, path := range paths {+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatalf("read %s: %v", path, err)+ }+ cases = append(cases, pdfFixture{name: filepath.Base(path), data: data})+ }+ if len(paths) == 0 {+ t.Log("no real PDFs in ../testdata/real_pdfs; generated fixture only")+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ eng, err := NewEngine(tc.data)+ if err != nil {+ t.Fatalf("NewEngine: %v", err)+ }+ defer eng.Close()++ pageCount, err := eng.PageCount()+ if err != nil {+ t.Fatalf("PageCount: %v", err)+ }+ if pageCount == 0 {+ t.Fatalf("PageCount = 0")+ }++ for page := 0; page < pageCount; page++ {+ wantW, wantH, wantErr := pdfium.PageSize(tc.data, page)+ gotD, gotOK := eng.pageSize(page)+ if wantErr != nil {+ if gotOK {+ t.Fatalf("page %d: cached size despite pdfium error %v", page, wantErr)+ }+ continue+ }+ if !gotOK {+ t.Fatalf("page %d: size missing from cache", page)+ }+ if gotD[0] != wantW || gotD[1] != wantH {+ t.Fatalf("page %d: cached %.4fx%.4f, want %.4fx%.4f", page, gotD[0], gotD[1], wantW, wantH)+ }+ }+ })+ }+}++// TestPageSizeBatchPartialFailure pins the failure contract: a page missing+// from the batch is retried on its own, the retry is cached, and the batch is+// never re-run because some page failed.+func TestPageSizeBatchPartialFailure(t *testing.T) {+ eng, err := NewEngine(buildTestPDF(t, 2, 0))+ if err != nil {+ t.Fatalf("NewEngine: %v", err)+ }+ defer eng.Close()++ var batchCalls, singleCalls int+ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {+ batchCalls+++ // Page 1 is unreadable in this pass; page 0 succeeded.+ return map[int][2]float64{0: {200, 100}}, 2, errors.New("page 1 unreadable")+ }+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {+ singleCalls+++ return 200, 100, nil+ }++ if d, ok := eng.pageSize(0); !ok || d != [2]float64{200, 100} {+ t.Fatalf("page 0: got %v ok=%v, want cached batch size", d, ok)+ }+ if singleCalls != 0 {+ t.Fatalf("page 0 must come from the batch, single-page lookups = %d", singleCalls)+ }++ if d, ok := eng.pageSize(1); !ok || d != [2]float64{200, 100} {+ t.Fatalf("page 1: got %v ok=%v, want single-page retry result", d, ok)+ }+ if singleCalls != 1 {+ t.Fatalf("page 1 must be retried once, got %d lookups", singleCalls)+ }++ if _, ok := eng.pageSize(1); !ok || singleCalls != 1 {+ t.Fatalf("page 1 retry must be cached, lookups = %d", singleCalls)+ }+ if _, ok := eng.pageSize(0); !ok || batchCalls != 1 {+ t.Fatalf("partial batch must not run again, batch calls = %d", batchCalls)+ }+}++// TestPageSizeFallbackWhenBothLookupsFail keeps the original fallback chain:+// only when the batch and the single-page retry both fail does PageSize answer+// with pdf_oxide's raw dimensions.+func TestPageSizeFallbackWhenBothLookupsFail(t *testing.T) {+ data := buildTestPDF(t, 1, 0)+ eng, err := NewEngine(data)+ if err != nil {+ t.Fatalf("NewEngine: %v", err)+ }+ defer eng.Close()++ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {+ return nil, 1, errors.New("batch failed")+ }+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {+ return 0, 0, errors.New("single page failed")+ }++ wantW, wantH, err := eng.doc.PageSize(0)+ if err != nil {+ t.Fatalf("oxide PageSize: %v", err)+ }+ gotW, gotH, err := eng.PageSize(0)+ if err != nil {+ t.Fatalf("PageSize: %v", err)+ }+ if gotW != wantW || gotH != wantH {+ t.Fatalf("PageSize = %.4fx%.4f, want oxide fallback %.4fx%.4f", gotW, gotH, wantW, wantH)+ }+}++// TestExtractCharsUsesSinglePageRetry covers the regression the batch cache+// could introduce: a page missing from the batch must still get its pdfium+// size (and therefore correct rotation) instead of silently falling back to+// the unrotated pdf_oxide dimensions.+func TestExtractCharsUsesSinglePageRetry(t *testing.T) {+ const rotate = 90+ data := buildTestPDF(t, 2, rotate)+ wantW, wantH, err := pdfium.PageSize(data, 1)+ if err != nil {+ t.Fatalf("pdfium.PageSize: %v", err)+ }++ eng, err := NewEngine(data)+ if err != nil {+ t.Fatalf("NewEngine: %v", err)+ }+ defer eng.Close()++ eng.pageSizesBatch = func(context.Context, []byte, []int) (map[int][2]float64, int, error) {+ return map[int][2]float64{0: {200, 100}}, 2, errors.New("page 1 unreadable")+ }+ eng.pageSizeSingle = func([]byte, int) (float64, float64, error) {+ return wantW, wantH, nil+ }++ w, h, err := eng.PageSize(1)+ if err != nil {+ t.Fatalf("PageSize: %v", err)+ }+ if w != wantW || h != wantH {+ t.Fatalf("PageSize = %.2fx%.2f, want %.2fx%.2f", w, h, wantW, wantH)+ }++ chars, err := eng.ExtractChars(1)+ if err != nil {+ t.Fatalf("ExtractChars: %v", err)+ }+ assertCharsInsidePage(t, chars, w, h)+}++// TestExtractCharsRotatedPageCoordinates is the clean-checkout coordinate+// check the previous manual tier owned: a rotated page must yield characters+// inside the post-rotation page box.+func TestExtractCharsRotatedPageCoordinates(t *testing.T) {+ cases := []struct {+ rotate int+ wantW float64+ wantH float64+ }{+ {0, 200, 100},+ {90, 100, 200},+ {180, 200, 100},+ {270, 100, 200},+ }+ const eps = 0.01+ for _, tc := range cases {+ t.Run(fmt.Sprintf("rotate-%d", tc.rotate), func(t *testing.T) {+ data := buildTestPDF(t, 1, tc.rotate)+ eng, err := NewEngine(data)+ if err != nil {+ t.Fatalf("NewEngine: %v", err)+ }+ defer eng.Close()++ w, h, err := eng.PageSize(0)+ if err != nil {+ t.Fatalf("PageSize: %v", err)+ }+ if math.Abs(w-tc.wantW) > eps || math.Abs(h-tc.wantH) > eps {+ t.Fatalf("PageSize = %.2fx%.2f, want %.2fx%.2f", w, h, tc.wantW, tc.wantH)+ }++ chars, err := eng.ExtractChars(0)+ if err != nil {+ t.Fatalf("ExtractChars: %v", err)+ }+ assertCharsInsidePage(t, chars, w, h)+ })+ }+}++// assertCharsInsidePage fails when any extracted character falls outside the+// post-rotation page box.+func assertCharsInsidePage(t *testing.T, chars []Char, w, h float64) {+ t.Helper()+ if len(chars) == 0 {+ t.Fatalf("no characters extracted")+ }+ const eps = 0.01+ for _, c := range chars {+ if c.X0 < -eps || c.X1 > w+eps || c.Top < -eps || c.Bottom > h+eps {+ t.Fatalf("char %q out of post-rotation box %.1fx%.1f: x=[%.2f,%.2f] y=[%.2f,%.2f]",+ c.Text, w, h, c.X0, c.X1, c.Top, c.Bottom)+ }+ }+}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.go b/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.gonew file mode 100644index 0000000000..2a6098a22a--- /dev/null+++ b/internal/deepdoc/parser/pdf/pdfoxide/pdf_fixture_test.go@@ -0,0 +1,54 @@+//go:build cgo++package pdfoxide++import (+ "bytes"+ "fmt"+ "testing"+)++// buildTestPDF returns a minimal but valid PDF with `pages` identical pages at+// the given /Rotate. The bytes are generated in the test so the PDF-parsing+// tests run in a clean checkout, without a tracked binary fixture.+func buildTestPDF(t *testing.T, pages, rotate int) []byte {+ t.Helper()+ if pages < 1 {+ pages = 1+ }++ var buf bytes.Buffer+ var offsets []int+ writeObj := func(s string) {+ offsets = append(offsets, buf.Len())+ buf.WriteString(s)+ }++ buf.WriteString("%PDF-1.4\n")+ writeObj("1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")++ kids := ""+ for p := 0; p < pages; p++ {+ kids += fmt.Sprintf("%d 0 R ", 3+p)+ }+ fontObj := 3 + pages+ contentObj := 4 + pages+ writeObj(fmt.Sprintf("2 0 obj\n<< /Type /Pages /Kids [%s] /Count %d >>\nendobj\n", kids, pages))+ for p := 0; p < pages; p++ {+ writeObj(fmt.Sprintf(+ "%d 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 100] /Rotate %d "++ "/Resources << /Font << /F1 %d 0 R >> >> /Contents %d 0 R >>\nendobj\n",+ 3+p, rotate, fontObj, contentObj))+ }+ writeObj(fmt.Sprintf("%d 0 obj\n<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>\nendobj\n", fontObj))+ stream := "BT /F1 24 Tf 20 40 Td (Hello) Tj ET"+ writeObj(fmt.Sprintf("%d 0 obj\n<< /Length %d >>\nstream\n%s\nendstream\nendobj\n", contentObj, len(stream), stream))++ xref := buf.Len()+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)+ for _, off := range offsets {+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)+ }+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)+ return buf.Bytes()+}diff --git a/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go b/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.goindex 1fa7911db7..c294ec3d02 100644--- a/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go+++ b/internal/deepdoc/parser/pdf/pdfoxide/pdf_oxide_engine.go@@ -3,8 +3,10 @@package pdfoxideimport (+ "context""image""math"+ "sync""ragflow/internal/deepdoc/parser/pdf/pdfium")@@ -20,9 +22,29 @@ type Char struct {}// Engine wraps pdf_oxide to extract chars and render pages.+//+// rawData and doc belong to exactly one document (NewEngine opens one and+// Close releases it), so the per-document lookups below are cached on the+// engine: the byte-level /Rotate and /CropBox scan runs once, and pdfium's+// page sizes are read in one pass instead of one full document reload per+// page.type Engine struct {doc *DocumentrawData []byte++ mu sync.Mutex+ directives map[int]pageDirectives+ // pageDims caches the page sizes pdfium has reported so far; dimsAttempted+ // records that the document-wide pass already ran, so a page missing from+ // the map is retried on its own rather than by re-reading every page.+ pageDims map[int][2]float64+ dimsAttempted bool++ // pageSizesBatch and pageSizeSingle are the pdfium lookups this engine+ // uses. They are fields rather than package variables so a test can+ // simulate a partial batch on one engine without touching global state.+ pageSizesBatch func(ctx context.Context, pdf []byte, pages []int) (map[int][2]float64, int, error)+ pageSizeSingle func(pdf []byte, page int) (float64, float64, error)}// NewEngine opens a PDF from bytes and returns an Engine.@@ -31,11 +53,91 @@ func NewEngine(pdfBytes []byte) (*Engine, error) {if err != nil {return nil, err}- return &Engine{doc: doc, rawData: pdfBytes}, nil+ return &Engine{+ doc: doc,+ rawData: pdfBytes,+ pageSizesBatch: pdfium.PageSizes,+ pageSizeSingle: pdfium.PageSize,+ }, nil}func (e *Engine) RawData() []byte { return e.rawData }+// loadPageDims fills pageDims with every page size pdfium can report from one+// batch pass. The pass runs once: successful pages are cached even when other+// pages fail, so a missing page is retried individually by pageSize instead of+// re-reading the whole document on every call. A failed pass is not retried in+// bulk; only the pages that missed out are looked up again.+//+// The pass has no caller context to honour: ExtractChars carries none, and+// threading one through the PDFEngine interface is a separate change.+func (e *Engine) loadPageDims() {+ e.mu.Lock()+ defer e.mu.Unlock()+ if e.dimsAttempted {+ return+ }+ e.dimsAttempted = true+ n, err := e.doc.PageCount()+ if err != nil || n <= 0 {+ return+ }+ idxs := make([]int, n)+ for i := range idxs {+ idxs[i] = i+ }+ sizes, _, _ := e.pageSizesBatch(context.Background(), e.rawData, idxs)+ if len(sizes) == 0 {+ return+ }+ if e.pageDims == nil {+ e.pageDims = make(map[int][2]float64, len(sizes))+ }+ for page, size := range sizes {+ e.pageDims[page] = size+ }+}++// pageSize returns a page's post-rotation size. It prefers the document-wide+// batch, then retries that one page directly, and reports false only when both+// fail. The retry result is cached so a page pdfium cannot size costs at most+// one extra lookup per document. Both ExtractChars and PageSize use this helper+// so the fallback rule cannot diverge between them.+func (e *Engine) pageSize(pageNum int) ([2]float64, bool) {+ e.loadPageDims()++ e.mu.Lock()+ d, ok := e.pageDims[pageNum]+ e.mu.Unlock()+ if ok {+ return d, true+ }++ w, h, err := e.pageSizeSingle(e.rawData, pageNum)+ if err != nil {+ return [2]float64{}, false+ }+ d = [2]float64{w, h}+ e.mu.Lock()+ if e.pageDims == nil {+ e.pageDims = make(map[int][2]float64, 1)+ }+ e.pageDims[pageNum] = d+ e.mu.Unlock()+ return d, true+}++// pageDirective returns the cached /Rotate and /CropBox of one page, scanning+// the raw bytes on first use.+func (e *Engine) pageDirective(pageNum int) pageDirectives {+ e.mu.Lock()+ defer e.mu.Unlock()+ if e.directives == nil {+ e.directives = scanPageDirectives(e.rawData)+ }+ return e.directives[pageNum]+}+func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {chars, err := e.doc.GetDedupePageChars(pageNum, 0.5)if err != nil {@@ -52,15 +154,16 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {// If dimensions are swapped, the page has implicit rotation// (inherited /Rotate or ContentBox rotation).rawW, rawH, _ := e.doc.PageSize(pageNum)- effW, effH, pdfErr := pdfium.PageSize(e.rawData, pageNum)- if pdfErr != nil {- effW, effH = rawW, rawH+ effW, effH := rawW, rawH+ if d, ok := e.pageSize(pageNum); ok {+ effW, effH = d[0], d[1]}dimSwapped := rawW > 0 && rawH > 0 && effW > 0 && effH > 0 &&math.Abs(rawW-effH) < 1 && math.Abs(rawH-effW) < 1- rawRot := parsePageRotationFromRaw(e.rawData, pageNum)+ dir := e.pageDirective(pageNum)+ rawRot := dir.rotateneedsRotate := falserotation90 := false@@ -86,7 +189,7 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {// CropBox correction — shift origin if CropBox differs from MediaBox.var cropDX, cropDY float64- realCrop, hasCrop := parseCropBoxFromRaw(e.rawData, pageNum)+ realCrop, hasCrop := dir.cropBox, dir.hasCropif hasCrop {cropH := realCrop[3] - realCrop[1]oxideCropH := rawH@@ -173,51 +276,6 @@ func (e *Engine) ExtractChars(pageNum int) ([]Char, error) {return result, nil}-// parsePageRotationFromRaw scans raw PDF bytes for /Rotate entries.-// Returns the rotation value for the given page index, or 0 if not found.-// NOTE: This only finds /Rotate defined directly on page objects.-// Inherited /Rotate (from parent Pages dict) is not detected here but-// is caught by the dimension-comparison fallback in ExtractChars.-func parsePageRotationFromRaw(data []byte, pageIdx int) int {- var rotations []int- rest := data- for {- idx := -1- for i := 0; i < len(rest)-7; i++ {- if rest[i] == '/' && rest[i+1] == 'R' && rest[i+2] == 'o' &&- rest[i+3] == 't' && rest[i+4] == 'a' && rest[i+5] == 't' &&- rest[i+6] == 'e' {- idx = i- break- }- }- if idx < 0 {- break- }- rest = rest[idx+7:]- for len(rest) > 0 && (rest[0] == ' ' || rest[0] == '\t' || rest[0] == '\n' || rest[0] == '\r') {- rest = rest[1:]- }- if len(rest) == 0 {- break- }- val := 0- i := 0- for i < len(rest) && rest[i] >= '0' && rest[i] <= '9' {- val = val*10 + int(rest[i]-'0')- i++- }- if i > 0 {- rotations = append(rotations, val)- }- rest = rest[i:]- }- if pageIdx < len(rotations) {- return rotations[pageIdx]- }- return 0-}-// RenderPageImage uses pdfium for page rendering — pdfium correctly// applies /Rotate so the output matches character coordinates and DLA.// There is no pdf_oxide fallback because pdf_oxide does not apply@@ -236,13 +294,14 @@ func (e *Engine) RenderPage(pageNum int, dpi float64) ([]byte, error) {// PageSize returns the effective page dimensions via pdfium, which// correctly applies /Rotate. pdf_oxide's own PageSize returns raw-// (unrotated) dimensions.+// (unrotated) dimensions. Sizes come from the document-wide cache, with a+// single-page pdfium retry for a page the batch could not size; only when+// both fail does it fall back to pdf_oxide, exactly as before.func (e *Engine) PageSize(pageNum int) (float64, float64, error) {- w, h, err := pdfium.PageSize(e.rawData, pageNum)- if err != nil {- return e.doc.PageSize(pageNum)+ if d, ok := e.pageSize(pageNum); ok {+ return d[0], d[1], nil}- return w, h, nil+ return e.doc.PageSize(pageNum)}func (e *Engine) PageCount() (int, error) { return e.doc.PageCount() }func (e *Engine) Close() error { e.doc.Close(); return nil }