Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): read PDF bookmarks into remote parse results (#21499)

CCLUCKY0c8af782cbe8
diff --git a/internal/deepdoc/parser/pdf/pdfium/pdfium.go b/internal/deepdoc/parser/pdf/pdfium/pdfium.go
index 3f899adbe8..c482fa16ca 100644
--- a/internal/deepdoc/parser/pdf/pdfium/pdfium.go
+++ b/internal/deepdoc/parser/pdf/pdfium/pdfium.go
@@ -56,15 +56,9 @@ import (
"unsafe"
pdfsync "ragflow/internal/deepdoc/parser/pdf/pdfsync"
+ doctype "ragflow/internal/deepdoc/parser/type"
)
-// Outline represents one entry in a PDF document outline (table of contents).
-type Outline struct {
- Title string
- Level int
- PageNumber int // 1-indexed, matching Python
-}
-
var initOnce sync.Once
// Init initializes the PDFium library. Safe to call multiple times.
@@ -248,11 +242,11 @@ func openPage(pdfData []byte, pageIdx int) (
// Traversal is iterative with an explicit stack to avoid stack overflow on deep
// outline trees. pdfium is not thread-safe; all FPDF_* calls are serialized by
// the shared pdfsync.Mu.
-func ExtractOutlines(pdfData []byte) []Outline {
+func ExtractOutlines(pdfData []byte) []doctype.Outline {
if len(pdfData) == 0 {
return nil
}
- var out []Outline
+ var out []doctype.Outline
pdfsync.With(func() {
Init()
@@ -270,7 +264,7 @@ func ExtractOutlines(pdfData []byte) []Outline {
level int
}
- var result []Outline
+ var result []doctype.Outline
stack := []frame{{bm: C.FPDFBookmark_GetFirstChild(doc, nil), level: 0}}
for len(stack) > 0 {
@@ -292,7 +286,7 @@ func ExtractOutlines(pdfData []byte) []Outline {
}
}
- result = append(result, Outline{Title: title, Level: top.level, PageNumber: pageNum})
+ result = append(result, doctype.Outline{Title: title, Level: top.level, PageNumber: pageNum})
// Push siblings after children so children are processed first (pre-order).
if sibling := C.FPDFBookmark_GetNextSibling(doc, top.bm); sibling != nil {
diff --git a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
index 799aad4a8e..23c4c5680a 100644
--- a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
+++ b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go
@@ -29,12 +29,7 @@ func (e *PDFOxideEngine) PageCount() (int, error) { return e.Inner.PageCount() }
func (e *PDFOxideEngine) Close() error { return e.Inner.Close() }
func (e *PDFOxideEngine) Outlines() ([]pdf.Outline, error) {
- ol := pdfium.ExtractOutlines(e.Inner.RawData())
- result := make([]pdf.Outline, len(ol))
- for i, o := range ol {
- result[i] = pdf.Outline{Title: o.Title, Level: o.Level, PageNumber: o.PageNumber}
- }
- return result, nil
+ return pdfium.ExtractOutlines(e.Inner.RawData()), nil
}
func (e *PDFOxideEngine) RenderPage(pageNum int, dpi float64) ([]byte, error) {
diff --git a/internal/ingestion/component/pdf_outline_cgo_test.go b/internal/ingestion/component/pdf_outline_cgo_test.go
new file mode 100644
index 0000000000..a8c8ec5fb5
--- /dev/null
+++ b/internal/ingestion/component/pdf_outline_cgo_test.go
@@ -0,0 +1,80 @@
+//go:build cgo
+
+package component
+
+import (
+ "archive/zip"
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+
+ "gorm.io/gorm"
+
+ modelModule "ragflow/internal/entity/models"
+ "ragflow/internal/ingestion/component/schema"
+ "ragflow/internal/utility"
+)
+
+// TestPDFDispatchAttachesPDFOutlines pins the outline attachment on the
+// ingestion dispatch boundary: the tenant parser serves content only, so the
+// bookmarks must come from the source bytes the component already holds.
+// Reverting the call in maybeDispatchPDFVision's normalize closure fails here
+// and nowhere else, because the built-in parse methods have their own boundary.
+func TestPDFDispatchAttachesPDFOutlines(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var buf bytes.Buffer
+ archive := zip.NewWriter(&buf)
+ file, _ := archive.Create("sample/sample.json")
+ _ = json.NewEncoder(file).Encode(map[string]any{"layouts": []any{map[string]any{"label": "Text", "content": "body"}}})
+ _ = archive.Close()
+ _, _ = w.Write(buf.Bytes())
+ }))
+ defer server.Close()
+ previousResolver := resolveTenantOCRModelByProvider
+ t.Cleanup(func() { resolveTenantOCRModelByProvider = previousResolver })
+ resolveTenantOCRModelByProvider = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return &monkeyOCRv2FakeDriver{}, "monkey", &modelModule.APIConfig{}, 0, nil
+ }
+ setup := schema.ParserSetup{"parse_method": "monkeyocrv2", "monkeyocrv2_server_url": server.URL}
+
+ result, handled, err := maybeDispatchPDFVision(t.Context(), nil, utility.FileTypePDF, "bookmarked.pdf", singleBookmarkPDF(t), nil, map[string]schema.ParserSetup{"pdf": setup})
+ if err != nil || !handled || !result.Produced() {
+ t.Fatalf("dispatch failed: handled=%v err=%v", handled, err)
+ }
+ outline, ok := result.File["outline"].([]map[string]any)
+ if !ok || len(outline) != 1 || outline[0]["title"] != "Chapter 1" {
+ t.Fatalf("File[outline] = %#v, want the bookmark read from the source PDF", result.File["outline"])
+ }
+}
+
+// singleBookmarkPDF builds the smallest PDF that carries one bookmark, so the
+// dispatch boundary has real bytes to extract from.
+func singleBookmarkPDF(t *testing.T) []byte {
+ t.Helper()
+ objects := []string{
+ "<< /Type /Catalog /Pages 2 0 R /Outlines 3 0 R >>",
+ "<< /Type /Pages /Kids [4 0 R] /Count 1 >>",
+ "<< /Type /Outlines /First 5 0 R /Last 5 0 R /Count 1 >>",
+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 6 0 R >>",
+ "<< /Title (Chapter 1) /Parent 3 0 R /Dest [4 0 R /XYZ null null null] >>",
+ "<< /Length 0 >>\nstream\n\nendstream",
+ }
+ var buf bytes.Buffer
+ buf.WriteString("%PDF-1.4\n")
+ offsets := make([]int, len(objects))
+ for i, obj := range objects {
+ offsets[i] = buf.Len()
+ fmt.Fprintf(&buf, "%d 0 obj\n%s\nendobj\n", i+1, obj)
+ }
+ xref := buf.Len()
+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1)
+ for _, off := range offsets {
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)
+ }
+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref)
+ return buf.Bytes()
+}
diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.go
index a9664970cf..b1af9f8749 100644
--- a/internal/ingestion/component/pdf_vision_dispatch.go
+++ b/internal/ingestion/component/pdf_vision_dispatch.go
@@ -91,7 +91,7 @@ func maybeDispatchPDFVision(
flatten, _ := setup["flatten_media_to_text"].(bool)
normalize := func(result parser.ParseResult) parser.ParseResult {
pdf := parser.PDFParser{FlattenMediaToText: flatten}
- return pdf.NormalizeResult(ctx, filename, result)
+ return parser.AttachPDFOutlines(pdf.NormalizeResult(ctx, filename, result), binary)
}
if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {
if err != nil {
diff --git a/internal/parser/parser/pdf_outline.go b/internal/parser/parser/pdf_outline.go
new file mode 100644
index 0000000000..10dabc6f15
--- /dev/null
+++ b/internal/parser/parser/pdf_outline.go
@@ -0,0 +1,26 @@
+package parser
+
+// pdfOutlinesFromPDF reads a PDF's bookmark tree. The native engine backs it
+// under cgo; without cgo it reports no bookmarks.
+var pdfOutlinesFromPDF = pdfOutlinesFromData
+
+// AttachPDFOutlines populates File["outline"] from the source PDF's bookmarks,
+// mutating result.File in place. Remote engines return document content only,
+// so their results carry an empty outline and the title chunker, which derives
+// heading levels from bookmarks, has nothing to work with. A result that
+// already carries an outline keeps it, so the engine that read the bookmarks
+// stays authoritative. Both boundaries that hand out a PDF parse result — the
+// built-in parse methods and the ingestion dispatch — hold the source bytes and
+// call this themselves.
+func AttachPDFOutlines(result ParseResult, data []byte) ParseResult {
+ if result.Err != nil || result.File == nil {
+ return result
+ }
+ if existing, ok := result.File["outline"].([]map[string]any); ok && len(existing) > 0 {
+ return result
+ }
+ if outlines := pdfOutlinesFromPDF(data); len(outlines) > 0 {
+ result.File["outline"] = outlinesToFileMeta(outlines)
+ }
+ return result
+}
diff --git a/internal/parser/parser/pdf_outline_cgo.go b/internal/parser/parser/pdf_outline_cgo.go
new file mode 100644
index 0000000000..fa84190813
--- /dev/null
+++ b/internal/parser/parser/pdf_outline_cgo.go
@@ -0,0 +1,12 @@
+//go:build cgo
+
+package parser
+
+import (
+ "ragflow/internal/deepdoc/parser/pdf/pdfium"
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+func pdfOutlinesFromData(data []byte) []deepdoctype.Outline {
+ return pdfium.ExtractOutlines(data)
+}
diff --git a/internal/parser/parser/pdf_outline_cgo_test.go b/internal/parser/parser/pdf_outline_cgo_test.go
new file mode 100644
index 0000000000..2a244c3e0f
--- /dev/null
+++ b/internal/parser/parser/pdf_outline_cgo_test.go
@@ -0,0 +1,40 @@
+//go:build cgo
+
+package parser
+
+import "testing"
+
+// TestPDFOutlineNativeExtractionFeedsRemoteResult is the native-path proof for
+// #21221: a bookmarked PDF parsed by an engine that reads no bookmarks itself
+// (plain_text) must come back with the outline extracted from the source file.
+func TestPDFOutlineNativeExtractionFeedsRemoteResult(t *testing.T) {
+ data := bookmarkedPDF(t)
+
+ p := &PDFParser{ParseMethod: "plain_text"}
+ res := p.ParseWithResult(t.Context(), "bookmarked.pdf", data)
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ outline, ok := res.File["outline"].([]map[string]any)
+ if !ok {
+ t.Fatalf("File[outline] type = %T, want []map[string]any", res.File["outline"])
+ }
+ want := []struct {
+ title string
+ level int
+ page int
+ }{
+ {"Chapter 1", 0, 1},
+ {"Section 1.1", 1, 2},
+ {"Chapter 2", 0, 2},
+ }
+ if len(outline) != len(want) {
+ t.Fatalf("outline = %#v, want %d entries", outline, len(want))
+ }
+ for i, w := range want {
+ got := outline[i]
+ if got["title"] != w.title || got["level"] != w.level || got["page_number"] != w.page {
+ t.Fatalf("outline[%d] = %#v, want title=%q level=%d page=%d", i, got, w.title, w.level, w.page)
+ }
+ }
+}
diff --git a/internal/parser/parser/pdf_outline_nocgo.go b/internal/parser/parser/pdf_outline_nocgo.go
new file mode 100644
index 0000000000..28665dbd5a
--- /dev/null
+++ b/internal/parser/parser/pdf_outline_nocgo.go
@@ -0,0 +1,9 @@
+//go:build !cgo
+
+package parser
+
+import (
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+func pdfOutlinesFromData([]byte) []deepdoctype.Outline { return nil }
diff --git a/internal/parser/parser/pdf_outline_test.go b/internal/parser/parser/pdf_outline_test.go
new file mode 100644
index 0000000000..de2318781f
--- /dev/null
+++ b/internal/parser/parser/pdf_outline_test.go
@@ -0,0 +1,139 @@
+package parser
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "testing"
+
+ deepdoctype "ragflow/internal/deepdoc/parser/type"
+)
+
+// stubPDFOutlines replaces the bookmark reader for one test and reports how
+// many times it was consulted. The seam is package-global, so tests using it
+// must not run in parallel.
+func stubPDFOutlines(t *testing.T, outlines []deepdoctype.Outline) *int {
+ t.Helper()
+ calls := 0
+ previous := pdfOutlinesFromPDF
+ pdfOutlinesFromPDF = func([]byte) []deepdoctype.Outline {
+ calls++
+ return outlines
+ }
+ t.Cleanup(func() { pdfOutlinesFromPDF = previous })
+ return &calls
+}
+
+// remotePDFResult is the file meta a remote engine produces: pdfFileMeta seeds
+// an empty outline.
+func remotePDFResult(filename string) ParseResult {
+ return pdfItemsToResult(filename, []map[string]any{{"text": "body", "page_number": 1}}, 3)
+}
+
+// TestAttachPDFOutlinesFillsRemoteOutline pins the #21221 fix for the remote
+// PDF engines: their result carries an empty outline, so the bookmarks read
+// from the source PDF are what the title chunker uses for heading levels.
+func TestAttachPDFOutlinesFillsRemoteOutline(t *testing.T) {
+ stubPDFOutlines(t, []deepdoctype.Outline{{Title: "1 Introduction", Level: 0, PageNumber: 2}})
+ result := AttachPDFOutlines(remotePDFResult("remote.pdf"), []byte("%PDF-1.4 fake"))
+ outline, ok := result.File["outline"].([]map[string]any)
+ if !ok || len(outline) != 1 {
+ t.Fatalf("File[outline] = %#v, want one entry", result.File["outline"])
+ }
+ if outline[0]["title"] != "1 Introduction" || outline[0]["level"] != 0 || outline[0]["page_number"] != 2 {
+ t.Fatalf("outline entry = %#v", outline[0])
+ }
+}
+
+// TestAttachPDFOutlinesKeepsEngineOutline asserts the engine that already read
+// the bookmarks stays authoritative and is not re-read.
+func TestAttachPDFOutlinesKeepsEngineOutline(t *testing.T) {
+ calls := stubPDFOutlines(t, []deepdoctype.Outline{{Title: "Ignored", Level: 0, PageNumber: 1}})
+ result := AttachPDFOutlines(ParseResult{
+ File: map[string]any{"name": "deep.pdf", "outline": []map[string]any{{"title": "Chapter 1", "level": 0, "page_number": 3}}},
+ }, []byte("%PDF-1.4 fake"))
+ if got := result.File["outline"].([]map[string]any); len(got) != 1 || got[0]["title"] != "Chapter 1" {
+ t.Fatalf("engine outline changed: %#v", result.File["outline"])
+ }
+ if *calls != 0 {
+ t.Fatalf("bookmark reader consulted %d times, want 0", *calls)
+ }
+}
+
+// TestAttachPDFOutlinesKeepsEmptyOutlineWhenAbsent covers a PDF without
+// bookmarks: the key stays an empty slice, never a nil the consumers would have
+// to special-case. An errored or file-less result is untouched.
+func TestAttachPDFOutlinesKeepsEmptyOutlineWhenAbsent(t *testing.T) {
+ stubPDFOutlines(t, nil)
+ result := AttachPDFOutlines(remotePDFResult("plain.pdf"), []byte("%PDF-1.4 fake"))
+ outline, ok := result.File["outline"].([]map[string]any)
+ if !ok || len(outline) != 0 {
+ t.Fatalf("File[outline] = %#v (ok=%v), want empty []map[string]any", result.File["outline"], ok)
+ }
+
+ if failed := AttachPDFOutlines(ParseResult{Err: errors.New("parse failed")}, []byte("%PDF")); failed.File != nil {
+ t.Fatalf("errored result gained a file meta: %#v", failed.File)
+ }
+ if got := AttachPDFOutlines(ParseResult{}, []byte("%PDF")); got.File != nil {
+ t.Fatalf("result without file meta gained one: %#v", got.File)
+ }
+}
+
+// TestParseWithResultAttachesOutline pins the attachment on the built-in parse
+// method boundary (ParseWithResult). The dispatched tenant and vision engines
+// normalize outside this package and have their own call site, pinned by
+// TestPDFDispatchAttachesPDFOutlines in the component package. The bookmark
+// reader is stubbed here, so only a structurally valid PDF is needed as input.
+func TestParseWithResultAttachesOutline(t *testing.T) {
+ stubPDFOutlines(t, []deepdoctype.Outline{{Title: "Appendix", Level: 1, PageNumber: 2}})
+ p := &PDFParser{ParseMethod: "plain_text"}
+ result := p.ParseWithResult(t.Context(), "bookmarked.pdf", bookmarkedPDF(t))
+ if result.Err != nil {
+ t.Fatalf("ParseWithResult: %v", result.Err)
+ }
+ outline, ok := result.File["outline"].([]map[string]any)
+ if !ok || len(outline) != 1 || outline[0]["title"] != "Appendix" {
+ t.Fatalf("File[outline] = %#v, want the stubbed bookmark", result.File["outline"])
+ }
+}
+
+// bookmarkedPDF builds a two-page PDF whose outline is Chapter 1 (page 1) with
+// a Section 1.1 child (page 2), then Chapter 2 (page 2). It is hand-written so
+// the native extraction test has a deterministic fixture with no binary asset.
+func bookmarkedPDF(t *testing.T) []byte {
+ t.Helper()
+ page := func(contentsRef int) string {
+ return fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 9 0 R >> >> /Contents %d 0 R >>", contentsRef)
+ }
+ body := func(text string) string {
+ content := fmt.Sprintf("BT /F1 24 Tf 72 700 Td (%s) Tj ET", text)
+ return fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(content), content)
+ }
+ objects := []string{
+ "<< /Type /Catalog /Pages 2 0 R /Outlines 3 0 R >>",
+ "<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>",
+ "<< /Type /Outlines /First 4 0 R /Last 6 0 R /Count 2 >>",
+ "<< /Title (Chapter 1) /Parent 3 0 R /First 8 0 R /Last 8 0 R /Count 1 /Next 6 0 R /Dest [5 0 R /XYZ null null null] >>",
+ page(10),
+ "<< /Title (Chapter 2) /Parent 3 0 R /Prev 4 0 R /Dest [7 0 R /XYZ null null null] >>",
+ page(11),
+ "<< /Title (Section 1.1) /Parent 4 0 R /Dest [7 0 R /XYZ null null null] >>",
+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
+ body("page one"),
+ body("page two"),
+ }
+ var buf bytes.Buffer
+ buf.WriteString("%PDF-1.4\n")
+ offsets := make([]int, len(objects))
+ for i, obj := range objects {
+ offsets[i] = buf.Len()
+ fmt.Fprintf(&buf, "%d 0 obj\n%s\nendobj\n", i+1, obj)
+ }
+ xref := buf.Len()
+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1)
+ for _, off := range offsets {
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)
+ }
+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref)
+ return buf.Bytes()
+}
diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.go
index 773ed4f38b..68af21b194 100644
--- a/internal/parser/parser/pdf_parser_common.go
+++ b/internal/parser/parser/pdf_parser_common.go
@@ -364,7 +364,7 @@ func (p *PDFParser) validateParseMethod() error {
func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {
result := p.parseWithResult(ctx, filename, data)
if result.Err == nil {
- return p.NormalizeResult(ctx, filename, result)
+ return AttachPDFOutlines(p.NormalizeResult(ctx, filename, result), data)
}
return result
}