RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): read PDF bookmarks into remote parse results (#21499)
CCLUCKY0c8af782cbe8
diff --git a/internal/deepdoc/parser/pdf/pdfium/pdfium.go b/internal/deepdoc/parser/pdf/pdfium/pdfium.goindex 3f899adbe8..c482fa16ca 100644--- a/internal/deepdoc/parser/pdf/pdfium/pdfium.go+++ b/internal/deepdoc/parser/pdf/pdfium/pdfium.go@@ -56,15 +56,9 @@ import ("unsafe"pdfsync "ragflow/internal/deepdoc/parser/pdf/pdfsync"+ doctype "ragflow/internal/deepdoc/parser/type")-// Outline represents one entry in a PDF document outline (table of contents).-type Outline struct {- Title string- Level int- PageNumber int // 1-indexed, matching Python-}-var initOnce sync.Once// Init initializes the PDFium library. Safe to call multiple times.@@ -248,11 +242,11 @@ func openPage(pdfData []byte, pageIdx int) (// Traversal is iterative with an explicit stack to avoid stack overflow on deep// outline trees. pdfium is not thread-safe; all FPDF_* calls are serialized by// the shared pdfsync.Mu.-func ExtractOutlines(pdfData []byte) []Outline {+func ExtractOutlines(pdfData []byte) []doctype.Outline {if len(pdfData) == 0 {return nil}- var out []Outline+ var out []doctype.Outlinepdfsync.With(func() {Init()@@ -270,7 +264,7 @@ func ExtractOutlines(pdfData []byte) []Outline {level int}- var result []Outline+ var result []doctype.Outlinestack := []frame{{bm: C.FPDFBookmark_GetFirstChild(doc, nil), level: 0}}for len(stack) > 0 {@@ -292,7 +286,7 @@ func ExtractOutlines(pdfData []byte) []Outline {}}- result = append(result, Outline{Title: title, Level: top.level, PageNumber: pageNum})+ result = append(result, doctype.Outline{Title: title, Level: top.level, PageNumber: pageNum})// Push siblings after children so children are processed first (pre-order).if sibling := C.FPDFBookmark_GetNextSibling(doc, top.bm); sibling != nil {diff --git a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go b/internal/deepdoc/parser/pdf/pdfoxide_bridge.goindex 799aad4a8e..23c4c5680a 100644--- a/internal/deepdoc/parser/pdf/pdfoxide_bridge.go+++ b/internal/deepdoc/parser/pdf/pdfoxide_bridge.go@@ -29,12 +29,7 @@ func (e *PDFOxideEngine) PageCount() (int, error) { return e.Inner.PageCount() }func (e *PDFOxideEngine) Close() error { return e.Inner.Close() }func (e *PDFOxideEngine) Outlines() ([]pdf.Outline, error) {- ol := pdfium.ExtractOutlines(e.Inner.RawData())- result := make([]pdf.Outline, len(ol))- for i, o := range ol {- result[i] = pdf.Outline{Title: o.Title, Level: o.Level, PageNumber: o.PageNumber}- }- return result, nil+ return pdfium.ExtractOutlines(e.Inner.RawData()), nil}func (e *PDFOxideEngine) RenderPage(pageNum int, dpi float64) ([]byte, error) {diff --git a/internal/ingestion/component/pdf_outline_cgo_test.go b/internal/ingestion/component/pdf_outline_cgo_test.gonew file mode 100644index 0000000000..a8c8ec5fb5--- /dev/null+++ b/internal/ingestion/component/pdf_outline_cgo_test.go@@ -0,0 +1,80 @@+//go:build cgo++package component++import (+ "archive/zip"+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "testing"++ "gorm.io/gorm"++ modelModule "ragflow/internal/entity/models"+ "ragflow/internal/ingestion/component/schema"+ "ragflow/internal/utility"+)++// TestPDFDispatchAttachesPDFOutlines pins the outline attachment on the+// ingestion dispatch boundary: the tenant parser serves content only, so the+// bookmarks must come from the source bytes the component already holds.+// Reverting the call in maybeDispatchPDFVision's normalize closure fails here+// and nowhere else, because the built-in parse methods have their own boundary.+func TestPDFDispatchAttachesPDFOutlines(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var buf bytes.Buffer+ archive := zip.NewWriter(&buf)+ file, _ := archive.Create("sample/sample.json")+ _ = json.NewEncoder(file).Encode(map[string]any{"layouts": []any{map[string]any{"label": "Text", "content": "body"}}})+ _ = archive.Close()+ _, _ = w.Write(buf.Bytes())+ }))+ defer server.Close()+ previousResolver := resolveTenantOCRModelByProvider+ t.Cleanup(func() { resolveTenantOCRModelByProvider = previousResolver })+ resolveTenantOCRModelByProvider = func(context.Context, *gorm.DB, string, string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return &monkeyOCRv2FakeDriver{}, "monkey", &modelModule.APIConfig{}, 0, nil+ }+ setup := schema.ParserSetup{"parse_method": "monkeyocrv2", "monkeyocrv2_server_url": server.URL}++ result, handled, err := maybeDispatchPDFVision(t.Context(), nil, utility.FileTypePDF, "bookmarked.pdf", singleBookmarkPDF(t), nil, map[string]schema.ParserSetup{"pdf": setup})+ if err != nil || !handled || !result.Produced() {+ t.Fatalf("dispatch failed: handled=%v err=%v", handled, err)+ }+ outline, ok := result.File["outline"].([]map[string]any)+ if !ok || len(outline) != 1 || outline[0]["title"] != "Chapter 1" {+ t.Fatalf("File[outline] = %#v, want the bookmark read from the source PDF", result.File["outline"])+ }+}++// singleBookmarkPDF builds the smallest PDF that carries one bookmark, so the+// dispatch boundary has real bytes to extract from.+func singleBookmarkPDF(t *testing.T) []byte {+ t.Helper()+ objects := []string{+ "<< /Type /Catalog /Pages 2 0 R /Outlines 3 0 R >>",+ "<< /Type /Pages /Kids [4 0 R] /Count 1 >>",+ "<< /Type /Outlines /First 5 0 R /Last 5 0 R /Count 1 >>",+ "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 6 0 R >>",+ "<< /Title (Chapter 1) /Parent 3 0 R /Dest [4 0 R /XYZ null null null] >>",+ "<< /Length 0 >>\nstream\n\nendstream",+ }+ var buf bytes.Buffer+ buf.WriteString("%PDF-1.4\n")+ offsets := make([]int, len(objects))+ for i, obj := range objects {+ offsets[i] = buf.Len()+ fmt.Fprintf(&buf, "%d 0 obj\n%s\nendobj\n", i+1, obj)+ }+ xref := buf.Len()+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1)+ for _, off := range offsets {+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)+ }+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref)+ return buf.Bytes()+}diff --git a/internal/ingestion/component/pdf_vision_dispatch.go b/internal/ingestion/component/pdf_vision_dispatch.goindex a9664970cf..b1af9f8749 100644--- a/internal/ingestion/component/pdf_vision_dispatch.go+++ b/internal/ingestion/component/pdf_vision_dispatch.go@@ -91,7 +91,7 @@ func maybeDispatchPDFVision(flatten, _ := setup["flatten_media_to_text"].(bool)normalize := func(result parser.ParseResult) parser.ParseResult {pdf := parser.PDFParser{FlattenMediaToText: flatten}- return pdf.NormalizeResult(ctx, filename, result)+ return parser.AttachPDFOutlines(pdf.NormalizeResult(ctx, filename, result), binary)}if result, handled, err := maybeDispatchTenantPDFParser(ctx, db, filename, binary, tenantID, setup); handled {if err != nil {diff --git a/internal/parser/parser/pdf_outline.go b/internal/parser/parser/pdf_outline.gonew file mode 100644index 0000000000..10dabc6f15--- /dev/null+++ b/internal/parser/parser/pdf_outline.go@@ -0,0 +1,26 @@+package parser++// pdfOutlinesFromPDF reads a PDF's bookmark tree. The native engine backs it+// under cgo; without cgo it reports no bookmarks.+var pdfOutlinesFromPDF = pdfOutlinesFromData++// AttachPDFOutlines populates File["outline"] from the source PDF's bookmarks,+// mutating result.File in place. Remote engines return document content only,+// so their results carry an empty outline and the title chunker, which derives+// heading levels from bookmarks, has nothing to work with. A result that+// already carries an outline keeps it, so the engine that read the bookmarks+// stays authoritative. Both boundaries that hand out a PDF parse result — the+// built-in parse methods and the ingestion dispatch — hold the source bytes and+// call this themselves.+func AttachPDFOutlines(result ParseResult, data []byte) ParseResult {+ if result.Err != nil || result.File == nil {+ return result+ }+ if existing, ok := result.File["outline"].([]map[string]any); ok && len(existing) > 0 {+ return result+ }+ if outlines := pdfOutlinesFromPDF(data); len(outlines) > 0 {+ result.File["outline"] = outlinesToFileMeta(outlines)+ }+ return result+}diff --git a/internal/parser/parser/pdf_outline_cgo.go b/internal/parser/parser/pdf_outline_cgo.gonew file mode 100644index 0000000000..fa84190813--- /dev/null+++ b/internal/parser/parser/pdf_outline_cgo.go@@ -0,0 +1,12 @@+//go:build cgo++package parser++import (+ "ragflow/internal/deepdoc/parser/pdf/pdfium"+ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++func pdfOutlinesFromData(data []byte) []deepdoctype.Outline {+ return pdfium.ExtractOutlines(data)+}diff --git a/internal/parser/parser/pdf_outline_cgo_test.go b/internal/parser/parser/pdf_outline_cgo_test.gonew file mode 100644index 0000000000..2a244c3e0f--- /dev/null+++ b/internal/parser/parser/pdf_outline_cgo_test.go@@ -0,0 +1,40 @@+//go:build cgo++package parser++import "testing"++// TestPDFOutlineNativeExtractionFeedsRemoteResult is the native-path proof for+// #21221: a bookmarked PDF parsed by an engine that reads no bookmarks itself+// (plain_text) must come back with the outline extracted from the source file.+func TestPDFOutlineNativeExtractionFeedsRemoteResult(t *testing.T) {+ data := bookmarkedPDF(t)++ p := &PDFParser{ParseMethod: "plain_text"}+ res := p.ParseWithResult(t.Context(), "bookmarked.pdf", data)+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ outline, ok := res.File["outline"].([]map[string]any)+ if !ok {+ t.Fatalf("File[outline] type = %T, want []map[string]any", res.File["outline"])+ }+ want := []struct {+ title string+ level int+ page int+ }{+ {"Chapter 1", 0, 1},+ {"Section 1.1", 1, 2},+ {"Chapter 2", 0, 2},+ }+ if len(outline) != len(want) {+ t.Fatalf("outline = %#v, want %d entries", outline, len(want))+ }+ for i, w := range want {+ got := outline[i]+ if got["title"] != w.title || got["level"] != w.level || got["page_number"] != w.page {+ t.Fatalf("outline[%d] = %#v, want title=%q level=%d page=%d", i, got, w.title, w.level, w.page)+ }+ }+}diff --git a/internal/parser/parser/pdf_outline_nocgo.go b/internal/parser/parser/pdf_outline_nocgo.gonew file mode 100644index 0000000000..28665dbd5a--- /dev/null+++ b/internal/parser/parser/pdf_outline_nocgo.go@@ -0,0 +1,9 @@+//go:build !cgo++package parser++import (+ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++func pdfOutlinesFromData([]byte) []deepdoctype.Outline { return nil }diff --git a/internal/parser/parser/pdf_outline_test.go b/internal/parser/parser/pdf_outline_test.gonew file mode 100644index 0000000000..de2318781f--- /dev/null+++ b/internal/parser/parser/pdf_outline_test.go@@ -0,0 +1,139 @@+package parser++import (+ "bytes"+ "errors"+ "fmt"+ "testing"++ deepdoctype "ragflow/internal/deepdoc/parser/type"+)++// stubPDFOutlines replaces the bookmark reader for one test and reports how+// many times it was consulted. The seam is package-global, so tests using it+// must not run in parallel.+func stubPDFOutlines(t *testing.T, outlines []deepdoctype.Outline) *int {+ t.Helper()+ calls := 0+ previous := pdfOutlinesFromPDF+ pdfOutlinesFromPDF = func([]byte) []deepdoctype.Outline {+ calls+++ return outlines+ }+ t.Cleanup(func() { pdfOutlinesFromPDF = previous })+ return &calls+}++// remotePDFResult is the file meta a remote engine produces: pdfFileMeta seeds+// an empty outline.+func remotePDFResult(filename string) ParseResult {+ return pdfItemsToResult(filename, []map[string]any{{"text": "body", "page_number": 1}}, 3)+}++// TestAttachPDFOutlinesFillsRemoteOutline pins the #21221 fix for the remote+// PDF engines: their result carries an empty outline, so the bookmarks read+// from the source PDF are what the title chunker uses for heading levels.+func TestAttachPDFOutlinesFillsRemoteOutline(t *testing.T) {+ stubPDFOutlines(t, []deepdoctype.Outline{{Title: "1 Introduction", Level: 0, PageNumber: 2}})+ result := AttachPDFOutlines(remotePDFResult("remote.pdf"), []byte("%PDF-1.4 fake"))+ outline, ok := result.File["outline"].([]map[string]any)+ if !ok || len(outline) != 1 {+ t.Fatalf("File[outline] = %#v, want one entry", result.File["outline"])+ }+ if outline[0]["title"] != "1 Introduction" || outline[0]["level"] != 0 || outline[0]["page_number"] != 2 {+ t.Fatalf("outline entry = %#v", outline[0])+ }+}++// TestAttachPDFOutlinesKeepsEngineOutline asserts the engine that already read+// the bookmarks stays authoritative and is not re-read.+func TestAttachPDFOutlinesKeepsEngineOutline(t *testing.T) {+ calls := stubPDFOutlines(t, []deepdoctype.Outline{{Title: "Ignored", Level: 0, PageNumber: 1}})+ result := AttachPDFOutlines(ParseResult{+ File: map[string]any{"name": "deep.pdf", "outline": []map[string]any{{"title": "Chapter 1", "level": 0, "page_number": 3}}},+ }, []byte("%PDF-1.4 fake"))+ if got := result.File["outline"].([]map[string]any); len(got) != 1 || got[0]["title"] != "Chapter 1" {+ t.Fatalf("engine outline changed: %#v", result.File["outline"])+ }+ if *calls != 0 {+ t.Fatalf("bookmark reader consulted %d times, want 0", *calls)+ }+}++// TestAttachPDFOutlinesKeepsEmptyOutlineWhenAbsent covers a PDF without+// bookmarks: the key stays an empty slice, never a nil the consumers would have+// to special-case. An errored or file-less result is untouched.+func TestAttachPDFOutlinesKeepsEmptyOutlineWhenAbsent(t *testing.T) {+ stubPDFOutlines(t, nil)+ result := AttachPDFOutlines(remotePDFResult("plain.pdf"), []byte("%PDF-1.4 fake"))+ outline, ok := result.File["outline"].([]map[string]any)+ if !ok || len(outline) != 0 {+ t.Fatalf("File[outline] = %#v (ok=%v), want empty []map[string]any", result.File["outline"], ok)+ }++ if failed := AttachPDFOutlines(ParseResult{Err: errors.New("parse failed")}, []byte("%PDF")); failed.File != nil {+ t.Fatalf("errored result gained a file meta: %#v", failed.File)+ }+ if got := AttachPDFOutlines(ParseResult{}, []byte("%PDF")); got.File != nil {+ t.Fatalf("result without file meta gained one: %#v", got.File)+ }+}++// TestParseWithResultAttachesOutline pins the attachment on the built-in parse+// method boundary (ParseWithResult). The dispatched tenant and vision engines+// normalize outside this package and have their own call site, pinned by+// TestPDFDispatchAttachesPDFOutlines in the component package. The bookmark+// reader is stubbed here, so only a structurally valid PDF is needed as input.+func TestParseWithResultAttachesOutline(t *testing.T) {+ stubPDFOutlines(t, []deepdoctype.Outline{{Title: "Appendix", Level: 1, PageNumber: 2}})+ p := &PDFParser{ParseMethod: "plain_text"}+ result := p.ParseWithResult(t.Context(), "bookmarked.pdf", bookmarkedPDF(t))+ if result.Err != nil {+ t.Fatalf("ParseWithResult: %v", result.Err)+ }+ outline, ok := result.File["outline"].([]map[string]any)+ if !ok || len(outline) != 1 || outline[0]["title"] != "Appendix" {+ t.Fatalf("File[outline] = %#v, want the stubbed bookmark", result.File["outline"])+ }+}++// bookmarkedPDF builds a two-page PDF whose outline is Chapter 1 (page 1) with+// a Section 1.1 child (page 2), then Chapter 2 (page 2). It is hand-written so+// the native extraction test has a deterministic fixture with no binary asset.+func bookmarkedPDF(t *testing.T) []byte {+ t.Helper()+ page := func(contentsRef int) string {+ return fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 9 0 R >> >> /Contents %d 0 R >>", contentsRef)+ }+ body := func(text string) string {+ content := fmt.Sprintf("BT /F1 24 Tf 72 700 Td (%s) Tj ET", text)+ return fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(content), content)+ }+ objects := []string{+ "<< /Type /Catalog /Pages 2 0 R /Outlines 3 0 R >>",+ "<< /Type /Pages /Kids [5 0 R 7 0 R] /Count 2 >>",+ "<< /Type /Outlines /First 4 0 R /Last 6 0 R /Count 2 >>",+ "<< /Title (Chapter 1) /Parent 3 0 R /First 8 0 R /Last 8 0 R /Count 1 /Next 6 0 R /Dest [5 0 R /XYZ null null null] >>",+ page(10),+ "<< /Title (Chapter 2) /Parent 3 0 R /Prev 4 0 R /Dest [7 0 R /XYZ null null null] >>",+ page(11),+ "<< /Title (Section 1.1) /Parent 4 0 R /Dest [7 0 R /XYZ null null null] >>",+ "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",+ body("page one"),+ body("page two"),+ }+ var buf bytes.Buffer+ buf.WriteString("%PDF-1.4\n")+ offsets := make([]int, len(objects))+ for i, obj := range objects {+ offsets[i] = buf.Len()+ fmt.Fprintf(&buf, "%d 0 obj\n%s\nendobj\n", i+1, obj)+ }+ xref := buf.Len()+ fmt.Fprintf(&buf, "xref\n0 %d\n0000000000 65535 f \n", len(objects)+1)+ for _, off := range offsets {+ fmt.Fprintf(&buf, "%010d 00000 n \n", off)+ }+ fmt.Fprintf(&buf, "trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(objects)+1, xref)+ return buf.Bytes()+}diff --git a/internal/parser/parser/pdf_parser_common.go b/internal/parser/parser/pdf_parser_common.goindex 773ed4f38b..68af21b194 100644--- a/internal/parser/parser/pdf_parser_common.go+++ b/internal/parser/parser/pdf_parser_common.go@@ -364,7 +364,7 @@ func (p *PDFParser) validateParseMethod() error {func (p *PDFParser) ParseWithResult(ctx context.Context, filename string, data []byte) ParseResult {result := p.parseWithResult(ctx, filename, data)if result.Err == nil {- return p.NormalizeResult(ctx, filename, result)+ return AttachPDFOutlines(p.NormalizeResult(ctx, filename, result), data)}return result}