Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): emit Docling page positions and real page_count (#21330)

Taranum Wasuacd6d887a548
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.go
index 587bcc97b5..ee0590576d 100644
--- a/internal/ingestion/component/parser_dispatch_test.go
+++ b/internal/ingestion/component/parser_dispatch_test.go
@@ -1818,6 +1818,10 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
return
}
switch {
+ // The parser tries standard conversions first (they carry page
+ // provenance) and falls back to chunked endpoints only when the
+ // server rejects the standard shape, so the third request is the
+ // chunked /v1 call and answers with a chunk list.
case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 1:
w.WriteHeader(http.StatusUnprocessableEntity)
_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
@@ -1826,7 +1830,7 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {
_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 3:
w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))
+ _, _ = w.Write([]byte(`[{"text":"# Docling Title\n\nDocling body.\n"}]`))
default:
http.NotFound(w, r)
}
diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.go
index ab9d414111..cfe2ce5d24 100644
--- a/internal/parser/parser/pdf_parser_docling.go
+++ b/internal/parser/parser/pdf_parser_docling.go
@@ -1,6 +1,7 @@
package parser
import (
+ "bytes"
"context"
"encoding/base64"
"encoding/json"
@@ -8,9 +9,13 @@ import (
"io"
"maps"
"ragflow/internal/common"
+ "regexp"
"strconv"
"strings"
+ "github.com/ledongthuc/pdf"
+ "go.uber.org/zap"
+
models "ragflow/internal/entity/models"
)
@@ -66,28 +71,34 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, pars
body func() map[string]any
chunked bool
}{
+ // Standard responses carry per-page provenance (json_content.prov), so
+ // they are tried first: positions are the point of this path. Chunked
+ // responses have no provenance and stay as the fallback for servers or
+ // documents the standard conversion rejects.
{
endpoint: "/v1/convert/source",
- chunked: true,
- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false, convertOptions) },
+ chunked: false,
+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, false, convertOptions) },
},
{
endpoint: "/v1alpha/convert/source",
- chunked: true,
- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true, convertOptions) },
+ chunked: false,
+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, true, convertOptions) },
},
{
endpoint: "/v1/convert/source",
- chunked: false,
- body: func() map[string]any { return doclingStandardPayload(filename, encoded, false, convertOptions) },
+ chunked: true,
+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false, convertOptions) },
},
{
endpoint: "/v1alpha/convert/source",
- chunked: false,
- body: func() map[string]any { return doclingStandardPayload(filename, encoded, true, convertOptions) },
+ chunked: true,
+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true, convertOptions) },
},
}
+ sourcePages := doclingSourcePageCount(data)
+
var lastErr error
for _, candidate := range payloads {
url := baseURL + candidate.endpoint
@@ -113,13 +124,16 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, pars
continue
}
if candidate.chunked {
- if res, ok := parseDoclingChunkedResult(filename, body); ok {
+ if res, ok := parseDoclingChunkedResult(filename, body, sourcePages); ok {
+ common.Warn("Docling chunked response carries no page provenance; positions unavailable",
+ zap.String("filename", filename))
+ res.Warnings = append(res.Warnings, "Docling chunked responses carry no page provenance; positions unavailable")
return res
}
lastErr = fmt.Errorf("%s: chunked response contained no usable text", candidate.endpoint)
continue
}
- if res, ok := parseDoclingStandardResult(ctx, filename, body); ok {
+ if res, ok := parseDoclingStandardResult(ctx, filename, body, sourcePages); ok {
return res
}
lastErr = fmt.Errorf("%s: standard response contained no parsed document", candidate.endpoint)
@@ -184,7 +198,7 @@ func doclingChunkedPayload(filename string, encoded string, alpha bool, convertO
return payload
}
-func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool) {
+func parseDoclingChunkedResult(filename string, body []byte, sourcePages int) (ParseResult, bool) {
var chunks []doclingChunk
if err := json.Unmarshal(body, &chunks); err != nil {
var wrapped struct {
@@ -208,14 +222,16 @@ func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool)
if len(texts) == 0 {
return ParseResult{}, false
}
- pageCount := 0
- if len(texts) > 0 {
- pageCount = len(texts)
+ // A chunked response carries no page metadata: report the source
+ // PDF's real page count when known, not the number of text chunks.
+ pageCount := len(texts)
+ if sourcePages > 0 {
+ pageCount = sourcePages
}
return doclingTextsToResult(filename, texts, pageCount), true
}
-func parseDoclingStandardResult(ctx context.Context, filename string, body []byte) (ParseResult, bool) {
+func parseDoclingStandardResult(ctx context.Context, filename string, body []byte, sourcePages int) (ParseResult, bool) {
var payload doclingResponse
if err := json.Unmarshal(body, &payload); err != nil {
return ParseResult{}, false
@@ -233,20 +249,256 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []byt
docs = append(docs, *result.Result)
}
}
+ pageCount := doclingPageCount(docs, sourcePages)
for _, doc := range docs {
if md := strings.TrimSpace(doc.MDContent); md != "" {
- return pdfMarkdownToResult(ctx, filename, md, len(docs)), true
+ return doclingAttachPositions(pdfMarkdownToResult(ctx, filename, md, pageCount), doc.JSONContent), true
}
if txt := strings.TrimSpace(doc.TextContent); txt != "" {
- return doclingTextsToResult(filename, []string{txt}, len(docs)), true
+ return doclingAttachPositions(doclingTextsToResult(filename, []string{txt}, pageCount), doc.JSONContent), true
}
if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {
- return pdfMarkdownToResult(ctx, filename, md, len(docs)), true
+ return doclingAttachPositions(pdfMarkdownToResult(ctx, filename, md, pageCount), doc.JSONContent), true
}
}
return ParseResult{}, false
}
+// doclingPageCount returns the number of PDF pages described by the Docling
+// documents: the size of json_content.pages when present, otherwise the
+// highest page_no found in any provenance entry. A single source converts to
+// one logical document, so payload shapes that repeat it (document,
+// documents, results) are duplicates: the maximum per-document count wins,
+// never the sum. It falls back to the number of documents only when the
+// response carries no page information.
+func doclingPageCount(docs []doclingDocument, sourcePages int) int {
+ total := 0
+ for _, doc := range docs {
+ count := 0
+ switch pages := doc.JSONContent["pages"].(type) {
+ case map[string]any:
+ count = len(pages)
+ case []any:
+ count = len(pages)
+ }
+ for _, entry := range doclingTextEntries(doc.JSONContent) {
+ for _, prov := range doclingProvs(entry) {
+ if page, ok := numericAny(prov["page_no"]); ok && int(page) > count {
+ count = int(page)
+ }
+ }
+ }
+ if count > total {
+ total = count
+ }
+ }
+ // Provenance can end before the final page when the backend extracts a
+ // partial document, so the source PDF page count is a lower bound.
+ if sourcePages > total {
+ return sourcePages
+ }
+ if total == 0 {
+ // No page metadata at all: report the number of returned documents.
+ return len(docs)
+ }
+ return total
+}
+
+// doclingMaxSourcePageCount bounds a catalog /Count read from the source PDF;
+// a corrupt or malicious value far beyond any real document is ignored.
+const doclingMaxSourcePageCount = 1_000_000
+
+// doclingSourcePageCount counts the pages of the source PDF so a Docling
+// response without page metadata still reports the real page count. It
+// returns 0 when the bytes cannot be opened or the count is implausible.
+func doclingSourcePageCount(data []byte) int {
+ if len(data) == 0 {
+ return 0
+ }
+ reader, err := pdf.NewReader(bytes.NewReader(data), int64(len(data)))
+ if err != nil {
+ return 0
+ }
+ if pages := reader.NumPage(); pages > 0 && pages <= doclingMaxSourcePageCount {
+ return pages
+ }
+ return 0
+}
+
+func doclingTextEntries(content map[string]any) []map[string]any {
+ var entries []map[string]any
+ texts, _ := content["texts"].([]any)
+ for _, raw := range texts {
+ if entry, ok := raw.(map[string]any); ok {
+ entries = append(entries, entry)
+ }
+ }
+ return entries
+}
+
+func doclingProvs(entry map[string]any) []map[string]any {
+ var provs []map[string]any
+ list, _ := entry["prov"].([]any)
+ for _, raw := range list {
+ if prov, ok := raw.(map[string]any); ok {
+ provs = append(provs, prov)
+ }
+ }
+ return provs
+}
+
+// doclingProvPosition converts a Docling provenance entry (1-based page_no and
+// bbox {l, t, r, b}) into a [page, left, right, top, bottom] row with top
+// above bottom. Docling permits bbox coord_origin BOTTOMLEFT (y measured from
+// the page bottom); those boxes are converted to the top-left crop origin
+// when the page height is known, otherwise the sorted values are kept.
+func doclingProvPosition(prov map[string]any, heights map[int]float64) []any {
+ page, ok := numericAny(prov["page_no"])
+ if !ok || page < 1 {
+ return nil
+ }
+ bbox, _ := prov["bbox"].(map[string]any)
+ var v [4]float64
+ for i, key := range []string{"l", "t", "r", "b"} {
+ n, ok := numericAny(bbox[key])
+ if !ok {
+ return nil
+ }
+ v[i] = n
+ }
+ left, top, right, bottom := v[0], v[1], v[2], v[3]
+ if origin, _ := bbox["coord_origin"].(string); strings.EqualFold(origin, "BOTTOMLEFT") {
+ h := heights[int(page)]
+ if h <= 0 {
+ // Without the page height the bottom-origin Y cannot be converted
+ // to the top-origin contract downstream; a missing position is
+ // safer than a silently wrong crop.
+ return nil
+ }
+ top, bottom = h-top, h-bottom
+ }
+ if left > right {
+ left, right = right, left
+ }
+ if top > bottom {
+ top, bottom = bottom, top
+ }
+ return []any{int(page), left, right, top, bottom}
+}
+
+// doclingPageHeights maps 1-based page numbers to page heights from the
+// json_content pages metadata, used to convert BOTTOMLEFT bounding boxes.
+func doclingPageHeights(content map[string]any) map[int]float64 {
+ heights := map[int]float64{}
+ collect := func(page map[string]any, keyNo int) {
+ no, ok := numericAny(page["page_no"])
+ if !ok && keyNo > 0 {
+ no, ok = float64(keyNo), true
+ }
+ if !ok || no < 1 {
+ return
+ }
+ size, _ := page["size"].(map[string]any)
+ if h, ok := numericAny(size["height"]); ok && h > 0 {
+ heights[int(no)] = h
+ }
+ }
+ switch pages := content["pages"].(type) {
+ case map[string]any:
+ for key, raw := range pages {
+ page, ok := raw.(map[string]any)
+ if !ok {
+ continue
+ }
+ keyNo, _ := strconv.Atoi(key)
+ collect(page, keyNo)
+ }
+ case []any:
+ for _, raw := range pages {
+ if page, ok := raw.(map[string]any); ok {
+ collect(page, 0)
+ }
+ }
+ }
+ return heights
+}
+
+func doclingNormalizeText(text string) string {
+ return strings.Join(strings.Fields(text), " ")
+}
+
+var doclingMarkdownPrefix = regexp.MustCompile(`^(#{1,6}\s+|[-*]\s+|\d+\.\s+)`)
+
+// doclingStripMarkdownPrefix removes one recognized Markdown heading or list
+// marker. Leading digits elsewhere are content ("2026 Outlook") and stay.
+func doclingStripMarkdownPrefix(text string) (string, bool) {
+ stripped := doclingMarkdownPrefix.ReplaceAllString(text, "")
+ return stripped, stripped != text
+}
+
+// doclingAttachPositions copies page positions from Docling's structured
+// json_content onto JSON items whose text matches a Docling text element. Items
+// without a match are left unchanged.
+func doclingAttachPositions(res ParseResult, content map[string]any) ParseResult {
+ if res.Err != nil || len(res.JSON) == 0 || len(content) == 0 {
+ return res
+ }
+ // Group positions per Docling text entry so repeated text keeps its own
+ // provenance: the i-th parsed occurrence takes the i-th entry's
+ // positions instead of every position of every identical entry.
+ byText := map[string][][][]any{}
+ heights := doclingPageHeights(content)
+ for _, entry := range doclingTextEntries(content) {
+ text := doclingNormalizeText(stringValue(entry["text"]))
+ if text == "" {
+ continue
+ }
+ var positions [][]any
+ for _, prov := range doclingProvs(entry) {
+ if pos := doclingProvPosition(prov, heights); pos != nil {
+ positions = append(positions, pos)
+ }
+ }
+ if len(positions) > 0 {
+ byText[text] = append(byText[text], positions)
+ }
+ }
+ if len(byText) == 0 {
+ return res
+ }
+ itemMaps := make([]map[string]any, len(res.JSON))
+ matched := false
+ used := map[string]int{}
+ for i, item := range res.JSON {
+ m := item.ToMap()
+ text := doclingNormalizeText(stringValue(m["text"]))
+ matches, ok := byText[text]
+ if !ok {
+ // List and heading markers are not content, but leading digits
+ // can be; only retry without a recognized Markdown prefix.
+ if stripped, changed := doclingStripMarkdownPrefix(text); changed {
+ if matches, ok = byText[stripped]; ok {
+ text = stripped
+ }
+ }
+ }
+ if ok && text != "" {
+ if idx := used[text]; idx < len(matches) {
+ m["positions"] = matches[idx]
+ m["_pdf_positions"] = matches[idx]
+ used[text]++
+ matched = true
+ }
+ }
+ itemMaps[i] = m
+ }
+ if !matched {
+ return res
+ }
+ res.JSON = MapsToItems(itemMaps)
+ return res
+}
+
func doclingTextsToResult(filename string, texts []string, pageCount int) ParseResult {
items := make([]map[string]any, 0, len(texts))
for _, text := range texts {
diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.go
index dac14b7b93..6f153f5934 100644
--- a/internal/parser/parser/pdf_parser_docling_test.go
+++ b/internal/parser/parser/pdf_parser_docling_test.go
@@ -1,8 +1,10 @@
package parser
import (
+ "bytes"
"encoding/base64"
"encoding/json"
+ "fmt"
"maps"
"net/http"
"net/http/httptest"
@@ -31,6 +33,12 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {
return
}
options, _ := body["options"].(map[string]any)
+ if _, isChunked := options["do_chunking"]; !isChunked {
+ // Standard payloads are tried first; this server only supports chunking.
+ w.WriteHeader(http.StatusUnprocessableEntity)
+ _, _ = w.Write([]byte(`{"detail":"standard conversion unsupported"}`))
+ return
+ }
if got, want := options["do_chunking"], true; got != want {
t.Errorf("do_chunking = %#v, want %#v", got, want)
return
@@ -55,8 +63,8 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {
t.Errorf("uploaded file = %q, want PDF bytes", got)
return
}
- if current != 1 {
- t.Errorf("request count = %d, want first chunked request only", current)
+ if current != 3 {
+ t.Errorf("chunked success on request %d, want 3 (after both standard endpoints 422)", current)
return
}
w.Header().Set("Content-Type", "application/json")
@@ -88,49 +96,44 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {
if !strings.Contains(got.String(), "Chunk A") || !strings.Contains(got.String(), "Chunk B") {
t.Fatalf("JSON text = %q, want both chunks", got.String())
}
+ warned := false
+ for _, w := range res.Warnings {
+ if strings.Contains(w, "positions unavailable") {
+ warned = true
+ }
+ }
+ if !warned {
+ t.Fatalf("chunked fallback must warn that positions are unavailable: %#v", res.Warnings)
+ }
}
-func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *testing.T) {
+func TestPDFParser_ParseWithResult_DoclingStandardPreferredIntegration(t *testing.T) {
withSSRFBypass(t)
var requestCount atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
current := requestCount.Add(1)
- switch current {
- case 1:
- if r.URL.Path != "/v1/convert/source" {
- t.Errorf("request 1 path = %q, want /v1/convert/source", r.URL.Path)
- return
- }
- w.WriteHeader(http.StatusUnprocessableEntity)
- _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
- case 2:
- if r.URL.Path != "/v1alpha/convert/source" {
- t.Errorf("request 2 path = %q, want /v1alpha/convert/source", r.URL.Path)
- return
- }
- w.WriteHeader(http.StatusUnprocessableEntity)
- _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
- case 3:
- if r.URL.Path != "/v1/convert/source" {
- t.Errorf("request 3 path = %q, want /v1/convert/source", r.URL.Path)
- return
- }
- var body map[string]any
- if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
- t.Errorf("Decode: %v", err)
- return
- }
- options, _ := body["options"].(map[string]any)
- if _, exists := options["do_chunking"]; exists {
- t.Errorf("standard fallback payload unexpectedly contains do_chunking: %#v", options)
- return
- }
- w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))
- default:
+ if current != 1 {
+ t.Errorf("request %d: standard conversion must succeed on the first request", current)
http.NotFound(w, r)
+ return
+ }
+ if r.URL.Path != "/v1/convert/source" {
+ t.Errorf("request 1 path = %q, want /v1/convert/source", r.URL.Path)
+ return
+ }
+ var body map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Errorf("Decode: %v", err)
+ return
+ }
+ options, _ := body["options"].(map[string]any)
+ if _, exists := options["do_chunking"]; exists {
+ t.Errorf("standard payload unexpectedly contains do_chunking: %#v", options)
+ return
}
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))
}))
defer server.Close()
@@ -151,7 +154,7 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *t
if len(res.JSON) == 0 {
t.Fatal("JSON is empty; want markdown-normalized items")
}
- if got, want := requestCount.Load(), int32(3); got != want {
+ if got, want := requestCount.Load(), int32(1); got != want {
t.Fatalf("requestCount = %d, want %d", got, want)
}
}
@@ -227,13 +230,13 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {
t.Errorf("request %d: options[%q] = %#v, want %#v", current, key, got, want)
}
}
- // Reject both chunked requests so the standard payload is checked too.
+ // Reject both standard requests so the chunked payload is checked too.
if current < 3 {
w.WriteHeader(http.StatusUnprocessableEntity)
return
}
w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))
+ _, _ = w.Write([]byte(`[{"text":"Chunk A"}]`))
}))
defer server.Close()
@@ -255,3 +258,256 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {
})
}
}
+
+const doclingPositionsResponse = `{"document":{"md_content":"# Docling Title\n\nDocling body.\n\nUnmatched paragraph.\n","json_content":{
+ "pages":{"1":{"page_no":1,"size":{"width":612,"height":792}},"2":{"page_no":2,"size":{"width":612,"height":792}},"3":{"page_no":3,"size":{"width":612,"height":792}}},
+ "texts":[
+ {"label":"section_header","text":"Docling Title","prov":[{"page_no":1,"bbox":{"l":10,"t":700,"r":200,"b":680,"coord_origin":"BOTTOMLEFT"}}]},
+ {"label":"text","text":"Docling body.","prov":[{"page_no":2,"bbox":{"l":15,"t":100,"r":300,"b":140,"coord_origin":"TOPLEFT"}}]}
+ ]}}}`
+
+func TestPDFParser_ParseWithResult_DoclingEmitsPositionsAndPageCount(t *testing.T) {
+ withSSRFBypass(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var body map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&body)
+ options, _ := body["options"].(map[string]any)
+ if _, chunked := options["do_chunking"]; chunked {
+ w.WriteHeader(http.StatusUnprocessableEntity)
+ _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(doclingPositionsResponse))
+ }))
+ defer server.Close()
+
+ pdf := NewPDFParser()
+ pdf.ConfigureFromSetup(map[string]any{
+ "parse_method": "Docling",
+ "output_format": "json",
+ "docling_server_url": server.URL,
+ })
+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))
+ if res.Err != nil {
+ t.Fatalf("ParseWithResult: %v", res.Err)
+ }
+ if got := res.File["page_count"]; got != 3 {
+ t.Fatalf("page_count = %v, want 3 (pages, not documents)", got)
+ }
+ want := map[string][]any{
+ // BOTTOMLEFT y=700/680 on a 792pt page convert to top-origin 92/112.
+ "Docling Title": {1, 10.0, 200.0, 92.0, 112.0},
+ "Docling body.": {2, 15.0, 300.0, 100.0, 140.0},
+ }
+ found := map[string]bool{}
+ for _, item := range res.JSON {
+ m := item.ToMap()
+ text := strings.TrimSpace(strings.TrimLeft(stringValue(m["text"]), "# "))
+ rows, ok := ExtractPDFPositions(m)
+ if w, expected := want[text]; expected {
+ if !ok || len(rows) != 1 || len(rows[0]) != 5 {
+ t.Fatalf("%q positions = %#v ok=%v", text, rows, ok)
+ }
+ for i := range w {
+ got, _ := numericAny(rows[0][i])
+ exp, _ := numericAny(w[i])
+ if got != exp {
+ t.Fatalf("%q positions[%d] = %v, want %v (row %#v)", text, i, rows[0][i], w[i], rows[0])
+ }
+ }
+ found[text] = true
+ } else if ok {
+ t.Fatalf("unmatched item %q must not carry positions: %#v", text, rows)
+ }
+ }
+ if len(found) != 2 {
+ t.Fatalf("matched items = %v, want both", found)
+ }
+}
+
+func TestDoclingPageCountFallsBackToProvenanceThenDocuments(t *testing.T) {
+ withProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{
+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 4.0}}},
+ }}}}
+ if got := doclingPageCount(withProv, 0); got != 4 {
+ t.Fatalf("provenance page count = %d, want 4", got)
+ }
+ noPages := []doclingDocument{{MDContent: "x"}}
+ if got := doclingPageCount(noPages, 50); got != 50 {
+ t.Fatalf("metadata-less response must report source PDF pages = %d, want 50", got)
+ }
+ if got := doclingPageCount(noPages, 0); got != 1 {
+ t.Fatalf("unreadable source page count = %d, want 1 (document count)", got)
+ }
+}
+
+func TestDoclingPageCountMaxAcrossDuplicateDocuments(t *testing.T) {
+ // A single source converts to one logical document; payload shapes that
+ // repeat it (document + documents + results) must not multiply the page
+ // count.
+ pageful := doclingDocument{JSONContent: map[string]any{
+ "pages": map[string]any{"1": map[string]any{"page_no": 1.0}, "2": map[string]any{"page_no": 2.0}, "3": map[string]any{"page_no": 3.0}},
+ }}
+ if got := doclingPageCount([]doclingDocument{pageful, pageful}, 0); got != 3 {
+ t.Fatalf("duplicated documents must not sum pages: got %v, want 3", got)
+ }
+}
+
+func TestDoclingSourcePageCount(t *testing.T) {
+ if got := doclingSourcePageCount(buildDoclingPDFWithCount(t, 50)); got != 50 {
+ t.Fatalf("source page count = %d, want 50", got)
+ }
+ if got := doclingSourcePageCount([]byte("%PDF-1.4\nmock")); got != 0 {
+ t.Fatalf("unopenable PDF must yield 0, got %d", got)
+ }
+ if got := doclingSourcePageCount(buildDoclingPDFWithCount(t, 1<<30)); got != 0 {
+ t.Fatalf("implausible /Count must be ignored, got %d", got)
+ }
+}
+
+// buildDoclingPDFWithCount emits a minimal valid PDF whose /Pages /Count is
+// the supplied value, mirroring the deepdoc plaintext test fixture.
+func buildDoclingPDFWithCount(t *testing.T, count int) []byte {
+ t.Helper()
+ var buf bytes.Buffer
+ buf.WriteString("%PDF-1.4\n")
+ off1 := buf.Len()
+ fmt.Fprintf(&buf, "1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")
+ off2 := buf.Len()
+ fmt.Fprintf(&buf, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count %d >>\nendobj\n", count)
+ off3 := buf.Len()
+ fmt.Fprintf(&buf, "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>\nendobj\n")
+ xrefStart := buf.Len()
+ buf.WriteString("xref\n0 4\n")
+ buf.WriteString("0000000000 65535 f \n")
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off1)
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off2)
+ fmt.Fprintf(&buf, "%010d 00000 n \n", off3)
+ fmt.Fprintf(&buf, "trailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xrefStart)
+ return buf.Bytes()
+}
+
+func TestDoclingAttachPositionsMatchesOccurrencesInOrder(t *testing.T) {
+ content := map[string]any{"texts": []any{
+ map[string]any{"text": "Repeated paragraph.", "prov": []any{
+ map[string]any{"page_no": 2.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},
+ }},
+ map[string]any{"text": "Repeated paragraph.", "prov": []any{
+ map[string]any{"page_no": 5.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},
+ }},
+ }}
+ res := ParseResult{JSON: MapsToItems([]map[string]any{
+ {"text": "Repeated paragraph."},
+ {"text": "Repeated paragraph."},
+ })}
+ res = doclingAttachPositions(res, content)
+ first, _ := res.JSON[0].ToMap()["_pdf_positions"].([][]any)
+ second, _ := res.JSON[1].ToMap()["_pdf_positions"].([][]any)
+ if len(first) != 1 || first[0][0] != 2 {
+ t.Fatalf("first occurrence must take page 2: %#v", first)
+ }
+ if len(second) != 1 || second[0][0] != 5 {
+ t.Fatalf("second occurrence must take page 5: %#v", second)
+ }
+}
+
+func TestDoclingAttachPositionsKeepsLeadingDigits(t *testing.T) {
+ content := map[string]any{"texts": []any{
+ map[string]any{"text": "2026 Outlook", "prov": []any{
+ map[string]any{"page_no": 3.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},
+ }},
+ map[string]any{"text": "Introduction", "prov": []any{
+ map[string]any{"page_no": 4.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},
+ }},
+ }}
+ res := ParseResult{JSON: MapsToItems([]map[string]any{
+ {"text": "2026 Outlook"},
+ {"text": "1. Introduction"},
+ })}
+ res = doclingAttachPositions(res, content)
+ digits, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)
+ if !ok || len(digits) != 1 || digits[0][0] != 3 {
+ t.Fatalf("leading digits are content and must match: %#v", res.JSON[0].ToMap())
+ }
+ numbered, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)
+ if !ok || len(numbered) != 1 || numbered[0][0] != 4 {
+ t.Fatalf("numbered list prefix must strip for the match: %#v", res.JSON[1].ToMap())
+ }
+}
+
+func TestDoclingChunkedResultPrefersSourcePageCount(t *testing.T) {
+ body := []byte(`[{"text":"chunk one"},{"text":"chunk two"}]`)
+ res, ok := parseDoclingChunkedResult("a.pdf", body, 9)
+ if !ok {
+ t.Fatal("chunked parse failed")
+ }
+ if got := res.File["page_count"]; got != 9 {
+ t.Fatalf("chunked page_count = %v, want source PDF count 9", got)
+ }
+ res, ok = parseDoclingChunkedResult("a.pdf", body, 0)
+ if !ok {
+ t.Fatal("chunked parse failed")
+ }
+ if got := res.File["page_count"]; got != 2 {
+ t.Fatalf("without source count, page_count = %v, want chunk count 2", got)
+ }
+}
+
+func TestDoclingPageCountSourceLowerBound(t *testing.T) {
+ shortProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{
+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 3.0}}},
+ }}}}
+ if got := doclingPageCount(shortProv, 9); got != 9 {
+ t.Fatalf("provenance ending early must not underreport: got %d, want 9", got)
+ }
+ fullProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{
+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 12.0}}},
+ }}}}
+ if got := doclingPageCount(fullProv, 9); got != 12 {
+ t.Fatalf("metadata above source count still wins: got %d, want 12", got)
+ }
+}
+
+func TestDoclingProvPositionConvertsBottomLeftOrigin(t *testing.T) {
+ heights := map[int]float64{1: 200}
+ prov := map[string]any{"page_no": 1.0, "bbox": map[string]any{
+ "l": 10.0, "t": 180.0, "r": 50.0, "b": 170.0, "coord_origin": "BOTTOMLEFT",
+ }}
+ pos := doclingProvPosition(prov, heights)
+ if pos == nil {
+ t.Fatal("position dropped")
+ }
+ // BOTTOMLEFT y=180/170 from the bottom become 20/30 from the top.
+ if pos[3] != 20.0 || pos[4] != 30.0 {
+ t.Fatalf("BOTTOMLEFT not converted: %#v", pos)
+ }
+ // Unknown page height: the bottom-origin Y cannot meet the top-origin
+ // contract, so the position is dropped rather than emitted wrong.
+ if pos = doclingProvPosition(prov, map[int]float64{}); pos != nil {
+ t.Fatalf("missing height must drop the position, got %#v", pos)
+ }
+ // TOPLEFT boxes are untouched even with a known height.
+ provTL := map[string]any{"page_no": 1.0, "bbox": map[string]any{
+ "l": 10.0, "t": 20.0, "r": 50.0, "b": 30.0, "coord_origin": "TOPLEFT",
+ }}
+ pos = doclingProvPosition(provTL, heights)
+ if pos == nil || pos[3] != 20.0 || pos[4] != 30.0 {
+ t.Fatalf("TOPLEFT must pass through unchanged: %#v", pos)
+ }
+}
+
+func TestDoclingPageHeightsFromPagesMetadata(t *testing.T) {
+ asMap := map[string]any{"pages": map[string]any{
+ "1": map[string]any{"size": map[string]any{"width": 100.0, "height": 200.0}},
+ }}
+ if h := doclingPageHeights(asMap)[1]; h != 200.0 {
+ t.Fatalf("map-form pages height = %v, want 200", h)
+ }
+ asList := map[string]any{"pages": []any{
+ map[string]any{"page_no": 2.0, "size": map[string]any{"height": 300.0}},
+ }}
+ if h := doclingPageHeights(asList)[2]; h != 300.0 {
+ t.Fatalf("list-form pages height = %v, want 300", h)
+ }
+}
diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.go
index 3e8f740272..d0ce47c710 100644
--- a/internal/parser/parser/pdf_result.go
+++ b/internal/parser/parser/pdf_result.go
@@ -158,6 +158,15 @@ func (p *PDFParser) normalizePDFMarkdown(ctx context.Context, filename, markdown
}
if converted.File == nil {
converted.File = fileMeta
+ } else {
+ // The markdown parser always sets File with at least the name, so a
+ // nil check alone would silently drop producer-seeded metadata like
+ // page_count. Merge any keys the parser did not supply.
+ for key, value := range fileMeta {
+ if _, ok := converted.File[key]; !ok {
+ converted.File[key] = value
+ }
+ }
}
return p.NormalizeResult(ctx, filename, converted)
}