RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): emit Docling page positions and real page_count (#21330)
Taranum Wasuacd6d887a548
diff --git a/internal/ingestion/component/parser_dispatch_test.go b/internal/ingestion/component/parser_dispatch_test.goindex 587bcc97b5..ee0590576d 100644--- a/internal/ingestion/component/parser_dispatch_test.go+++ b/internal/ingestion/component/parser_dispatch_test.go@@ -1818,6 +1818,10 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {return}switch {+ // The parser tries standard conversions first (they carry page+ // provenance) and falls back to chunked endpoints only when the+ // server rejects the standard shape, so the third request is the+ // chunked /v1 call and answers with a chunk list.case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 1:w.WriteHeader(http.StatusUnprocessableEntity)_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))@@ -1826,7 +1830,7 @@ func TestDispatch_PDFDoclingMarkdown_UsesConfiguredBackend(t *testing.T) {_, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))case r.Method == http.MethodPost && r.URL.Path == "/v1/convert/source" && requestCount == 3:w.Header().Set("Content-Type", "application/json")- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))+ _, _ = w.Write([]byte(`[{"text":"# Docling Title\n\nDocling body.\n"}]`))default:http.NotFound(w, r)}diff --git a/internal/parser/parser/pdf_parser_docling.go b/internal/parser/parser/pdf_parser_docling.goindex ab9d414111..cfe2ce5d24 100644--- a/internal/parser/parser/pdf_parser_docling.go+++ b/internal/parser/parser/pdf_parser_docling.go@@ -1,6 +1,7 @@package parserimport (+ "bytes""context""encoding/base64""encoding/json"@@ -8,9 +9,13 @@ import ("io""maps""ragflow/internal/common"+ "regexp""strconv""strings"+ "github.com/ledongthuc/pdf"+ "go.uber.org/zap"+models "ragflow/internal/entity/models")@@ -66,28 +71,34 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parsbody func() map[string]anychunked bool}{+ // Standard responses carry per-page provenance (json_content.prov), so+ // they are tried first: positions are the point of this path. Chunked+ // responses have no provenance and stay as the fallback for servers or+ // documents the standard conversion rejects.{endpoint: "/v1/convert/source",- chunked: true,- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false, convertOptions) },+ chunked: false,+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, false, convertOptions) },},{endpoint: "/v1alpha/convert/source",- chunked: true,- body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true, convertOptions) },+ chunked: false,+ body: func() map[string]any { return doclingStandardPayload(filename, encoded, true, convertOptions) },},{endpoint: "/v1/convert/source",- chunked: false,- body: func() map[string]any { return doclingStandardPayload(filename, encoded, false, convertOptions) },+ chunked: true,+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, false, convertOptions) },},{endpoint: "/v1alpha/convert/source",- chunked: false,- body: func() map[string]any { return doclingStandardPayload(filename, encoded, true, convertOptions) },+ chunked: true,+ body: func() map[string]any { return doclingChunkedPayload(filename, encoded, true, convertOptions) },},}+ sourcePages := doclingSourcePageCount(data)+var lastErr errorfor _, candidate := range payloads {url := baseURL + candidate.endpoint@@ -113,13 +124,16 @@ func parsePDFWithDocling(ctx context.Context, filename string, data []byte, parscontinue}if candidate.chunked {- if res, ok := parseDoclingChunkedResult(filename, body); ok {+ if res, ok := parseDoclingChunkedResult(filename, body, sourcePages); ok {+ common.Warn("Docling chunked response carries no page provenance; positions unavailable",+ zap.String("filename", filename))+ res.Warnings = append(res.Warnings, "Docling chunked responses carry no page provenance; positions unavailable")return res}lastErr = fmt.Errorf("%s: chunked response contained no usable text", candidate.endpoint)continue}- if res, ok := parseDoclingStandardResult(ctx, filename, body); ok {+ if res, ok := parseDoclingStandardResult(ctx, filename, body, sourcePages); ok {return res}lastErr = fmt.Errorf("%s: standard response contained no parsed document", candidate.endpoint)@@ -184,7 +198,7 @@ func doclingChunkedPayload(filename string, encoded string, alpha bool, convertOreturn payload}-func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool) {+func parseDoclingChunkedResult(filename string, body []byte, sourcePages int) (ParseResult, bool) {var chunks []doclingChunkif err := json.Unmarshal(body, &chunks); err != nil {var wrapped struct {@@ -208,14 +222,16 @@ func parseDoclingChunkedResult(filename string, body []byte) (ParseResult, bool)if len(texts) == 0 {return ParseResult{}, false}- pageCount := 0- if len(texts) > 0 {- pageCount = len(texts)+ // A chunked response carries no page metadata: report the source+ // PDF's real page count when known, not the number of text chunks.+ pageCount := len(texts)+ if sourcePages > 0 {+ pageCount = sourcePages}return doclingTextsToResult(filename, texts, pageCount), true}-func parseDoclingStandardResult(ctx context.Context, filename string, body []byte) (ParseResult, bool) {+func parseDoclingStandardResult(ctx context.Context, filename string, body []byte, sourcePages int) (ParseResult, bool) {var payload doclingResponseif err := json.Unmarshal(body, &payload); err != nil {return ParseResult{}, false@@ -233,20 +249,256 @@ func parseDoclingStandardResult(ctx context.Context, filename string, body []bytdocs = append(docs, *result.Result)}}+ pageCount := doclingPageCount(docs, sourcePages)for _, doc := range docs {if md := strings.TrimSpace(doc.MDContent); md != "" {- return pdfMarkdownToResult(ctx, filename, md, len(docs)), true+ return doclingAttachPositions(pdfMarkdownToResult(ctx, filename, md, pageCount), doc.JSONContent), true}if txt := strings.TrimSpace(doc.TextContent); txt != "" {- return doclingTextsToResult(filename, []string{txt}, len(docs)), true+ return doclingAttachPositions(doclingTextsToResult(filename, []string{txt}, pageCount), doc.JSONContent), true}if md, _ := doc.JSONContent["md_content"].(string); strings.TrimSpace(md) != "" {- return pdfMarkdownToResult(ctx, filename, md, len(docs)), true+ return doclingAttachPositions(pdfMarkdownToResult(ctx, filename, md, pageCount), doc.JSONContent), true}}return ParseResult{}, false}+// doclingPageCount returns the number of PDF pages described by the Docling+// documents: the size of json_content.pages when present, otherwise the+// highest page_no found in any provenance entry. A single source converts to+// one logical document, so payload shapes that repeat it (document,+// documents, results) are duplicates: the maximum per-document count wins,+// never the sum. It falls back to the number of documents only when the+// response carries no page information.+func doclingPageCount(docs []doclingDocument, sourcePages int) int {+ total := 0+ for _, doc := range docs {+ count := 0+ switch pages := doc.JSONContent["pages"].(type) {+ case map[string]any:+ count = len(pages)+ case []any:+ count = len(pages)+ }+ for _, entry := range doclingTextEntries(doc.JSONContent) {+ for _, prov := range doclingProvs(entry) {+ if page, ok := numericAny(prov["page_no"]); ok && int(page) > count {+ count = int(page)+ }+ }+ }+ if count > total {+ total = count+ }+ }+ // Provenance can end before the final page when the backend extracts a+ // partial document, so the source PDF page count is a lower bound.+ if sourcePages > total {+ return sourcePages+ }+ if total == 0 {+ // No page metadata at all: report the number of returned documents.+ return len(docs)+ }+ return total+}++// doclingMaxSourcePageCount bounds a catalog /Count read from the source PDF;+// a corrupt or malicious value far beyond any real document is ignored.+const doclingMaxSourcePageCount = 1_000_000++// doclingSourcePageCount counts the pages of the source PDF so a Docling+// response without page metadata still reports the real page count. It+// returns 0 when the bytes cannot be opened or the count is implausible.+func doclingSourcePageCount(data []byte) int {+ if len(data) == 0 {+ return 0+ }+ reader, err := pdf.NewReader(bytes.NewReader(data), int64(len(data)))+ if err != nil {+ return 0+ }+ if pages := reader.NumPage(); pages > 0 && pages <= doclingMaxSourcePageCount {+ return pages+ }+ return 0+}++func doclingTextEntries(content map[string]any) []map[string]any {+ var entries []map[string]any+ texts, _ := content["texts"].([]any)+ for _, raw := range texts {+ if entry, ok := raw.(map[string]any); ok {+ entries = append(entries, entry)+ }+ }+ return entries+}++func doclingProvs(entry map[string]any) []map[string]any {+ var provs []map[string]any+ list, _ := entry["prov"].([]any)+ for _, raw := range list {+ if prov, ok := raw.(map[string]any); ok {+ provs = append(provs, prov)+ }+ }+ return provs+}++// doclingProvPosition converts a Docling provenance entry (1-based page_no and+// bbox {l, t, r, b}) into a [page, left, right, top, bottom] row with top+// above bottom. Docling permits bbox coord_origin BOTTOMLEFT (y measured from+// the page bottom); those boxes are converted to the top-left crop origin+// when the page height is known, otherwise the sorted values are kept.+func doclingProvPosition(prov map[string]any, heights map[int]float64) []any {+ page, ok := numericAny(prov["page_no"])+ if !ok || page < 1 {+ return nil+ }+ bbox, _ := prov["bbox"].(map[string]any)+ var v [4]float64+ for i, key := range []string{"l", "t", "r", "b"} {+ n, ok := numericAny(bbox[key])+ if !ok {+ return nil+ }+ v[i] = n+ }+ left, top, right, bottom := v[0], v[1], v[2], v[3]+ if origin, _ := bbox["coord_origin"].(string); strings.EqualFold(origin, "BOTTOMLEFT") {+ h := heights[int(page)]+ if h <= 0 {+ // Without the page height the bottom-origin Y cannot be converted+ // to the top-origin contract downstream; a missing position is+ // safer than a silently wrong crop.+ return nil+ }+ top, bottom = h-top, h-bottom+ }+ if left > right {+ left, right = right, left+ }+ if top > bottom {+ top, bottom = bottom, top+ }+ return []any{int(page), left, right, top, bottom}+}++// doclingPageHeights maps 1-based page numbers to page heights from the+// json_content pages metadata, used to convert BOTTOMLEFT bounding boxes.+func doclingPageHeights(content map[string]any) map[int]float64 {+ heights := map[int]float64{}+ collect := func(page map[string]any, keyNo int) {+ no, ok := numericAny(page["page_no"])+ if !ok && keyNo > 0 {+ no, ok = float64(keyNo), true+ }+ if !ok || no < 1 {+ return+ }+ size, _ := page["size"].(map[string]any)+ if h, ok := numericAny(size["height"]); ok && h > 0 {+ heights[int(no)] = h+ }+ }+ switch pages := content["pages"].(type) {+ case map[string]any:+ for key, raw := range pages {+ page, ok := raw.(map[string]any)+ if !ok {+ continue+ }+ keyNo, _ := strconv.Atoi(key)+ collect(page, keyNo)+ }+ case []any:+ for _, raw := range pages {+ if page, ok := raw.(map[string]any); ok {+ collect(page, 0)+ }+ }+ }+ return heights+}++func doclingNormalizeText(text string) string {+ return strings.Join(strings.Fields(text), " ")+}++var doclingMarkdownPrefix = regexp.MustCompile(`^(#{1,6}\s+|[-*]\s+|\d+\.\s+)`)++// doclingStripMarkdownPrefix removes one recognized Markdown heading or list+// marker. Leading digits elsewhere are content ("2026 Outlook") and stay.+func doclingStripMarkdownPrefix(text string) (string, bool) {+ stripped := doclingMarkdownPrefix.ReplaceAllString(text, "")+ return stripped, stripped != text+}++// doclingAttachPositions copies page positions from Docling's structured+// json_content onto JSON items whose text matches a Docling text element. Items+// without a match are left unchanged.+func doclingAttachPositions(res ParseResult, content map[string]any) ParseResult {+ if res.Err != nil || len(res.JSON) == 0 || len(content) == 0 {+ return res+ }+ // Group positions per Docling text entry so repeated text keeps its own+ // provenance: the i-th parsed occurrence takes the i-th entry's+ // positions instead of every position of every identical entry.+ byText := map[string][][][]any{}+ heights := doclingPageHeights(content)+ for _, entry := range doclingTextEntries(content) {+ text := doclingNormalizeText(stringValue(entry["text"]))+ if text == "" {+ continue+ }+ var positions [][]any+ for _, prov := range doclingProvs(entry) {+ if pos := doclingProvPosition(prov, heights); pos != nil {+ positions = append(positions, pos)+ }+ }+ if len(positions) > 0 {+ byText[text] = append(byText[text], positions)+ }+ }+ if len(byText) == 0 {+ return res+ }+ itemMaps := make([]map[string]any, len(res.JSON))+ matched := false+ used := map[string]int{}+ for i, item := range res.JSON {+ m := item.ToMap()+ text := doclingNormalizeText(stringValue(m["text"]))+ matches, ok := byText[text]+ if !ok {+ // List and heading markers are not content, but leading digits+ // can be; only retry without a recognized Markdown prefix.+ if stripped, changed := doclingStripMarkdownPrefix(text); changed {+ if matches, ok = byText[stripped]; ok {+ text = stripped+ }+ }+ }+ if ok && text != "" {+ if idx := used[text]; idx < len(matches) {+ m["positions"] = matches[idx]+ m["_pdf_positions"] = matches[idx]+ used[text]+++ matched = true+ }+ }+ itemMaps[i] = m+ }+ if !matched {+ return res+ }+ res.JSON = MapsToItems(itemMaps)+ return res+}+func doclingTextsToResult(filename string, texts []string, pageCount int) ParseResult {items := make([]map[string]any, 0, len(texts))for _, text := range texts {diff --git a/internal/parser/parser/pdf_parser_docling_test.go b/internal/parser/parser/pdf_parser_docling_test.goindex dac14b7b93..6f153f5934 100644--- a/internal/parser/parser/pdf_parser_docling_test.go+++ b/internal/parser/parser/pdf_parser_docling_test.go@@ -1,8 +1,10 @@package parserimport (+ "bytes""encoding/base64""encoding/json"+ "fmt""maps""net/http""net/http/httptest"@@ -31,6 +33,12 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {return}options, _ := body["options"].(map[string]any)+ if _, isChunked := options["do_chunking"]; !isChunked {+ // Standard payloads are tried first; this server only supports chunking.+ w.WriteHeader(http.StatusUnprocessableEntity)+ _, _ = w.Write([]byte(`{"detail":"standard conversion unsupported"}`))+ return+ }if got, want := options["do_chunking"], true; got != want {t.Errorf("do_chunking = %#v, want %#v", got, want)return@@ -55,8 +63,8 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {t.Errorf("uploaded file = %q, want PDF bytes", got)return}- if current != 1 {- t.Errorf("request count = %d, want first chunked request only", current)+ if current != 3 {+ t.Errorf("chunked success on request %d, want 3 (after both standard endpoints 422)", current)return}w.Header().Set("Content-Type", "application/json")@@ -88,49 +96,44 @@ func TestPDFParser_ParseWithResult_DoclingChunkedIntegration(t *testing.T) {if !strings.Contains(got.String(), "Chunk A") || !strings.Contains(got.String(), "Chunk B") {t.Fatalf("JSON text = %q, want both chunks", got.String())}+ warned := false+ for _, w := range res.Warnings {+ if strings.Contains(w, "positions unavailable") {+ warned = true+ }+ }+ if !warned {+ t.Fatalf("chunked fallback must warn that positions are unavailable: %#v", res.Warnings)+ }}-func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *testing.T) {+func TestPDFParser_ParseWithResult_DoclingStandardPreferredIntegration(t *testing.T) {withSSRFBypass(t)var requestCount atomic.Int32server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {current := requestCount.Add(1)- switch current {- case 1:- if r.URL.Path != "/v1/convert/source" {- t.Errorf("request 1 path = %q, want /v1/convert/source", r.URL.Path)- return- }- w.WriteHeader(http.StatusUnprocessableEntity)- _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))- case 2:- if r.URL.Path != "/v1alpha/convert/source" {- t.Errorf("request 2 path = %q, want /v1alpha/convert/source", r.URL.Path)- return- }- w.WriteHeader(http.StatusUnprocessableEntity)- _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))- case 3:- if r.URL.Path != "/v1/convert/source" {- t.Errorf("request 3 path = %q, want /v1/convert/source", r.URL.Path)- return- }- var body map[string]any- if err := json.NewDecoder(r.Body).Decode(&body); err != nil {- t.Errorf("Decode: %v", err)- return- }- options, _ := body["options"].(map[string]any)- if _, exists := options["do_chunking"]; exists {- t.Errorf("standard fallback payload unexpectedly contains do_chunking: %#v", options)- return- }- w.Header().Set("Content-Type", "application/json")- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))- default:+ if current != 1 {+ t.Errorf("request %d: standard conversion must succeed on the first request", current)http.NotFound(w, r)+ return+ }+ if r.URL.Path != "/v1/convert/source" {+ t.Errorf("request 1 path = %q, want /v1/convert/source", r.URL.Path)+ return+ }+ var body map[string]any+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {+ t.Errorf("Decode: %v", err)+ return+ }+ options, _ := body["options"].(map[string]any)+ if _, exists := options["do_chunking"]; exists {+ t.Errorf("standard payload unexpectedly contains do_chunking: %#v", options)+ return}+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))}))defer server.Close()@@ -151,7 +154,7 @@ func TestPDFParser_ParseWithResult_DoclingFallbackToStandardJSONIntegration(t *tif len(res.JSON) == 0 {t.Fatal("JSON is empty; want markdown-normalized items")}- if got, want := requestCount.Load(), int32(3); got != want {+ if got, want := requestCount.Load(), int32(1); got != want {t.Fatalf("requestCount = %d, want %d", got, want)}}@@ -227,13 +230,13 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {t.Errorf("request %d: options[%q] = %#v, want %#v", current, key, got, want)}}- // Reject both chunked requests so the standard payload is checked too.+ // Reject both standard requests so the chunked payload is checked too.if current < 3 {w.WriteHeader(http.StatusUnprocessableEntity)return}w.Header().Set("Content-Type", "application/json")- _, _ = w.Write([]byte(`{"document":{"md_content":"# Docling Title\n\nDocling body.\n"}}`))+ _, _ = w.Write([]byte(`[{"text":"Chunk A"}]`))}))defer server.Close()@@ -255,3 +258,256 @@ func TestPDFParser_ParseWithResult_DoclingConvertOptions(t *testing.T) {})}}++const doclingPositionsResponse = `{"document":{"md_content":"# Docling Title\n\nDocling body.\n\nUnmatched paragraph.\n","json_content":{+ "pages":{"1":{"page_no":1,"size":{"width":612,"height":792}},"2":{"page_no":2,"size":{"width":612,"height":792}},"3":{"page_no":3,"size":{"width":612,"height":792}}},+ "texts":[+ {"label":"section_header","text":"Docling Title","prov":[{"page_no":1,"bbox":{"l":10,"t":700,"r":200,"b":680,"coord_origin":"BOTTOMLEFT"}}]},+ {"label":"text","text":"Docling body.","prov":[{"page_no":2,"bbox":{"l":15,"t":100,"r":300,"b":140,"coord_origin":"TOPLEFT"}}]}+ ]}}}`++func TestPDFParser_ParseWithResult_DoclingEmitsPositionsAndPageCount(t *testing.T) {+ withSSRFBypass(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ var body map[string]any+ _ = json.NewDecoder(r.Body).Decode(&body)+ options, _ := body["options"].(map[string]any)+ if _, chunked := options["do_chunking"]; chunked {+ w.WriteHeader(http.StatusUnprocessableEntity)+ _, _ = w.Write([]byte(`{"detail":"chunking unsupported"}`))+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(doclingPositionsResponse))+ }))+ defer server.Close()++ pdf := NewPDFParser()+ pdf.ConfigureFromSetup(map[string]any{+ "parse_method": "Docling",+ "output_format": "json",+ "docling_server_url": server.URL,+ })+ res := pdf.ParseWithResult(t.Context(), "sample.pdf", []byte("%PDF-1.4\nmock"))+ if res.Err != nil {+ t.Fatalf("ParseWithResult: %v", res.Err)+ }+ if got := res.File["page_count"]; got != 3 {+ t.Fatalf("page_count = %v, want 3 (pages, not documents)", got)+ }+ want := map[string][]any{+ // BOTTOMLEFT y=700/680 on a 792pt page convert to top-origin 92/112.+ "Docling Title": {1, 10.0, 200.0, 92.0, 112.0},+ "Docling body.": {2, 15.0, 300.0, 100.0, 140.0},+ }+ found := map[string]bool{}+ for _, item := range res.JSON {+ m := item.ToMap()+ text := strings.TrimSpace(strings.TrimLeft(stringValue(m["text"]), "# "))+ rows, ok := ExtractPDFPositions(m)+ if w, expected := want[text]; expected {+ if !ok || len(rows) != 1 || len(rows[0]) != 5 {+ t.Fatalf("%q positions = %#v ok=%v", text, rows, ok)+ }+ for i := range w {+ got, _ := numericAny(rows[0][i])+ exp, _ := numericAny(w[i])+ if got != exp {+ t.Fatalf("%q positions[%d] = %v, want %v (row %#v)", text, i, rows[0][i], w[i], rows[0])+ }+ }+ found[text] = true+ } else if ok {+ t.Fatalf("unmatched item %q must not carry positions: %#v", text, rows)+ }+ }+ if len(found) != 2 {+ t.Fatalf("matched items = %v, want both", found)+ }+}++func TestDoclingPageCountFallsBackToProvenanceThenDocuments(t *testing.T) {+ withProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 4.0}}},+ }}}}+ if got := doclingPageCount(withProv, 0); got != 4 {+ t.Fatalf("provenance page count = %d, want 4", got)+ }+ noPages := []doclingDocument{{MDContent: "x"}}+ if got := doclingPageCount(noPages, 50); got != 50 {+ t.Fatalf("metadata-less response must report source PDF pages = %d, want 50", got)+ }+ if got := doclingPageCount(noPages, 0); got != 1 {+ t.Fatalf("unreadable source page count = %d, want 1 (document count)", got)+ }+}++func TestDoclingPageCountMaxAcrossDuplicateDocuments(t *testing.T) {+ // A single source converts to one logical document; payload shapes that+ // repeat it (document + documents + results) must not multiply the page+ // count.+ pageful := doclingDocument{JSONContent: map[string]any{+ "pages": map[string]any{"1": map[string]any{"page_no": 1.0}, "2": map[string]any{"page_no": 2.0}, "3": map[string]any{"page_no": 3.0}},+ }}+ if got := doclingPageCount([]doclingDocument{pageful, pageful}, 0); got != 3 {+ t.Fatalf("duplicated documents must not sum pages: got %v, want 3", got)+ }+}++func TestDoclingSourcePageCount(t *testing.T) {+ if got := doclingSourcePageCount(buildDoclingPDFWithCount(t, 50)); got != 50 {+ t.Fatalf("source page count = %d, want 50", got)+ }+ if got := doclingSourcePageCount([]byte("%PDF-1.4\nmock")); got != 0 {+ t.Fatalf("unopenable PDF must yield 0, got %d", got)+ }+ if got := doclingSourcePageCount(buildDoclingPDFWithCount(t, 1<<30)); got != 0 {+ t.Fatalf("implausible /Count must be ignored, got %d", got)+ }+}++// buildDoclingPDFWithCount emits a minimal valid PDF whose /Pages /Count is+// the supplied value, mirroring the deepdoc plaintext test fixture.+func buildDoclingPDFWithCount(t *testing.T, count int) []byte {+ t.Helper()+ var buf bytes.Buffer+ buf.WriteString("%PDF-1.4\n")+ off1 := buf.Len()+ fmt.Fprintf(&buf, "1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n")+ off2 := buf.Len()+ fmt.Fprintf(&buf, "2 0 obj\n<< /Type /Pages /Kids [3 0 R] /Count %d >>\nendobj\n", count)+ off3 := buf.Len()+ fmt.Fprintf(&buf, "3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << >> >>\nendobj\n")+ xrefStart := buf.Len()+ buf.WriteString("xref\n0 4\n")+ buf.WriteString("0000000000 65535 f \n")+ fmt.Fprintf(&buf, "%010d 00000 n \n", off1)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off2)+ fmt.Fprintf(&buf, "%010d 00000 n \n", off3)+ fmt.Fprintf(&buf, "trailer\n<< /Size 4 /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n", xrefStart)+ return buf.Bytes()+}++func TestDoclingAttachPositionsMatchesOccurrencesInOrder(t *testing.T) {+ content := map[string]any{"texts": []any{+ map[string]any{"text": "Repeated paragraph.", "prov": []any{+ map[string]any{"page_no": 2.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},+ }},+ map[string]any{"text": "Repeated paragraph.", "prov": []any{+ map[string]any{"page_no": 5.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},+ }},+ }}+ res := ParseResult{JSON: MapsToItems([]map[string]any{+ {"text": "Repeated paragraph."},+ {"text": "Repeated paragraph."},+ })}+ res = doclingAttachPositions(res, content)+ first, _ := res.JSON[0].ToMap()["_pdf_positions"].([][]any)+ second, _ := res.JSON[1].ToMap()["_pdf_positions"].([][]any)+ if len(first) != 1 || first[0][0] != 2 {+ t.Fatalf("first occurrence must take page 2: %#v", first)+ }+ if len(second) != 1 || second[0][0] != 5 {+ t.Fatalf("second occurrence must take page 5: %#v", second)+ }+}++func TestDoclingAttachPositionsKeepsLeadingDigits(t *testing.T) {+ content := map[string]any{"texts": []any{+ map[string]any{"text": "2026 Outlook", "prov": []any{+ map[string]any{"page_no": 3.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},+ }},+ map[string]any{"text": "Introduction", "prov": []any{+ map[string]any{"page_no": 4.0, "bbox": map[string]any{"l": 10.0, "t": 20.0, "r": 100.0, "b": 40.0}},+ }},+ }}+ res := ParseResult{JSON: MapsToItems([]map[string]any{+ {"text": "2026 Outlook"},+ {"text": "1. Introduction"},+ })}+ res = doclingAttachPositions(res, content)+ digits, ok := res.JSON[0].ToMap()["_pdf_positions"].([][]any)+ if !ok || len(digits) != 1 || digits[0][0] != 3 {+ t.Fatalf("leading digits are content and must match: %#v", res.JSON[0].ToMap())+ }+ numbered, ok := res.JSON[1].ToMap()["_pdf_positions"].([][]any)+ if !ok || len(numbered) != 1 || numbered[0][0] != 4 {+ t.Fatalf("numbered list prefix must strip for the match: %#v", res.JSON[1].ToMap())+ }+}++func TestDoclingChunkedResultPrefersSourcePageCount(t *testing.T) {+ body := []byte(`[{"text":"chunk one"},{"text":"chunk two"}]`)+ res, ok := parseDoclingChunkedResult("a.pdf", body, 9)+ if !ok {+ t.Fatal("chunked parse failed")+ }+ if got := res.File["page_count"]; got != 9 {+ t.Fatalf("chunked page_count = %v, want source PDF count 9", got)+ }+ res, ok = parseDoclingChunkedResult("a.pdf", body, 0)+ if !ok {+ t.Fatal("chunked parse failed")+ }+ if got := res.File["page_count"]; got != 2 {+ t.Fatalf("without source count, page_count = %v, want chunk count 2", got)+ }+}++func TestDoclingPageCountSourceLowerBound(t *testing.T) {+ shortProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 3.0}}},+ }}}}+ if got := doclingPageCount(shortProv, 9); got != 9 {+ t.Fatalf("provenance ending early must not underreport: got %d, want 9", got)+ }+ fullProv := []doclingDocument{{JSONContent: map[string]any{"texts": []any{+ map[string]any{"text": "a", "prov": []any{map[string]any{"page_no": 12.0}}},+ }}}}+ if got := doclingPageCount(fullProv, 9); got != 12 {+ t.Fatalf("metadata above source count still wins: got %d, want 12", got)+ }+}++func TestDoclingProvPositionConvertsBottomLeftOrigin(t *testing.T) {+ heights := map[int]float64{1: 200}+ prov := map[string]any{"page_no": 1.0, "bbox": map[string]any{+ "l": 10.0, "t": 180.0, "r": 50.0, "b": 170.0, "coord_origin": "BOTTOMLEFT",+ }}+ pos := doclingProvPosition(prov, heights)+ if pos == nil {+ t.Fatal("position dropped")+ }+ // BOTTOMLEFT y=180/170 from the bottom become 20/30 from the top.+ if pos[3] != 20.0 || pos[4] != 30.0 {+ t.Fatalf("BOTTOMLEFT not converted: %#v", pos)+ }+ // Unknown page height: the bottom-origin Y cannot meet the top-origin+ // contract, so the position is dropped rather than emitted wrong.+ if pos = doclingProvPosition(prov, map[int]float64{}); pos != nil {+ t.Fatalf("missing height must drop the position, got %#v", pos)+ }+ // TOPLEFT boxes are untouched even with a known height.+ provTL := map[string]any{"page_no": 1.0, "bbox": map[string]any{+ "l": 10.0, "t": 20.0, "r": 50.0, "b": 30.0, "coord_origin": "TOPLEFT",+ }}+ pos = doclingProvPosition(provTL, heights)+ if pos == nil || pos[3] != 20.0 || pos[4] != 30.0 {+ t.Fatalf("TOPLEFT must pass through unchanged: %#v", pos)+ }+}++func TestDoclingPageHeightsFromPagesMetadata(t *testing.T) {+ asMap := map[string]any{"pages": map[string]any{+ "1": map[string]any{"size": map[string]any{"width": 100.0, "height": 200.0}},+ }}+ if h := doclingPageHeights(asMap)[1]; h != 200.0 {+ t.Fatalf("map-form pages height = %v, want 200", h)+ }+ asList := map[string]any{"pages": []any{+ map[string]any{"page_no": 2.0, "size": map[string]any{"height": 300.0}},+ }}+ if h := doclingPageHeights(asList)[2]; h != 300.0 {+ t.Fatalf("list-form pages height = %v, want 300", h)+ }+}diff --git a/internal/parser/parser/pdf_result.go b/internal/parser/parser/pdf_result.goindex 3e8f740272..d0ce47c710 100644--- a/internal/parser/parser/pdf_result.go+++ b/internal/parser/parser/pdf_result.go@@ -158,6 +158,15 @@ func (p *PDFParser) normalizePDFMarkdown(ctx context.Context, filename, markdown}if converted.File == nil {converted.File = fileMeta+ } else {+ // The markdown parser always sets File with at least the name, so a+ // nil check alone would silently drop producer-seeded metadata like+ // page_count. Merge any keys the parser did not supply.+ for key, value := range fileMeta {+ if _, ok := converted.File[key]; !ok {+ converted.File[key] = value+ }+ }}return p.NormalizeResult(ctx, filename, converted)}