Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(pdf): resolve bookmark page mapping (#1412)

Qin Haojie121fc936eb8a
diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.py
index cb4916615..97c1ce6d1 100644
--- a/openviking/parse/parsers/pdf.py
+++ b/openviking/parse/parsers/pdf.py
@@ -232,6 +232,9 @@ class PDFParser(BaseParser):
"images_extracted": 0,
"tables_extracted": 0,
"bookmarks_found": 0,
+ "bookmarks_resolved": 0,
+ "bookmarks_unresolved": 0,
+ "headings_found": 0,
"heading_source": "none",
}
@@ -242,27 +245,48 @@ class PDFParser(BaseParser):
# Extract structure (bookmarks → font fallback)
detection_mode = self.config.heading_detection
bookmarks = []
+ raw_bookmarks = []
heading_source = "none"
if detection_mode in ("bookmarks", "auto"):
- bookmarks = self._extract_bookmarks(pdf)
+ raw_bookmarks = self._extract_bookmarks(pdf)
+ meta["bookmarks_found"] = len(raw_bookmarks)
+ bookmarks = [bm for bm in raw_bookmarks if bm["page_num"] is not None]
+ meta["bookmarks_resolved"] = len(bookmarks)
+ meta["bookmarks_unresolved"] = len(raw_bookmarks) - len(bookmarks)
+
if bookmarks:
heading_source = "bookmarks"
+ elif raw_bookmarks:
+ logger.info(
+ "Bookmark detection found %d entries but none resolved to pages; "
+ "ignoring bookmark headings",
+ len(raw_bookmarks),
+ )
if not bookmarks and detection_mode in ("font", "auto"):
bookmarks = self._detect_headings_by_font(pdf)
if bookmarks:
heading_source = "font_analysis"
- meta["bookmarks_found"] = len(bookmarks)
+ meta["headings_found"] = len(bookmarks)
meta["heading_source"] = heading_source
- logger.info(f"Heading detection: {heading_source}, found {len(bookmarks)} headings")
+ logger.info(
+ "Heading detection: source=%s, headings=%d, bookmarks=%d, resolved=%d, "
+ "unresolved=%d",
+ heading_source,
+ len(bookmarks),
+ meta["bookmarks_found"],
+ meta["bookmarks_resolved"],
+ meta["bookmarks_unresolved"],
+ )
# Group bookmarks by page_num
bookmarks_by_page = defaultdict(list)
for bm in bookmarks:
- # Fall back to page 1 for unresolvable destinations
- page = bm["page_num"] or 1
+ page = bm["page_num"]
+ if page is None:
+ continue
bookmarks_by_page[page].append(bm)
for page_num, page in enumerate(pdf.pages, 1):
@@ -315,8 +339,6 @@ class PDFParser(BaseParser):
f"Failed to extract image {img_idx + 1} on page {page_num}: {img_err}"
)
- # Note: bookmarks with unresolvable page numbers are injected at page 1
-
if not parts:
logger.warning(f"No content extracted from {pdf_path}")
return "", meta
@@ -324,7 +346,9 @@ class PDFParser(BaseParser):
markdown_content = "\n\n".join(parts)
logger.info(
f"Local conversion: {meta['pages_processed']}/{meta['total_pages']} pages, "
- f"{meta['bookmarks_found']} bookmarks ({meta['heading_source']}), "
+ f"{meta['headings_found']} headings ({meta['heading_source']}, "
+ f"bookmarks={meta['bookmarks_found']}, "
+ f"resolved={meta['bookmarks_resolved']}), "
f"{meta['images_extracted']} images, {meta['tables_extracted']} tables → "
f"{len(markdown_content)} chars"
)
@@ -344,16 +368,11 @@ class PDFParser(BaseParser):
if not hasattr(pdf, "doc") or not hasattr(pdf.doc, "get_outlines"):
return []
- outlines = pdf.doc.get_outlines()
+ outlines = list(pdf.doc.get_outlines())
if not outlines:
return []
- # Build objid → page_number mapping
- objid_to_num = {
- page.page_obj.objid: i + 1
- for i, page in enumerate(pdf.pages)
- if hasattr(page, "page_obj") and hasattr(page.page_obj, "objid")
- }
+ page_ref_to_num = self._build_page_number_map(pdf)
bookmarks = []
for level, title, dest, _action, _se in outlines:
@@ -363,18 +382,9 @@ class PDFParser(BaseParser):
page_num = None
try:
if dest and len(dest) > 0:
- page_ref = dest[0]
- if hasattr(page_ref, "objid"):
- page_num = objid_to_num.get(page_ref.objid)
- elif hasattr(page_ref, "resolve"):
- resolved = page_ref.resolve()
- if hasattr(resolved, "objid"):
- page_num = objid_to_num.get(resolved.objid)
- elif isinstance(page_ref, int):
- # 0-based integer page index (common in many PDF producers)
- candidate = page_ref + 1
- if 1 <= candidate <= len(pdf.pages):
- page_num = candidate
+ page_num = self._resolve_bookmark_page(
+ dest[0], page_ref_to_num, len(pdf.pages)
+ )
except Exception:
pass
@@ -392,6 +402,50 @@ class PDFParser(BaseParser):
logger.warning(f"Failed to extract bookmarks: {e}")
return []
+ def _build_page_number_map(self, pdf) -> Dict[int, int]:
+ """Build a lookup from PDF page object ids to 1-based page numbers.
+
+ pdfminer outlines and link annotations reference page objects by object id.
+ In pdfplumber these ids are exposed as ``page.page_obj.pageid``; some mocks
+ or alternate inputs may still expose ``objid``, so we keep both.
+ """
+ page_ref_to_num: Dict[int, int] = {}
+ for page_num, page in enumerate(pdf.pages, 1):
+ page_obj = getattr(page, "page_obj", None)
+ if page_obj is None:
+ continue
+
+ for attr_name in ("pageid", "objid"):
+ ref_id = getattr(page_obj, attr_name, None)
+ if isinstance(ref_id, int):
+ page_ref_to_num.setdefault(ref_id, page_num)
+
+ return page_ref_to_num
+
+ def _resolve_bookmark_page(
+ self, page_ref: Any, page_ref_to_num: Dict[int, int], total_pages: int
+ ) -> Optional[int]:
+ """Resolve a bookmark destination to a 1-based page number."""
+ ref_id = getattr(page_ref, "objid", None)
+ if isinstance(ref_id, int):
+ return page_ref_to_num.get(ref_id)
+
+ if isinstance(page_ref, int):
+ # 0-based integer page index (common in many PDF producers)
+ candidate = page_ref + 1
+ if 1 <= candidate <= total_pages:
+ return candidate
+ return None
+
+ if hasattr(page_ref, "resolve"):
+ resolved = page_ref.resolve()
+ for attr_name in ("pageid", "objid"):
+ resolved_id = getattr(resolved, attr_name, None)
+ if isinstance(resolved_id, int):
+ return page_ref_to_num.get(resolved_id)
+
+ return None
+
def _detect_headings_by_font(self, pdf) -> List[Dict[str, Any]]:
"""Detect headings by font size analysis.
diff --git a/tests/parse/test_pdf_bookmark_extraction.py b/tests/parse/test_pdf_bookmark_extraction.py
index 4f6fc97e2..14cb53ec4 100644
--- a/tests/parse/test_pdf_bookmark_extraction.py
+++ b/tests/parse/test_pdf_bookmark_extraction.py
@@ -7,11 +7,39 @@ Verifies that _extract_bookmarks correctly extracts bookmark entries
and that _convert_local injects them as markdown headings.
"""
-from unittest.mock import MagicMock
+from contextlib import nullcontext
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
from openviking.parse.parsers.pdf import PDFParser
+def _make_page(*, pageid=None, objid=None):
+ """Create a minimal page stub for bookmark extraction tests."""
+ return SimpleNamespace(page_obj=SimpleNamespace(pageid=pageid, objid=objid))
+
+
+def _make_ref(objid):
+ """Create a minimal PDF object reference stub."""
+ return SimpleNamespace(objid=objid)
+
+
+class _FakePage:
+ """Minimal pdfplumber page stub for _convert_local tests."""
+
+ def __init__(self, text: str):
+ self._text = text
+ self.images = []
+
+ def extract_text(self):
+ return self._text
+
+ def extract_tables(self):
+ return []
+
+
class TestExtractBookmarks:
"""Test PDF bookmark extraction logic."""
@@ -23,18 +51,12 @@ class TestExtractBookmarks:
# Mock pdfplumber PDF object
mock_pdf = MagicMock()
- # Mock pages with objid for page mapping
- mock_page1 = MagicMock()
- mock_page1.page_obj.objid = 100
- mock_page2 = MagicMock()
- mock_page2.page_obj.objid = 200
- mock_pdf.pages = [mock_page1, mock_page2]
+ # Real pdfminer outlines point at page.page_obj.pageid
+ mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)]
# Mock page reference objects for bookmark destinations
- mock_ref1 = MagicMock()
- mock_ref1.objid = 100 # Points to page 1
- mock_ref2 = MagicMock()
- mock_ref2.objid = 200 # Points to page 2
+ mock_ref1 = _make_ref(100) # Points to page 1
+ mock_ref2 = _make_ref(200) # Points to page 2
# Mock document outlines: (level, title, dest, action, structelem)
mock_pdf.doc.get_outlines.return_value = [
@@ -50,6 +72,19 @@ class TestExtractBookmarks:
assert bookmarks[1] == {"title": "Section 1.1", "level": 2, "page_num": 1}
assert bookmarks[2] == {"title": "Chapter 2", "level": 1, "page_num": 2}
+ def test_extract_bookmarks_falls_back_to_objid_mapping(self):
+ """Objid-based mapping remains supported for tests and alternate backends."""
+ mock_pdf = MagicMock()
+ mock_pdf.pages = [_make_page(objid=100), _make_page(objid=200)]
+
+ mock_pdf.doc.get_outlines.return_value = [
+ (1, "Chapter 1", [_make_ref(100), "/Fit"], None, None),
+ (1, "Chapter 2", [_make_ref(200), "/Fit"], None, None),
+ ]
+
+ bookmarks = self.parser._extract_bookmarks(mock_pdf)
+ assert [b["page_num"] for b in bookmarks] == [1, 2]
+
def test_extract_bookmarks_no_outlines(self):
"""Returns empty list when PDF has no outlines."""
mock_pdf = MagicMock()
@@ -108,12 +143,7 @@ class TestExtractBookmarks:
def test_extract_bookmarks_integer_page_index(self):
"""Bookmarks with integer destination (0-based) are resolved correctly."""
mock_pdf = MagicMock()
-
- mock_page1 = MagicMock()
- mock_page1.page_obj.objid = 100
- mock_page2 = MagicMock()
- mock_page2.page_obj.objid = 200
- mock_pdf.pages = [mock_page1, mock_page2]
+ mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)]
# Integer page indices instead of object references
mock_pdf.doc.get_outlines.return_value = [
@@ -131,10 +161,7 @@ class TestExtractBookmarks:
def test_extract_bookmarks_integer_page_index_out_of_range(self):
"""Out-of-range integer page indices are treated as unresolved."""
mock_pdf = MagicMock()
-
- mock_page1 = MagicMock()
- mock_page1.page_obj.objid = 100
- mock_pdf.pages = [mock_page1] # Only 1 page
+ mock_pdf.pages = [_make_page(pageid=100)] # Only 1 page
mock_pdf.doc.get_outlines.return_value = [
(1, "Valid", [0, "/Fit"], None, None),
@@ -156,3 +183,67 @@ class TestExtractBookmarks:
bookmarks = self.parser._extract_bookmarks(mock_pdf)
assert bookmarks == []
+
+
+class TestConvertLocalBookmarks:
+ """Test bookmark injection behavior in local PDF conversion."""
+
+ @pytest.mark.asyncio
+ async def test_convert_local_skips_unresolved_bookmarks(self):
+ parser = PDFParser()
+ fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")])
+ fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf))
+
+ with (
+ patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber),
+ patch.object(
+ parser,
+ "_extract_bookmarks",
+ return_value=[
+ {"level": 1, "title": "Broken Bookmark", "page_num": None},
+ {"level": 1, "title": "Chapter 2", "page_num": 2},
+ ],
+ ),
+ ):
+ markdown, meta = await parser._convert_local(
+ "dummy.pdf", storage=MagicMock(), resource_name="dummy"
+ )
+
+ assert "Broken Bookmark" not in markdown
+ assert "\n# Chapter 2\n" in markdown
+ assert meta["bookmarks_found"] == 2
+ assert meta["bookmarks_resolved"] == 1
+ assert meta["bookmarks_unresolved"] == 1
+ assert meta["headings_found"] == 1
+ assert meta["heading_source"] == "bookmarks"
+
+ @pytest.mark.asyncio
+ async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self):
+ parser = PDFParser()
+ fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")])
+ fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf))
+
+ with (
+ patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber),
+ patch.object(
+ parser,
+ "_extract_bookmarks",
+ return_value=[{"level": 1, "title": "Broken Bookmark", "page_num": None}],
+ ),
+ patch.object(
+ parser,
+ "_detect_headings_by_font",
+ return_value=[{"level": 1, "title": "Font Heading", "page_num": 2}],
+ ),
+ ):
+ markdown, meta = await parser._convert_local(
+ "dummy.pdf", storage=MagicMock(), resource_name="dummy"
+ )
+
+ assert "Broken Bookmark" not in markdown
+ assert "\n# Font Heading\n" in markdown
+ assert meta["bookmarks_found"] == 1
+ assert meta["bookmarks_resolved"] == 0
+ assert meta["bookmarks_unresolved"] == 1
+ assert meta["headings_found"] == 1
+ assert meta["heading_source"] == "font_analysis"