Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(pdf): resolve bookmark page mapping (#1412)
Qin Haojie121fc936eb8a
diff --git a/openviking/parse/parsers/pdf.py b/openviking/parse/parsers/pdf.pyindex cb4916615..97c1ce6d1 100644--- a/openviking/parse/parsers/pdf.py+++ b/openviking/parse/parsers/pdf.py@@ -232,6 +232,9 @@ class PDFParser(BaseParser):"images_extracted": 0,"tables_extracted": 0,"bookmarks_found": 0,+ "bookmarks_resolved": 0,+ "bookmarks_unresolved": 0,+ "headings_found": 0,"heading_source": "none",}@@ -242,27 +245,48 @@ class PDFParser(BaseParser):# Extract structure (bookmarks → font fallback)detection_mode = self.config.heading_detectionbookmarks = []+ raw_bookmarks = []heading_source = "none"if detection_mode in ("bookmarks", "auto"):- bookmarks = self._extract_bookmarks(pdf)+ raw_bookmarks = self._extract_bookmarks(pdf)+ meta["bookmarks_found"] = len(raw_bookmarks)+ bookmarks = [bm for bm in raw_bookmarks if bm["page_num"] is not None]+ meta["bookmarks_resolved"] = len(bookmarks)+ meta["bookmarks_unresolved"] = len(raw_bookmarks) - len(bookmarks)+if bookmarks:heading_source = "bookmarks"+ elif raw_bookmarks:+ logger.info(+ "Bookmark detection found %d entries but none resolved to pages; "+ "ignoring bookmark headings",+ len(raw_bookmarks),+ )if not bookmarks and detection_mode in ("font", "auto"):bookmarks = self._detect_headings_by_font(pdf)if bookmarks:heading_source = "font_analysis"- meta["bookmarks_found"] = len(bookmarks)+ meta["headings_found"] = len(bookmarks)meta["heading_source"] = heading_source- logger.info(f"Heading detection: {heading_source}, found {len(bookmarks)} headings")+ logger.info(+ "Heading detection: source=%s, headings=%d, bookmarks=%d, resolved=%d, "+ "unresolved=%d",+ heading_source,+ len(bookmarks),+ meta["bookmarks_found"],+ meta["bookmarks_resolved"],+ meta["bookmarks_unresolved"],+ )# Group bookmarks by page_numbookmarks_by_page = defaultdict(list)for bm in bookmarks:- # Fall back to page 1 for unresolvable destinations- page = bm["page_num"] or 1+ page = bm["page_num"]+ if page is None:+ continuebookmarks_by_page[page].append(bm)for page_num, page in enumerate(pdf.pages, 1):@@ -315,8 +339,6 @@ class PDFParser(BaseParser):f"Failed to extract image {img_idx + 1} on page {page_num}: {img_err}")- # Note: bookmarks with unresolvable page numbers are injected at page 1-if not parts:logger.warning(f"No content extracted from {pdf_path}")return "", meta@@ -324,7 +346,9 @@ class PDFParser(BaseParser):markdown_content = "\n\n".join(parts)logger.info(f"Local conversion: {meta['pages_processed']}/{meta['total_pages']} pages, "- f"{meta['bookmarks_found']} bookmarks ({meta['heading_source']}), "+ f"{meta['headings_found']} headings ({meta['heading_source']}, "+ f"bookmarks={meta['bookmarks_found']}, "+ f"resolved={meta['bookmarks_resolved']}), "f"{meta['images_extracted']} images, {meta['tables_extracted']} tables → "f"{len(markdown_content)} chars")@@ -344,16 +368,11 @@ class PDFParser(BaseParser):if not hasattr(pdf, "doc") or not hasattr(pdf.doc, "get_outlines"):return []- outlines = pdf.doc.get_outlines()+ outlines = list(pdf.doc.get_outlines())if not outlines:return []- # Build objid → page_number mapping- objid_to_num = {- page.page_obj.objid: i + 1- for i, page in enumerate(pdf.pages)- if hasattr(page, "page_obj") and hasattr(page.page_obj, "objid")- }+ page_ref_to_num = self._build_page_number_map(pdf)bookmarks = []for level, title, dest, _action, _se in outlines:@@ -363,18 +382,9 @@ class PDFParser(BaseParser):page_num = Nonetry:if dest and len(dest) > 0:- page_ref = dest[0]- if hasattr(page_ref, "objid"):- page_num = objid_to_num.get(page_ref.objid)- elif hasattr(page_ref, "resolve"):- resolved = page_ref.resolve()- if hasattr(resolved, "objid"):- page_num = objid_to_num.get(resolved.objid)- elif isinstance(page_ref, int):- # 0-based integer page index (common in many PDF producers)- candidate = page_ref + 1- if 1 <= candidate <= len(pdf.pages):- page_num = candidate+ page_num = self._resolve_bookmark_page(+ dest[0], page_ref_to_num, len(pdf.pages)+ )except Exception:pass@@ -392,6 +402,50 @@ class PDFParser(BaseParser):logger.warning(f"Failed to extract bookmarks: {e}")return []+ def _build_page_number_map(self, pdf) -> Dict[int, int]:+ """Build a lookup from PDF page object ids to 1-based page numbers.++ pdfminer outlines and link annotations reference page objects by object id.+ In pdfplumber these ids are exposed as ``page.page_obj.pageid``; some mocks+ or alternate inputs may still expose ``objid``, so we keep both.+ """+ page_ref_to_num: Dict[int, int] = {}+ for page_num, page in enumerate(pdf.pages, 1):+ page_obj = getattr(page, "page_obj", None)+ if page_obj is None:+ continue++ for attr_name in ("pageid", "objid"):+ ref_id = getattr(page_obj, attr_name, None)+ if isinstance(ref_id, int):+ page_ref_to_num.setdefault(ref_id, page_num)++ return page_ref_to_num++ def _resolve_bookmark_page(+ self, page_ref: Any, page_ref_to_num: Dict[int, int], total_pages: int+ ) -> Optional[int]:+ """Resolve a bookmark destination to a 1-based page number."""+ ref_id = getattr(page_ref, "objid", None)+ if isinstance(ref_id, int):+ return page_ref_to_num.get(ref_id)++ if isinstance(page_ref, int):+ # 0-based integer page index (common in many PDF producers)+ candidate = page_ref + 1+ if 1 <= candidate <= total_pages:+ return candidate+ return None++ if hasattr(page_ref, "resolve"):+ resolved = page_ref.resolve()+ for attr_name in ("pageid", "objid"):+ resolved_id = getattr(resolved, attr_name, None)+ if isinstance(resolved_id, int):+ return page_ref_to_num.get(resolved_id)++ return None+def _detect_headings_by_font(self, pdf) -> List[Dict[str, Any]]:"""Detect headings by font size analysis.diff --git a/tests/parse/test_pdf_bookmark_extraction.py b/tests/parse/test_pdf_bookmark_extraction.pyindex 4f6fc97e2..14cb53ec4 100644--- a/tests/parse/test_pdf_bookmark_extraction.py+++ b/tests/parse/test_pdf_bookmark_extraction.py@@ -7,11 +7,39 @@ Verifies that _extract_bookmarks correctly extracts bookmark entriesand that _convert_local injects them as markdown headings."""-from unittest.mock import MagicMock+from contextlib import nullcontext+from types import SimpleNamespace+from unittest.mock import MagicMock, patch++import pytestfrom openviking.parse.parsers.pdf import PDFParser+def _make_page(*, pageid=None, objid=None):+ """Create a minimal page stub for bookmark extraction tests."""+ return SimpleNamespace(page_obj=SimpleNamespace(pageid=pageid, objid=objid))+++def _make_ref(objid):+ """Create a minimal PDF object reference stub."""+ return SimpleNamespace(objid=objid)+++class _FakePage:+ """Minimal pdfplumber page stub for _convert_local tests."""++ def __init__(self, text: str):+ self._text = text+ self.images = []++ def extract_text(self):+ return self._text++ def extract_tables(self):+ return []++class TestExtractBookmarks:"""Test PDF bookmark extraction logic."""@@ -23,18 +51,12 @@ class TestExtractBookmarks:# Mock pdfplumber PDF objectmock_pdf = MagicMock()- # Mock pages with objid for page mapping- mock_page1 = MagicMock()- mock_page1.page_obj.objid = 100- mock_page2 = MagicMock()- mock_page2.page_obj.objid = 200- mock_pdf.pages = [mock_page1, mock_page2]+ # Real pdfminer outlines point at page.page_obj.pageid+ mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)]# Mock page reference objects for bookmark destinations- mock_ref1 = MagicMock()- mock_ref1.objid = 100 # Points to page 1- mock_ref2 = MagicMock()- mock_ref2.objid = 200 # Points to page 2+ mock_ref1 = _make_ref(100) # Points to page 1+ mock_ref2 = _make_ref(200) # Points to page 2# Mock document outlines: (level, title, dest, action, structelem)mock_pdf.doc.get_outlines.return_value = [@@ -50,6 +72,19 @@ class TestExtractBookmarks:assert bookmarks[1] == {"title": "Section 1.1", "level": 2, "page_num": 1}assert bookmarks[2] == {"title": "Chapter 2", "level": 1, "page_num": 2}+ def test_extract_bookmarks_falls_back_to_objid_mapping(self):+ """Objid-based mapping remains supported for tests and alternate backends."""+ mock_pdf = MagicMock()+ mock_pdf.pages = [_make_page(objid=100), _make_page(objid=200)]++ mock_pdf.doc.get_outlines.return_value = [+ (1, "Chapter 1", [_make_ref(100), "/Fit"], None, None),+ (1, "Chapter 2", [_make_ref(200), "/Fit"], None, None),+ ]++ bookmarks = self.parser._extract_bookmarks(mock_pdf)+ assert [b["page_num"] for b in bookmarks] == [1, 2]+def test_extract_bookmarks_no_outlines(self):"""Returns empty list when PDF has no outlines."""mock_pdf = MagicMock()@@ -108,12 +143,7 @@ class TestExtractBookmarks:def test_extract_bookmarks_integer_page_index(self):"""Bookmarks with integer destination (0-based) are resolved correctly."""mock_pdf = MagicMock()-- mock_page1 = MagicMock()- mock_page1.page_obj.objid = 100- mock_page2 = MagicMock()- mock_page2.page_obj.objid = 200- mock_pdf.pages = [mock_page1, mock_page2]+ mock_pdf.pages = [_make_page(pageid=100), _make_page(pageid=200)]# Integer page indices instead of object referencesmock_pdf.doc.get_outlines.return_value = [@@ -131,10 +161,7 @@ class TestExtractBookmarks:def test_extract_bookmarks_integer_page_index_out_of_range(self):"""Out-of-range integer page indices are treated as unresolved."""mock_pdf = MagicMock()-- mock_page1 = MagicMock()- mock_page1.page_obj.objid = 100- mock_pdf.pages = [mock_page1] # Only 1 page+ mock_pdf.pages = [_make_page(pageid=100)] # Only 1 pagemock_pdf.doc.get_outlines.return_value = [(1, "Valid", [0, "/Fit"], None, None),@@ -156,3 +183,67 @@ class TestExtractBookmarks:bookmarks = self.parser._extract_bookmarks(mock_pdf)assert bookmarks == []+++class TestConvertLocalBookmarks:+ """Test bookmark injection behavior in local PDF conversion."""++ @pytest.mark.asyncio+ async def test_convert_local_skips_unresolved_bookmarks(self):+ parser = PDFParser()+ fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")])+ fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf))++ with (+ patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber),+ patch.object(+ parser,+ "_extract_bookmarks",+ return_value=[+ {"level": 1, "title": "Broken Bookmark", "page_num": None},+ {"level": 1, "title": "Chapter 2", "page_num": 2},+ ],+ ),+ ):+ markdown, meta = await parser._convert_local(+ "dummy.pdf", storage=MagicMock(), resource_name="dummy"+ )++ assert "Broken Bookmark" not in markdown+ assert "\n# Chapter 2\n" in markdown+ assert meta["bookmarks_found"] == 2+ assert meta["bookmarks_resolved"] == 1+ assert meta["bookmarks_unresolved"] == 1+ assert meta["headings_found"] == 1+ assert meta["heading_source"] == "bookmarks"++ @pytest.mark.asyncio+ async def test_convert_local_falls_back_to_font_when_bookmarks_unresolved(self):+ parser = PDFParser()+ fake_pdf = SimpleNamespace(pages=[_FakePage("Page one"), _FakePage("Page two")])+ fake_pdfplumber = SimpleNamespace(open=lambda _path: nullcontext(fake_pdf))++ with (+ patch("openviking.parse.parsers.pdf.lazy_import", return_value=fake_pdfplumber),+ patch.object(+ parser,+ "_extract_bookmarks",+ return_value=[{"level": 1, "title": "Broken Bookmark", "page_num": None}],+ ),+ patch.object(+ parser,+ "_detect_headings_by_font",+ return_value=[{"level": 1, "title": "Font Heading", "page_num": 2}],+ ),+ ):+ markdown, meta = await parser._convert_local(+ "dummy.pdf", storage=MagicMock(), resource_name="dummy"+ )++ assert "Broken Bookmark" not in markdown+ assert "\n# Font Heading\n" in markdown+ assert meta["bookmarks_found"] == 1+ assert meta["bookmarks_resolved"] == 0+ assert meta["bookmarks_unresolved"] == 1+ assert meta["headings_found"] == 1+ assert meta["heading_source"] == "font_analysis"