Skip to content

Turn any technical book PDF into a Claude Code skill — ready to study, reference, and use while you work.

feat(pdf): add pdf-inspector smart routing and provenance (#203)

Virgilio Junior67f52fdf1fa4
diff --git a/book_to_skill/cli.py b/book_to_skill/cli.py
index 439bb8d..83d0599 100644
--- a/book_to_skill/cli.py
+++ b/book_to_skill/cli.py
@@ -1,5 +1,10 @@
import sys
from book_to_skill.utils import main as utils_main
+from book_to_skill.pdf_inspector_integration import (
+ enrich_pdf_inspector_metadata,
+ install_pdf_inspector_hook,
+)
+
def main():
# Force UTF-8 stdout/stderr to avoid UnicodeEncodeError on Windows console
@@ -9,7 +14,13 @@ def main():
except (AttributeError, ValueError):
# Ignore if the stream does not support reconfigure (e.g. mock streams during testing)
pass
+
+ # pdf-inspector is an optional accelerator/trust layer. When unavailable,
+ # this hook is a no-op and the legacy extraction chain behaves unchanged.
+ install_pdf_inspector_hook()
utils_main()
+ enrich_pdf_inspector_metadata()
+
# Expose main for packaging console scripts entry points
if __name__ == "__main__":
diff --git a/book_to_skill/config.py b/book_to_skill/config.py
index b33130f..c4a27df 100644
--- a/book_to_skill/config.py
+++ b/book_to_skill/config.py
@@ -45,6 +45,7 @@ SUPPORTED_EXTENSIONS = {
}
PYTHON_DEPENDENCIES = {
+ "pdf_inspector": "pdf-inspector>=1.15,<2",
"docling": "docling",
"pypdf": "pypdf",
"pdfminer": "pdfminer.six",
diff --git a/book_to_skill/dependencies.py b/book_to_skill/dependencies.py
index a4c9bd4..9166eb6 100644
--- a/book_to_skill/dependencies.py
+++ b/book_to_skill/dependencies.py
@@ -14,6 +14,13 @@ from book_to_skill.config import PYTHON_DEPENDENCIES, HTML_EXTENSIONS
# format and what it needs. `modules` are optional Python packages (any one is
# enough unless noted); `system` are external commands resolved via PATH.
DEPENDENCY_GROUPS = [
+ {
+ "label": "PDF (smart inspection / native Markdown)",
+ "modules": ["pdf_inspector"],
+ "any_of_modules": True,
+ "system": [],
+ "note": "optional fast classifier/provenance layer; falls back to the existing PDF chain",
+ },
{
"label": "PDF (text-heavy)",
"modules": ["pypdf", "pdfminer"],
diff --git a/book_to_skill/pdf_inspector_integration.py b/book_to_skill/pdf_inspector_integration.py
new file mode 100644
index 0000000..d784955
--- /dev/null
+++ b/book_to_skill/pdf_inspector_integration.py
@@ -0,0 +1,260 @@
+from __future__ import annotations
+
+import importlib.metadata
+import json
+import os
+import sys
+from pathlib import Path
+from typing import Any
+
+_MIN_NATIVE_CONFIDENCE = 0.90
+_INSPECTIONS: dict[str, dict[str, Any]] = {}
+
+
+def _normalise_pdf_type(value: Any) -> str:
+ text = str(value or "").strip()
+ if not text:
+ return "unknown"
+ # Handles both Python binding strings ("text_based") and enum-ish values
+ # such as "PdfType.TextBased" without depending on one library revision.
+ text = text.split(".")[-1]
+ out = []
+ for index, char in enumerate(text):
+ if index and char.isupper() and text[index - 1].islower():
+ out.append("_")
+ out.append(char.lower())
+ return "".join(out).replace("-", "_")
+
+
+def _package_version() -> str | None:
+ try:
+ return importlib.metadata.version("pdf-inspector")
+ except importlib.metadata.PackageNotFoundError:
+ return None
+
+
+def _ocr_reasons(entries: Any) -> list[dict[str, Any]]:
+ normalised: list[dict[str, Any]] = []
+ for entry in entries or []:
+ page = getattr(entry, "page", None)
+ reasons = list(getattr(entry, "reasons", None) or [])
+ normalised.append({"page": page, "reasons": reasons})
+ return normalised
+
+
+def inspect_pdf(path: str | Path) -> tuple[str | None, dict[str, Any] | None]:
+ """Inspect a PDF with Firecrawl pdf-inspector when it is installed.
+
+ The returned Markdown is intentionally conservative: it is only considered
+ usable when pdf-inspector classifies the document as native text, reports no
+ OCR-routed pages or encoding problems, and gives a high confidence score.
+ All other cases return metadata only so the existing Book-to-Skill fallback
+ chain remains authoritative.
+ """
+ try:
+ import pdf_inspector
+ except ImportError:
+ return None, None
+
+ try:
+ result = pdf_inspector.process_pdf(str(path))
+ except Exception as exc:
+ print(
+ f" [warn] pdf-inspector preflight failed: {type(exc).__name__}: {exc}",
+ file=sys.stderr,
+ )
+ return None, None
+
+ pdf_type = _normalise_pdf_type(getattr(result, "pdf_type", None))
+ confidence = float(getattr(result, "confidence", 0.0) or 0.0)
+ pages_needing_ocr = list(getattr(result, "pages_needing_ocr", None) or [])
+ has_encoding_issues = bool(getattr(result, "has_encoding_issues", False))
+ markdown = getattr(result, "markdown", None)
+
+ native_markdown_trusted = bool(
+ isinstance(markdown, str)
+ and markdown.strip()
+ and pdf_type == "text_based"
+ and confidence >= _MIN_NATIVE_CONFIDENCE
+ and not pages_needing_ocr
+ and not has_encoding_issues
+ )
+
+ metadata: dict[str, Any] = {
+ "engine": "pdf-inspector",
+ "version": _package_version(),
+ "pdf_type": pdf_type,
+ "confidence": round(confidence, 4),
+ "native_markdown_trusted": native_markdown_trusted,
+ "pages_needing_ocr": pages_needing_ocr,
+ "ocr_reasons_by_page": _ocr_reasons(
+ getattr(result, "ocr_reasons_by_page", None)
+ ),
+ "pages_with_tables": list(getattr(result, "pages_with_tables", None) or []),
+ "pages_with_columns": list(getattr(result, "pages_with_columns", None) or []),
+ "has_encoding_issues": has_encoding_issues,
+ "is_complex_layout": bool(getattr(result, "is_complex_layout", False)),
+ "page_count": int(getattr(result, "page_count", 0) or 0),
+ }
+ return (markdown if native_markdown_trusted else None), metadata
+
+
+def _looks_like_pdf(path: Path) -> bool:
+ if path.suffix.lower() == ".pdf":
+ return True
+ try:
+ with path.open("rb") as handle:
+ return handle.read(4) == b"%PDF"
+ except OSError:
+ return False
+
+
+def _result_from_inspector(
+ utils_module: Any,
+ input_path: Path,
+ markdown: str,
+ inspection: dict[str, Any],
+) -> dict[str, Any] | None:
+ text, removed_invisible = utils_module.sanitize_extracted_text(markdown)
+ if removed_invisible:
+ print(
+ f" [security] removed {removed_invisible} invisible Unicode "
+ f"code point(s) from {input_path.name}",
+ file=sys.stderr,
+ )
+ if not text.strip():
+ return None
+
+ confidence = inspection.get("confidence", 0.0)
+ print(
+ f"Mode: text — using pdf-inspector "
+ f"(native text, confidence {confidence:.2f})... OK"
+ )
+
+ structure = utils_module.detect_structure(text)
+ print(
+ f" chapters: {structure['chapters_detected']} "
+ f"({structure['chapters_method']})"
+ )
+
+ pages = inspection.get("page_count") or utils_module.count_pages(str(input_path))
+ tokens = utils_module.estimate_tokens(text)
+ file_size_mb = os.path.getsize(input_path) / (1024 * 1024)
+
+ return {
+ "source_file": str(input_path.resolve()),
+ "filename": input_path.name,
+ "format": "pdf",
+ "extraction_method": "pdf-inspector",
+ "file_size_mb": round(file_size_mb, 2),
+ "pages": pages,
+ "pages_label": "pages",
+ "chars": len(text),
+ "words": len(text.split()),
+ "estimated_tokens": tokens,
+ "images_dropped": None,
+ "text": text,
+ **structure,
+ }
+
+
+def _fallback_reason(inspection: dict[str, Any]) -> str:
+ if inspection.get("pdf_type") != "text_based":
+ return f"classified as {inspection.get('pdf_type', 'unknown')}"
+ if inspection.get("has_encoding_issues"):
+ return "encoding issues detected"
+ if inspection.get("pages_needing_ocr"):
+ pages = ", ".join(str(p) for p in inspection["pages_needing_ocr"][:8])
+ suffix = "..." if len(inspection["pages_needing_ocr"]) > 8 else ""
+ return f"OCR recommended for page(s) {pages}{suffix}"
+ if float(inspection.get("confidence", 0.0) or 0.0) < _MIN_NATIVE_CONFIDENCE:
+ return f"confidence below {_MIN_NATIVE_CONFIDENCE:.2f}"
+ return "native Markdown was not trustworthy"
+
+
+def install_pdf_inspector_hook(utils_module: Any | None = None) -> None:
+ """Wrap ``extract_single_file`` without changing the legacy extractor.
+
+ Text-mode PDFs can take the fast native Markdown path when pdf-inspector says
+ it is safe. Technical PDFs and uncertain documents continue through the
+ existing Docling/pdftotext/pypdf/pdfminer implementation unchanged.
+ """
+ if utils_module is None:
+ from book_to_skill import utils as utils_module
+
+ original = utils_module.extract_single_file
+ if getattr(original, "_book_to_skill_pdf_inspector_hook", False):
+ return
+
+ def wrapped(input_path: Path, extraction_mode: str, install_mode: str) -> dict[str, Any]:
+ if not _looks_like_pdf(input_path):
+ return original(input_path, extraction_mode, install_mode)
+
+ markdown, inspection = inspect_pdf(input_path)
+ if inspection is not None:
+ _INSPECTIONS[str(input_path.resolve())] = inspection
+
+ if extraction_mode == "text" and markdown and inspection:
+ result = _result_from_inspector(utils_module, input_path, markdown, inspection)
+ if result is not None:
+ return result
+
+ if inspection is not None:
+ print(
+ f"pdf-inspector: {_fallback_reason(inspection)}; "
+ "using the existing extraction chain.",
+ file=sys.stderr,
+ )
+ return original(input_path, extraction_mode, install_mode)
+
+ wrapped._book_to_skill_pdf_inspector_hook = True # type: ignore[attr-defined]
+ wrapped._book_to_skill_original = original # type: ignore[attr-defined]
+ utils_module.extract_single_file = wrapped
+
+
+def enrich_pdf_inspector_metadata(metadata_path: str | Path | None = None) -> None:
+ """Persist inspection/provenance into the run's existing ``metadata.json``."""
+ if not _INSPECTIONS:
+ return
+
+ if metadata_path is None:
+ from book_to_skill.config import OUTPUT_META
+
+ metadata_path = OUTPUT_META
+
+ path = Path(metadata_path)
+ if not path.exists():
+ return
+
+ try:
+ payload = json.loads(path.read_text(encoding="utf-8"))
+ except (OSError, json.JSONDecodeError) as exc:
+ print(f" [warn] could not enrich PDF metadata: {exc}", file=sys.stderr)
+ return
+
+ matched = 0
+ for source in payload.get("sources", []):
+ source_file = source.get("source_file")
+ if not source_file:
+ continue
+ inspection = _INSPECTIONS.get(str(Path(source_file).resolve()))
+ if inspection is None:
+ continue
+ source["pdf_inspector"] = inspection
+ matched += 1
+
+ if matched == 1 and payload.get("total_sources") == 1:
+ only_source = payload.get("sources", [{}])[0]
+ if "pdf_inspector" in only_source:
+ payload["pdf_inspector"] = only_source["pdf_inspector"]
+
+ if not matched:
+ return
+
+ tmp = path.with_suffix(path.suffix + ".tmp")
+ tmp.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
+ tmp.replace(path)
+
+
+def _reset_state_for_tests() -> None:
+ _INSPECTIONS.clear()
diff --git a/pyproject.toml b/pyproject.toml
index 8fac3e9..9a72fa8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,7 +16,7 @@ book-to-skill = "book_to_skill.cli:main"
[project.optional-dependencies]
html = ["trafilatura"]
epub = ["ebooklib", "beautifulsoup4"]
-pdf = ["pypdf", "pdfminer.six"]
+pdf = ["pdf-inspector>=1.15,<2", "pypdf", "pdfminer.six"]
docx = ["python-docx"]
rtf = ["striprtf"]
technical = ["docling"]
@@ -24,6 +24,7 @@ all = [
"trafilatura",
"ebooklib",
"beautifulsoup4",
+ "pdf-inspector>=1.15,<2",
"pypdf",
"pdfminer.six",
"python-docx",
diff --git a/tests/test_pdf_inspector_integration.py b/tests/test_pdf_inspector_integration.py
new file mode 100644
index 0000000..e174b51
--- /dev/null
+++ b/tests/test_pdf_inspector_integration.py
@@ -0,0 +1,166 @@
+import json
+import sys
+from types import SimpleNamespace
+
+import book_to_skill.pdf_inspector_integration as integration
+
+
+def _fake_result(**overrides):
+ values = {
+ "pdf_type": "text_based",
+ "confidence": 0.99,
+ "markdown": "# Chapter 1\n\nUseful text",
+ "page_count": 12,
+ "pages_needing_ocr": [],
+ "ocr_reasons_by_page": [],
+ "pages_with_tables": [3],
+ "pages_with_columns": [4],
+ "has_encoding_issues": False,
+ "is_complex_layout": True,
+ }
+ values.update(overrides)
+ return SimpleNamespace(**values)
+
+
+def test_inspect_pdf_accepts_only_high_confidence_native_markdown(monkeypatch):
+ fake_module = SimpleNamespace(process_pdf=lambda _path: _fake_result())
+ monkeypatch.setitem(sys.modules, "pdf_inspector", fake_module)
+
+ markdown, metadata = integration.inspect_pdf("book.pdf")
+
+ assert markdown.startswith("# Chapter 1")
+ assert metadata["pdf_type"] == "text_based"
+ assert metadata["native_markdown_trusted"] is True
+ assert metadata["pages_with_tables"] == [3]
+ assert metadata["pages_with_columns"] == [4]
+
+
+def test_inspect_pdf_rejects_native_markdown_when_ocr_is_recommended(monkeypatch):
+ reason = SimpleNamespace(page=7, reasons=["suspected_garbled_text"])
+ fake_module = SimpleNamespace(
+ process_pdf=lambda _path: _fake_result(
+ pages_needing_ocr=[7],
+ ocr_reasons_by_page=[reason],
+ )
+ )
+ monkeypatch.setitem(sys.modules, "pdf_inspector", fake_module)
+
+ markdown, metadata = integration.inspect_pdf("book.pdf")
+
+ assert markdown is None
+ assert metadata["native_markdown_trusted"] is False
+ assert metadata["pages_needing_ocr"] == [7]
+ assert metadata["ocr_reasons_by_page"] == [
+ {"page": 7, "reasons": ["suspected_garbled_text"]}
+ ]
+
+
+def test_hook_uses_inspector_for_clean_text_pdf(tmp_path, monkeypatch):
+ integration._reset_state_for_tests()
+ pdf = tmp_path / "book.pdf"
+ pdf.write_bytes(b"%PDF-1.7\nfixture")
+
+ original_calls = []
+
+ def original(*args):
+ original_calls.append(args)
+ return {"extraction_method": "legacy"}
+
+ fake_utils = SimpleNamespace(
+ extract_single_file=original,
+ sanitize_extracted_text=lambda text: (text, 0),
+ detect_structure=lambda _text: {
+ "chapters_detected": 1,
+ "chapters_method": "numeric",
+ "chapter_headings_sample": ["Chapter 1"],
+ "has_toc": False,
+ },
+ count_pages=lambda _path: 12,
+ estimate_tokens=lambda _text: 42,
+ )
+ inspection = {
+ "confidence": 0.99,
+ "page_count": 12,
+ "pdf_type": "text_based",
+ "native_markdown_trusted": True,
+ "pages_needing_ocr": [],
+ "has_encoding_issues": False,
+ }
+ monkeypatch.setattr(
+ integration,
+ "inspect_pdf",
+ lambda _path: ("Chapter 1\nBody", inspection),
+ )
+
+ integration.install_pdf_inspector_hook(fake_utils)
+ result = fake_utils.extract_single_file(pdf, "text", "no")
+
+ assert result["extraction_method"] == "pdf-inspector"
+ assert result["pages"] == 12
+ assert original_calls == []
+ assert integration._INSPECTIONS[str(pdf.resolve())] == inspection
+
+
+def test_hook_keeps_technical_mode_on_existing_pipeline(tmp_path, monkeypatch):
+ integration._reset_state_for_tests()
+ pdf = tmp_path / "book.pdf"
+ pdf.write_bytes(b"%PDF-1.7\nfixture")
+
+ def original(*_args):
+ return {"extraction_method": "docling"}
+
+ fake_utils = SimpleNamespace(extract_single_file=original)
+ inspection = {
+ "confidence": 0.99,
+ "page_count": 12,
+ "pdf_type": "text_based",
+ "native_markdown_trusted": True,
+ "pages_needing_ocr": [],
+ "has_encoding_issues": False,
+ }
+ monkeypatch.setattr(
+ integration,
+ "inspect_pdf",
+ lambda _path: ("# Native Markdown", inspection),
+ )
+
+ integration.install_pdf_inspector_hook(fake_utils)
+ result = fake_utils.extract_single_file(pdf, "technical", "no")
+
+ assert result["extraction_method"] == "docling"
+
+
+def test_metadata_is_enriched_with_machine_readable_pdf_signals(tmp_path):
+ integration._reset_state_for_tests()
+ source = tmp_path / "book.pdf"
+ source.write_bytes(b"%PDF-1.7\nfixture")
+ metadata_path = tmp_path / "metadata.json"
+ metadata_path.write_text(
+ json.dumps(
+ {
+ "total_sources": 1,
+ "sources": [
+ {
+ "source_file": str(source.resolve()),
+ "filename": "book.pdf",
+ }
+ ],
+ }
+ ),
+ encoding="utf-8",
+ )
+ inspection = {
+ "engine": "pdf-inspector",
+ "pdf_type": "mixed",
+ "confidence": 0.87,
+ "native_markdown_trusted": False,
+ "pages_needing_ocr": [5],
+ "ocr_reasons_by_page": [{"page": 5, "reasons": ["no_text"]}],
+ }
+ integration._INSPECTIONS[str(source.resolve())] = inspection
+
+ integration.enrich_pdf_inspector_metadata(metadata_path)
+ payload = json.loads(metadata_path.read_text(encoding="utf-8"))
+
+ assert payload["sources"][0]["pdf_inspector"] == inspection
+ assert payload["pdf_inspector"] == inspection