Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(parsing): add opt-in normalization for tiny scanned PDF pages

ZyPulse1220a8e55238
diff --git a/deeptutor/api/routers/settings.py b/deeptutor/api/routers/settings.py
index 41fd591c5..cbad31045 100644
--- a/deeptutor/api/routers/settings.py
+++ b/deeptutor/api/routers/settings.py
@@ -1242,6 +1242,7 @@ async def update_mineru_settings(payload: MinerUSettingsUpdate):
"enable_formula": payload.enable_formula,
"enable_table": payload.enable_table,
"is_ocr": payload.is_ocr,
+ "normalize_tiny_scans": current.get("normalize_tiny_scans", False),
"allow_local_model_download": payload.allow_local_model_download,
}
)
diff --git a/deeptutor/services/config/runtime_settings.py b/deeptutor/services/config/runtime_settings.py
index 03313f63e..237cdaa32 100644
--- a/deeptutor/services/config/runtime_settings.py
+++ b/deeptutor/services/config/runtime_settings.py
@@ -185,6 +185,7 @@ _DEFAULT_MINERU_ENGINE: dict[str, Any] = {
"enable_formula": True,
"enable_table": True,
"is_ocr": False,
+ "normalize_tiny_scans": False,
"allow_local_model_download": False,
}
@@ -1154,6 +1155,7 @@ class RuntimeSettingsService:
"enable_formula": _coerce_bool(settings.get("enable_formula"), True),
"enable_table": _coerce_bool(settings.get("enable_table"), True),
"is_ocr": _coerce_bool(settings.get("is_ocr"), False),
+ "normalize_tiny_scans": _coerce_bool(settings.get("normalize_tiny_scans"), False),
"allow_local_model_download": _coerce_bool(
settings.get("allow_local_model_download"), False
),
diff --git a/deeptutor/services/parsing/engines/mineru/backend.py b/deeptutor/services/parsing/engines/mineru/backend.py
index e262106c6..fd7ab9673 100644
--- a/deeptutor/services/parsing/engines/mineru/backend.py
+++ b/deeptutor/services/parsing/engines/mineru/backend.py
@@ -57,9 +57,11 @@ def parse_document_to_workdir(
if cfg.is_cloud:
from .cloud import parse_cloud
+ from .normalization import working_copy
logger.info("Parsing %s via MinerU cloud API", source_path.name)
- return parse_cloud(source_path, output_base, cfg, on_progress=on_output)
+ with working_copy(source_path, output_base, cfg, on_output) as upload_path:
+ return parse_cloud(upload_path, output_base, cfg, on_progress=on_output)
return _parse_local(source_path, output_base, config=cfg, on_output=on_output)
diff --git a/deeptutor/services/parsing/engines/mineru/config.py b/deeptutor/services/parsing/engines/mineru/config.py
index ac69a2ba3..399be753e 100644
--- a/deeptutor/services/parsing/engines/mineru/config.py
+++ b/deeptutor/services/parsing/engines/mineru/config.py
@@ -49,6 +49,8 @@ class MinerUConfig:
enable_formula: bool = True
enable_table: bool = True
is_ocr: bool = False
+ # Cloud-only opt-in; preserves all parser choices and the original PDF.
+ normalize_tiny_scans: bool = False
# When False (default), a local parse fails fast instead of letting the
# MinerU CLI silently download multi-GB model weights on first run. The user
# opts in explicitly (Settings → Document Parsing) or via the one-click
@@ -94,6 +96,7 @@ def resolve_mineru_config() -> MinerUConfig:
enable_formula=bool(settings.get("enable_formula", True)),
enable_table=bool(settings.get("enable_table", True)),
is_ocr=bool(settings.get("is_ocr", False)),
+ normalize_tiny_scans=bool(settings.get("normalize_tiny_scans", False)),
allow_local_model_download=bool(settings.get("allow_local_model_download", False)),
)
diff --git a/deeptutor/services/parsing/engines/mineru/engine.py b/deeptutor/services/parsing/engines/mineru/engine.py
index 02cc579a3..624618453 100644
--- a/deeptutor/services/parsing/engines/mineru/engine.py
+++ b/deeptutor/services/parsing/engines/mineru/engine.py
@@ -37,6 +37,8 @@ class MinerUParser:
return MINERU_SUPPORTED_FORMATS
def signature(self, config: MinerUConfig) -> ParserSignature:
+ from .normalization import POLICY
+
version = f"cloud:{config.api_base_url}" if config.is_cloud else package_version("mineru")
return ParserSignature.build(
"mineru",
@@ -48,6 +50,11 @@ class MinerUParser:
"enable_formula": config.enable_formula,
"enable_table": config.enable_table,
"is_ocr": config.is_ocr,
+ **(
+ {"normalization": POLICY}
+ if config.is_cloud and config.normalize_tiny_scans
+ else {}
+ ),
},
)
diff --git a/deeptutor/services/parsing/engines/mineru/normalization.py b/deeptutor/services/parsing/engines/mineru/normalization.py
new file mode 100644
index 000000000..3e62bec6e
--- /dev/null
+++ b/deeptutor/services/parsing/engines/mineru/normalization.py
@@ -0,0 +1,134 @@
+"""Opt-in working copies for abnormally small, high-density scanned pages.
+
+The source is never rewritten. This preserves compressed image/soft-mask
+streams, not arbitrary PDF signatures or interactive-document semantics.
+Rotated, cropped, annotated, vector, and non-default UserUnit pages are skipped.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterator
+from contextlib import contextmanager
+import hashlib
+import logging
+from pathlib import Path
+import tempfile
+from urllib.parse import urlsplit
+
+from .config import MinerUConfig, MinerUError
+
+POLICY = "tiny-scan-v1"
+logger = logging.getLogger(__name__)
+
+
+def _plan(source: Path) -> list[tuple[int, float]]:
+ try:
+ import pymupdf
+ except ImportError as exc:
+ raise MinerUError(
+ "Tiny-scan normalization requires PyMuPDF. Install the "
+ "deeptutor[parse-pymupdf4llm] extra or disable normalize_tiny_scans."
+ ) from exc
+
+ changes: list[tuple[int, float]] = []
+ with pymupdf.open(source) as doc:
+ if doc.is_encrypted:
+ return changes
+ for index, page in enumerate(doc):
+ rect = page.rect
+ if (
+ page.rotation
+ or page.cropbox != page.mediabox
+ or rect.x0 != 0
+ or rect.y0 != 0
+ or not 0 < min(rect.width, rect.height)
+ or max(rect.width, rect.height) >= 144
+ or not 0.4 <= rect.width / rect.height <= 2.5
+ or page.get_text().strip()
+ or page.get_drawings()
+ or page.first_annot is not None
+ or page.first_widget is not None
+ or page.get_links()
+ ):
+ continue
+ unit = doc.xref_get_key(page.xref, "UserUnit")
+ if unit[0] != "null" and float(unit[1]) != 1:
+ continue
+ for image in page.get_image_info(xrefs=True):
+ box = pymupdf.Rect(image["bbox"])
+ if box.is_empty or image.get("xref", 0) == 0:
+ continue
+ coverage = (box & rect).get_area() / rect.get_area()
+ density = min(image["width"], image["height"]) * 72 / max(box.width, box.height)
+ if (
+ min(image["width"], image["height"]) >= 1000
+ and coverage >= 0.8
+ and density >= 1200
+ ):
+ changes.append((index, 768 / max(rect.width, rect.height)))
+ break
+ return changes
+
+
+def _image_streams(path: Path) -> list[list[str]]:
+ import pymupdf
+
+ with pymupdf.open(path) as doc:
+ pages = []
+ for page in doc:
+ streams = []
+ for item in page.get_images(full=True):
+ for xref in item[:2]:
+ if xref:
+ streams.append(hashlib.sha256(doc.xref_stream_raw(xref)).hexdigest())
+ pages.append(sorted(streams))
+ return pages
+
+
+@contextmanager
+def working_copy(
+ source: Path,
+ output: Path,
+ config: MinerUConfig,
+ on_progress: Callable[[str], None] | None = None,
+) -> Iterator[Path]:
+ url = urlsplit(config.api_base_url)
+ if (
+ not config.normalize_tiny_scans
+ or not config.is_cloud
+ or source.suffix.lower() != ".pdf"
+ or url.scheme != "https"
+ or url.hostname not in {"mineru.net", "www.mineru.net"}
+ ):
+ yield source
+ return
+ changes = _plan(source)
+ if not changes:
+ yield source
+ return
+
+ from pypdf import PdfReader, PdfWriter
+
+ output.parent.mkdir(parents=True, exist_ok=True)
+ with tempfile.TemporaryDirectory(prefix=".mineru-normalized-", dir=output.parent) as tmp:
+ target = Path(tmp) / source.name
+ before = _image_streams(source)
+ with PdfReader(source) as reader:
+ writer = PdfWriter()
+ try:
+ writer.clone_document_from_reader(reader)
+ for index, factor in changes:
+ writer.pages[index].scale_by(factor)
+ writer.write(target)
+ finally:
+ writer.close()
+ if _image_streams(target) != before:
+ raise MinerUError("PDF normalization changed image streams; refusing upload")
+ if on_progress:
+ try:
+ on_progress(
+ f"MinerU: normalized {len(changes)} tiny scanned page(s) in a temporary copy"
+ )
+ except Exception:
+ logger.debug("Normalization progress callback failed", exc_info=True)
+ yield target
diff --git a/docs-for-user/SOURCE_VISUALS.md b/docs-for-user/SOURCE_VISUALS.md
index 674b8d1fc..e27a10fbc 100644
--- a/docs-for-user/SOURCE_VISUALS.md
+++ b/docs-for-user/SOURCE_VISUALS.md
@@ -26,3 +26,33 @@ source image assets. Images that are only vector drawing commands, or pages
that require OCR when no usable OCR engine is configured, may be absent.
Other RAG providers do not yet index this visual manifest. Interactive
exercises, visual annotations, and mastery updates are separate future work.
+
+## Tiny scanned PDF pages with MinerU
+
+Some scanned PDFs encode a full-resolution page in an unusually small physical
+page box. MinerU's official cloud backend can optionally enlarge these pages in
+a temporary upload copy. The original file, page order, compressed image bytes,
+and soft masks are preserved; language, OCR, model, formula, and table settings
+remain as configured. This is a geometry workaround, not a guarantee of better
+OCR or preservation of arbitrary interactive PDF semantics.
+
+The option defaults to off. An administrator can enable it through the existing
+`PUT /api/settings/document-parsing` endpoint with this partial payload:
+
+```json
+{"engines": {"mineru": {"normalize_tiny_scans": true}}}
+```
+
+Use `false` to disable it. The option also lives at
+`engines.mineru.normalize_tiny_scans` in `document_parsing.json`. Enabling it
+changes the cloud parse-cache signature; old parses remain intact. Existing
+indexes are not rebuilt automatically. The optional
+`deeptutor[parse-pymupdf4llm]` extra provides the required PyMuPDF dependency.
+
+Only text-free pages below 144 points on their longest side, with a
+high-resolution image covering at least 80% of the page at an apparent density
+of at least 1200 DPI, qualify. The longest side is scaled to 768 points.
+Rotated, cropped, annotated, vector-bearing, or non-default UserUnit pages are
+left unchanged. Local MinerU, custom cloud endpoints, normal PDFs, and non-PDF
+inputs keep their existing behavior. Temporary copies are removed after success
+or failure, and upload is refused if compressed image streams change.
diff --git a/tests/services/parsing/test_mineru_normalization.py b/tests/services/parsing/test_mineru_normalization.py
new file mode 100644
index 000000000..b98ac4c11
--- /dev/null
+++ b/tests/services/parsing/test_mineru_normalization.py
@@ -0,0 +1,178 @@
+"""Synthetic PDFs only; no cloud transport or production settings."""
+
+from dataclasses import replace
+import io
+
+from PIL import Image
+import pytest
+
+pymupdf = pytest.importorskip("pymupdf")
+
+from deeptutor.services.parsing.engines.mineru import normalization
+from deeptutor.services.parsing.engines.mineru.config import MinerUConfig, MinerUError
+from deeptutor.services.parsing.engines.mineru.engine import MinerUParser
+
+CFG = MinerUConfig(mode="cloud", normalize_tiny_scans=True, language="en", model_version="vlm")
+
+
+def scan(path, *, kind="tiny", alpha=False):
+ data = io.BytesIO()
+ image = Image.new(
+ "RGBA" if alpha else "RGB", (1920, 2992), (255, 255, 255, 128) if alpha else "white"
+ )
+ image.save(data, format="PNG" if alpha else "JPEG")
+ with pymupdf.open() as doc:
+ for mode in ["tiny", "normal", "rotated"] if kind == "mixed" else [kind]:
+ page = doc.new_page(
+ width=492 if mode == "normal" else 41, height=768 if mode == "normal" else 64
+ )
+ page.insert_image(page.rect, stream=data.getvalue())
+ if mode == "rotated":
+ page.set_rotation(90)
+ elif mode == "cropped":
+ page.set_cropbox(pymupdf.Rect(1, 1, 40, 63))
+ elif mode == "unit":
+ doc.xref_set_key(page.xref, "UserUnit", "2")
+ elif mode == "text":
+ page.insert_text((2, 10), "text", fontsize=5)
+ elif mode == "vector":
+ page.draw_line((1, 1), (20, 20))
+ elif mode == "annotated":
+ page.add_text_annot((10, 10), "note")
+ doc.save(path)
+ return path
+
+
[email protected]("alpha", [False, True])
+def test_normalizes_copy_preserves_streams_source_and_parser_choices(tmp_path, alpha):
+ source = scan(tmp_path / "scan.pdf", alpha=alpha)
+ before = source.read_bytes()
+ messages = []
+ with normalization.working_copy(source, tmp_path / "out", CFG, messages.append) as copy:
+ assert copy != source and copy.name == source.name
+ assert normalization._image_streams(copy) == normalization._image_streams(source)
+ with pymupdf.open(copy) as doc:
+ assert doc[0].rect == pymupdf.Rect(0, 0, 492, 768)
+ assert (CFG.language, CFG.model_version, CFG.is_ocr) == ("en", "vlm", False)
+ assert source.read_bytes() == before
+ assert not copy.exists()
+ assert len(messages) == 1
+
+
+ "kind", ["normal", "rotated", "cropped", "unit", "text", "vector", "annotated"]
+)
+def test_conservative_passthrough(tmp_path, kind):
+ source = scan(tmp_path / "scan.pdf", kind=kind)
+ before = source.read_bytes()
+ with normalization.working_copy(source, tmp_path / "out", CFG) as copy:
+ assert copy == source
+ assert source.read_bytes() == before
+
+
+ "config",
+ [
+ MinerUConfig(),
+ replace(CFG, normalize_tiny_scans=False),
+ replace(CFG, mode="local"),
+ replace(CFG, api_base_url="https://example.test"),
+ ],
+)
+def test_disabled_or_out_of_scope_does_not_inspect_pdf(tmp_path, config, monkeypatch):
+ def fail(_):
+ pytest.fail("must not inspect a disabled/out-of-scope input")
+
+ monkeypatch.setattr(normalization, "_plan", fail)
+ source = tmp_path / "scan.pdf"
+ with normalization.working_copy(source, tmp_path / "out", config) as copy:
+ assert copy == source
+
+
+def test_mixed_pages_preserve_order_and_skip_complex_geometry(tmp_path):
+ source = scan(tmp_path / "mixed.pdf", kind="mixed")
+ with normalization.working_copy(source, tmp_path / "out", CFG) as copy:
+ with pymupdf.open(copy) as doc:
+ assert len(doc) == 3
+ assert doc[0].rect == doc[1].rect == pymupdf.Rect(0, 0, 492, 768)
+ assert doc[2].rotation == 90
+ assert doc[2].rect == pymupdf.Rect(0, 0, 64, 41)
+
+
+def test_cleanup_after_cloud_failure(tmp_path):
+ source = scan(tmp_path / "scan.pdf")
+ with (
+ pytest.raises(RuntimeError),
+ normalization.working_copy(source, tmp_path / "out", CFG) as copy,
+ ):
+ raise RuntimeError("cloud failed")
+ assert not copy.exists()
+ assert not list(tmp_path.glob(".mineru-normalized-*"))
+
+
+def test_rejects_changed_streams(tmp_path, monkeypatch):
+ source = scan(tmp_path / "scan.pdf")
+ monkeypatch.setattr(normalization, "_image_streams", lambda path: [[str(path)]])
+ with (
+ pytest.raises(MinerUError, match="changed image streams"),
+ normalization.working_copy(source, tmp_path / "out", CFG),
+ ):
+ pytest.fail("must not upload a modified image stream")
+ assert not list(tmp_path.glob(".mineru-normalized-*"))
+
+
+def test_signature_is_opt_in_and_invalidates_previous_cloud_cache():
+ parser = MinerUParser()
+ disabled = replace(CFG, normalize_tiny_scans=False)
+ assert parser.signature(CFG).hash() != parser.signature(disabled).hash()
+ assert "normalization" not in dict(parser.signature(disabled).params)
+ assert parser.signature(replace(CFG, mode="local")) == parser.signature(
+ replace(disabled, mode="local")
+ )
+
+
+def test_backend_uploads_copy_and_keeps_configuration(tmp_path, monkeypatch):
+ from deeptutor.services.parsing.engines.mineru import backend, cloud
+
+ source = scan(tmp_path / "scan.pdf")
+ calls = []
+
+ def parse(path, output, config, **kwargs):
+ calls.append(path)
+ assert path != source and path.name == source.name
+ assert config is CFG
+ with pymupdf.open(path) as doc:
+ assert doc[0].rect.height == 768
+ return output / path.stem
+
+ monkeypatch.setattr(cloud, "parse_cloud", parse)
+ assert (
+ backend.parse_document_to_workdir(source, tmp_path / "out", config=CFG)
+ == tmp_path / "out" / "scan"
+ )
+ assert len(calls) == 1 and not calls[0].exists()
+
+
+async def test_settings_enable_disable_and_legacy_save_preserves_opt_in(tmp_path, monkeypatch):
+ from deeptutor.api.routers import settings
+ from deeptutor.services.config.runtime_settings import RuntimeSettingsService
+ from deeptutor.services.parsing.engines.mineru import config
+
+ service = RuntimeSettingsService(tmp_path / "settings", process_env={})
+ monkeypatch.setattr(settings, "_require_settings_admin", lambda: None)
+ monkeypatch.setattr(settings, "get_runtime_settings_service", lambda: service)
+ monkeypatch.setattr(settings, "_document_parsing_payload", service.load_document_parsing)
+ monkeypatch.setattr(settings, "_mineru_settings_payload", service.load_mineru)
+ monkeypatch.setattr(config, "load_mineru_settings", service.load_mineru)
+ assert not config.resolve_mineru_config().normalize_tiny_scans
+ await settings.update_document_parsing_settings(
+ settings.DocumentParsingUpdate(engines={"mineru": {"normalize_tiny_scans": True}})
+ )
+ assert config.resolve_mineru_config().normalize_tiny_scans
+ await settings.update_mineru_settings(settings.MinerUSettingsUpdate(language="en"))
+ assert config.resolve_mineru_config().normalize_tiny_scans
+ await settings.update_document_parsing_settings(
+ settings.DocumentParsingUpdate(engines={"mineru": {"normalize_tiny_scans": False}})
+ )
+ assert not config.resolve_mineru_config().normalize_tiny_scans