Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

perf(documents): bound an inlined raster's long edge before admission

Fancie-Personale7c909ec94fe
diff --git a/deeptutor/reading/extract.py b/deeptutor/reading/extract.py
index 9a8364588..f9b1361ca 100644
--- a/deeptutor/reading/extract.py
+++ b/deeptutor/reading/extract.py
@@ -338,10 +338,11 @@ def _slides_text_and_images(source: Path) -> tuple[str, tuple[EmbeddedImage, ...
"""Slide text with image markers, preferring the rich extraction.
Falls back to the shared extractor (text only) when the rich path cannot
- read the deck, so ingest never fails on a picture quirk.
+ read the deck, so ingest never fails on a picture quirk. The reading budget
+ keeps slide pictures at source resolution, like the PDF ingest above.
"""
try:
- rich = extract_pptx_rich(source.read_bytes())
+ rich = extract_pptx_rich(source.read_bytes(), budget=reading_image_budget())
except Exception:
rich = None
if rich is not None and any(slide.strip() for slide in rich.slides):
@@ -368,12 +369,13 @@ def _extract_sections(source: Path) -> Extraction:
DOCX goes through the rich extraction first so embedded images survive
with their section mapping; every other format (and a failed rich parse)
- uses the shared text-only extractor.
+ uses the shared text-only extractor. The reading budget keeps those images
+ at source resolution for the reader pane, like the PDF ingest above.
"""
images: tuple[EmbeddedImage, ...] = ()
if source.suffix.lower() == ".docx":
try:
- rich = extract_docx_rich(source.read_bytes())
+ rich = extract_docx_rich(source.read_bytes(), budget=reading_image_budget())
except Exception:
rich = None
if rich is not None and any(paragraph.strip() for paragraph in rich.paragraphs):
diff --git a/deeptutor/utils/document_images.py b/deeptutor/utils/document_images.py
index e3c6240d0..2a10969d0 100644
--- a/deeptutor/utils/document_images.py
+++ b/deeptutor/utils/document_images.py
@@ -48,6 +48,32 @@ MAX_TOTAL_IMAGE_BYTES_PER_DOC = 12 * 1024 * 1024
MAX_IMAGE_BYTES = 4 * 1024 * 1024
MIN_IMAGE_BYTES = 1_200 # below this it is a bullet point or a divider, not a figure
+#: Long-edge bound for an image that will be base64-inlined into one provider
+#: request. Byte budgets alone do not bound what a *page-sized* raster costs:
+#: a scanned A4 page arrives at ~2900x4032 (~600 KB), so seven of them are
+#: ~4.2 MB of body and ~5.6 MB encoded, while a vision model is served a raster
+#: scaled toward ~1.5k on the long edge anyway. Bounding the long edge keeps the
+#: detail a model can actually read and drops the rest. The reading-area ingest
+#: is exempt (``max_inline_edge=0``) because those bytes are written to disk for
+#: the reader pane and never inflated into a request.
+#:
+#: The number is deliberately looser than ``PAGE_RENDER_DPI`` (110 dpi, ~1286 px
+#: on A4, ~132 KB/page), which the reading viewport already uses for its
+#: whole-page fallback raster: a harvested scan is the only channel carrying a
+#: page's content, so it keeps more resolution than a supplementary render.
+MAX_INLINE_IMAGE_EDGE = 1600
+
+#: Matches ``PAGE_RENDER_DPI``'s jpg_quality so a fitted raster is encoded the
+#: same way the reading render already is.
+INLINE_IMAGE_JPEG_QUALITY = 82
+
+#: Fitting decodes pixels, so refuse images past this many pixels before Pillow
+#: allocates for them. Mirrors ``MAX_IMAGE_PIXELS`` in
+#: ``services/rag/visual_assets`` (kept as a literal so this utils module stays
+#: free of a services import). Oversized candidates are left untouched and fall
+#: through to the existing byte budgets.
+MAX_INLINE_FIT_PIXELS = 30_000_000
+
def reading_image_budget() -> ImageBudget:
"""Looser per-document caps for the reading-area ingest path.
@@ -59,11 +85,15 @@ def reading_image_budget() -> ImageBudget:
request can carry. ``max_images_per_page=4`` mirrors
``READING_VIEWPORT_MAX_IMAGES`` in ``services/session/_turn_runtime_shared``
(kept as a literal so this utils module stays free of a services import).
+
+ ``max_inline_edge=0`` keeps source resolution: these bytes are stored for
+ the reader pane to display, not inlined into a request.
"""
return ImageBudget(
max_images=120,
max_total_bytes=60 * 1024 * 1024,
max_images_per_page=4,
+ max_inline_edge=0,
)
@@ -108,6 +138,10 @@ class ImageBudget:
# page from monopolising the document budget when a single request can only
# carry a handful of images.
max_images_per_page: int = 0
+ # 0 = keep the source resolution. Non-zero fits every admitted raster within
+ # this long-edge bound, for budgets whose images are inlined into one
+ # request. See ``MAX_INLINE_IMAGE_EDGE``.
+ max_inline_edge: int = MAX_INLINE_IMAGE_EDGE
count: int = 0
total_bytes: int = 0
skipped_vector: int = 0
@@ -122,7 +156,13 @@ class ImageBudget:
self.skipped_other += 1
def try_admit(self, ext: str, data: bytes) -> EmbeddedImage | None:
- """Admit one candidate image, applying every budget. ``None`` = skip."""
+ """Admit one candidate image, applying every budget. ``None`` = skip.
+
+ The candidate is fitted to ``max_inline_edge`` first, so the byte
+ budgets below weigh what will actually ship, and a page-sized raster no
+ provider would read at source size still qualifies.
+ """
+ ext, data = fit_inline_image(ext, data, max_edge=self.max_inline_edge)
if len(data) < self.min_image_bytes or len(data) > self.max_image_bytes:
self.skipped_budget += 1
return None
@@ -221,6 +261,64 @@ def _parse_xml(raw: bytes) -> Any:
return DefusedElementTree.fromstring(raw)
+def fit_inline_image(ext: str, data: bytes, *, max_edge: int) -> tuple[str, bytes]:
+ """Fit a raster within ``max_edge`` so one inline request stays affordable.
+
+ Returns the (possibly re-encoded) image and its extension. An image already
+ within the bound is returned untouched, byte for byte, so small figures and
+ screenshots keep their original format and quality — re-encoding is reserved
+ for images that would otherwise carry detail no vision model reads.
+
+ ``max_edge=0`` disables fitting. Every failure — an undecodable candidate, a
+ pixel count past :data:`MAX_INLINE_FIT_PIXELS`, a re-encode that would not
+ even shrink the payload — returns the original bytes: a transport-size
+ optimisation must never be the reason an attachment becomes unreadable.
+ """
+ if max_edge <= 0 or not data:
+ return ext, data
+ try:
+ from PIL import Image
+ except ImportError: # pragma: no cover - Pillow ships with the image extras
+ return ext, data
+ try:
+ with Image.open(io.BytesIO(data)) as probe:
+ width, height = probe.size
+ if max(width, height) <= max_edge or width * height > MAX_INLINE_FIT_PIXELS:
+ return ext, data
+ probe.load()
+ has_alpha = probe.mode in {"RGBA", "LA"} or (
+ probe.mode == "P" and "transparency" in probe.info
+ )
+ if has_alpha:
+ # JPEG has no alpha channel. Composite onto white — a page or a
+ # slide — rather than letting a transparent figure turn black.
+ rgba = probe.convert("RGBA")
+ fitted = Image.new("RGB", rgba.size, (255, 255, 255))
+ fitted.paste(rgba, mask=rgba.split()[-1])
+ rgba.close()
+ else:
+ fitted = probe.convert("RGB")
+ except Exception:
+ logger.debug("inline image fit probe failed", exc_info=True)
+ return ext, data
+ try:
+ scale = max_edge / max(width, height)
+ target = (max(1, round(width * scale)), max(1, round(height * scale)))
+ buffer = io.BytesIO()
+ fitted.resize(target, Image.LANCZOS).save(
+ buffer, format="JPEG", quality=INLINE_IMAGE_JPEG_QUALITY, optimize=True
+ )
+ shrunk = buffer.getvalue()
+ except Exception:
+ logger.debug("inline image fit failed", exc_info=True)
+ return ext, data
+ finally:
+ fitted.close()
+ if not shrunk or len(shrunk) >= len(data):
+ return ext, data
+ return ".jpg", shrunk
+
+
def _normalise_ext(ext: str) -> str:
ext = (ext or "").lower()
if ext == ".jpeg":
@@ -592,9 +690,11 @@ __all__ = [
"PptxRich",
"PdfImages",
"MAX_IMAGES_PER_DOC",
+ "MAX_INLINE_IMAGE_EDGE",
"SUPPORTED_IMAGE_EXTENSIONS",
"build_marker",
"find_markers",
+ "fit_inline_image",
"image_index_from_name",
"reading_image_budget",
"extract_docx_rich",
diff --git a/tests/reading/test_engine.py b/tests/reading/test_engine.py
index 666dd0167..32206b876 100644
--- a/tests/reading/test_engine.py
+++ b/tests/reading/test_engine.py
@@ -37,6 +37,7 @@ from deeptutor.reading.extract import (
)
from deeptutor.reading.models import parse_text_selectors
from deeptutor.reading.search import normalise, search_units, terms_of
+from deeptutor.utils.document_images import MAX_INLINE_IMAGE_EDGE
pymupdf = pytest.importorskip("pymupdf")
@@ -337,6 +338,55 @@ def test_pptx_slides_become_units_when_the_extractor_marks_them(tmp_path: Path)
assert "First slide body" in extraction.units[0]
+def _large_noise_jpeg(size: tuple[int, int] = (2000, 1200)) -> bytes:
+ """A noisy JPEG past the inline long-edge bound, well inside byte budgets."""
+ import random
+
+ from PIL import Image as PILImage
+
+ raw = random.Random(9).randbytes(size[0] * size[1] * 3)
+ buf = io.BytesIO()
+ PILImage.frombytes("RGB", size, raw).save(buf, format="JPEG", quality=95)
+ return buf.getvalue()
+
+
[email protected]("suffix", ["docx", "pptx"])
+def test_office_reading_media_keeps_source_resolution(tmp_path: Path, suffix: str) -> None:
+ """Reader-pane media is stored whole; the inline transport bound is for chat.
+
+ Both Office ingests used the chat-attachment budget by default, so a bound
+ meant for images that are base64-inlined into one request would have shrunk
+ what the reader pane displays.
+ """
+ from PIL import Image as PILImage
+
+ image = _large_noise_jpeg()
+ path = tmp_path / f"material.{suffix}"
+ if suffix == "docx":
+ from docx import Document as DocxDocument
+
+ doc = DocxDocument()
+ doc.add_paragraph("A picture follows.")
+ doc.add_picture(io.BytesIO(image))
+ doc.save(str(path))
+ else:
+ from pptx import Presentation
+ from pptx.util import Inches
+
+ prs = Presentation()
+ slide = prs.slides.add_slide(prs.slide_layouts[5])
+ slide.shapes.add_picture(io.BytesIO(image), Inches(1), Inches(1), Inches(4), Inches(3))
+ prs.save(str(path))
+
+ extraction = extract_material(path)
+
+ assert extraction.media, "the embedded picture should become locator media"
+ stored = extraction.media[0].data
+ assert stored == image, "reading media is kept byte-for-byte"
+ with PILImage.open(io.BytesIO(stored)) as whole:
+ assert max(whole.size) > MAX_INLINE_IMAGE_EDGE
+
+
def test_empty_and_unreadable_sources_raise_reading_error(tmp_path: Path) -> None:
missing = tmp_path / "nope.pdf"
with pytest.raises(ReadingError):
diff --git a/tests/utils/test_document_images.py b/tests/utils/test_document_images.py
index c909dcdf5..e45fa2ff8 100644
--- a/tests/utils/test_document_images.py
+++ b/tests/utils/test_document_images.py
@@ -17,13 +17,16 @@ from pptx.util import Inches as PptxInches
import pytest
from deeptutor.utils.document_images import (
+ MAX_INLINE_IMAGE_EDGE,
ImageBudget,
build_marker,
extract_docx_rich,
extract_pdf_images,
extract_pptx_rich,
find_markers,
+ fit_inline_image,
image_index_from_name,
+ reading_image_budget,
)
@@ -236,3 +239,123 @@ def test_pdf_images_default_budget_caps_at_twelve(tmp_path) -> None:
assert len(result.collection.images) == 12
assert result.collection.skipped_budget == 2
assert result.collection.skipped_page_budget == 0
+
+
+# ---------------------------------------------------------------------------
+# Inline transport bound
+# ---------------------------------------------------------------------------
+
+
+def _noise_jpeg(size: tuple[int, int], quality: int = 95, seed: int = 7) -> bytes:
+ """A large noisy JPEG: inside every byte budget, past the long-edge bound."""
+ from PIL import Image as PILImage
+
+ raw = random.Random(seed).randbytes(size[0] * size[1] * 3)
+ image = PILImage.frombytes("RGB", size, raw)
+ buf = io.BytesIO()
+ image.save(buf, format="JPEG", quality=quality)
+ return buf.getvalue()
+
+
+def test_inline_budget_fits_a_page_sized_raster() -> None:
+ """A scan past the long edge is fitted before admission, aspect preserved."""
+ from PIL import Image as PILImage
+
+ source = _noise_jpeg((2000, 1200))
+ budget = ImageBudget()
+ admitted = budget.try_admit(".jpg", source)
+
+ assert admitted is not None
+ assert admitted.name.endswith(".jpg")
+ assert admitted.mime_type == "image/jpeg"
+ assert len(admitted.data) < len(source)
+ with PILImage.open(io.BytesIO(admitted.data)) as fitted:
+ assert max(fitted.size) <= MAX_INLINE_IMAGE_EDGE
+ assert fitted.size == (1600, 960)
+ # The budget accounts the bytes that will actually ship, not the source.
+ assert budget.total_bytes == len(admitted.data)
+
+
+def test_reading_budget_keeps_source_resolution() -> None:
+ """Images stored for the reader pane are written as-is, never refitted."""
+ source = _noise_jpeg((2000, 1200))
+ admitted = reading_image_budget().try_admit(".jpg", source)
+
+ assert admitted is not None
+ assert admitted.data == source
+ assert admitted.mime_type == "image/jpeg"
+
+
+def test_inline_budget_leaves_small_images_untouched() -> None:
+ """Under the bound, the original bytes and format survive unchanged."""
+ source = _png_bytes((400, 300))
+ admitted = ImageBudget().try_admit(".png", source)
+
+ assert admitted is not None
+ assert admitted.data == source
+ assert admitted.mime_type == "image/png"
+
+
+def test_inline_budget_bounds_office_pictures_too() -> None:
+ """DOCX/PPTX pictures reach the same admission seam as PDF page rasters."""
+ from PIL import Image as PILImage
+
+ rich = extract_docx_rich(_docx_with_image(["A picture follows"], _noise_jpeg((2000, 1200))))
+
+ assert rich.collection.images, "the embedded picture should be extracted"
+ with PILImage.open(io.BytesIO(rich.collection.images[0].data)) as fitted:
+ assert max(fitted.size) <= MAX_INLINE_IMAGE_EDGE
+
+
+def test_fit_returns_undecodable_bytes_unchanged() -> None:
+ """A size optimisation must never be why an attachment becomes unreadable."""
+ payload = b"\x89PNG\r\n\x1a\n" + b"not really a png" * 64
+
+ assert fit_inline_image(".png", payload, max_edge=MAX_INLINE_IMAGE_EDGE) == (".png", payload)
+
+
+def test_fit_composites_transparency_onto_white() -> None:
+ """JPEG has no alpha channel; transparent areas must not turn black."""
+ from PIL import Image as PILImage
+ from PIL import ImageDraw as PILImageDraw
+
+ raw = random.Random(5).randbytes(2000 * 1200 * 3)
+ image = PILImage.frombytes("RGB", (2000, 1200), raw).convert("RGBA")
+ image.putalpha(0)
+ PILImageDraw.Draw(image).rectangle([0, 0, 300, 300], fill=(255, 0, 0, 255))
+ buf = io.BytesIO()
+ image.save(buf, format="PNG")
+
+ ext, data = fit_inline_image(".png", buf.getvalue(), max_edge=MAX_INLINE_IMAGE_EDGE)
+
+ assert ext == ".jpg"
+ with PILImage.open(io.BytesIO(data)) as fitted:
+ assert fitted.mode == "RGB"
+ assert max(fitted.size) <= MAX_INLINE_IMAGE_EDGE
+ # A transparent corner composites onto white, not onto black.
+ assert fitted.getpixel((fitted.size[0] - 4, fitted.size[1] - 4)) == (255, 255, 255)
+ opaque = fitted.getpixel((20, 20))
+ assert opaque[0] > 200 and opaque[1] < 80 and opaque[2] < 80
+
+
+def test_scanned_page_rides_bounded_but_reads_whole(tmp_path) -> None:
+ """The reported case: a whole-page scan, inline vs on-disk resolution."""
+ pymupdf = pytest.importorskip("pymupdf")
+ from PIL import Image as PILImage
+
+ doc = pymupdf.open()
+ page = doc.new_page(width=612, height=792)
+ page.insert_image(pymupdf.Rect(0, 0, 612, 792), stream=_noise_jpeg((2000, 1200)))
+ pdf_path = tmp_path / "scan.pdf"
+ doc.save(str(pdf_path))
+ doc.close()
+ data = pdf_path.read_bytes()
+
+ inline = extract_pdf_images(data).collection.images
+ stored = extract_pdf_images(data, budget=reading_image_budget()).collection.images
+
+ assert inline and stored
+ with PILImage.open(io.BytesIO(inline[0].data)) as fitted:
+ assert max(fitted.size) <= MAX_INLINE_IMAGE_EDGE
+ with PILImage.open(io.BytesIO(stored[0].data)) as whole:
+ assert max(whole.size) > MAX_INLINE_IMAGE_EDGE