Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
"""PDF parsing with per-page routing between native text and scanned images.
Design (aligned with how MinerU / Docling / DeepDoc route PDFs):
* The dominant signal for "this page is scanned" is the **image-area coverage
ratio** (image bounding-box area / page area), not the raw character count.
A scanned page is essentially one big image covering the whole page, even
when it carries a (often low-quality) embedded OCR text layer. Trusting that
embedded text layer is what produced garbled RAG content before.
* Pages are classified independently so hybrid PDFs (some native, some scanned)
are handled correctly. Native pages contribute their text layer; scanned
pages are rendered to JPEG and tagged ``image_source_type=scanned_pdf`` so the
Go App performs OCR/VLM on them (docreader itself never runs OCR).
No external services (e.g. MinerU) are required: the builtin engine is fully
self-sufficient using pypdfium2 + the Go-side OCR that already exists.
"""
import base64
from collections import Counter, deque
import io
import logging
import os
import re
import statistics
import threading
from typing import Iterator
from docreader.config import CONFIG
from docreader.models.document import Document
from docreader.parser.base_parser import BaseParser
from docreader.parser.concurrency import parser_worker_limit
from docreader.parser.source_locator import PageGeometry, page_blocks
from docreader.parser.stream import ParsedDocumentStream, StreamedImage
logger = logging.getLogger(__name__)
# pdfium (the C library behind pypdfium2) is process-global and NOT
# thread-safe: two gRPC worker threads parsing PDFs at the same time corrupt
# its shared state and can deadlock the whole process indefinitely (observed
# as requests stuck in "Parsing document with PDFParser" forever, taking down
# all subsequent uploads too). Every pdfium touch — text extraction, page
# rendering, image extraction — must be serialized behind this single global
# lock. Concurrent PDF uploads are then processed one at a time instead of
# hanging. Non-PDF parsers (docx, xlsx, ...) are unaffected and still run
# concurrently across the gRPC thread pool.
_PDFIUM_LOCK = threading.Lock()
def _env_float(name: str, default: float) -> float:
v = os.environ.get(name)
if v is None or not str(v).strip():
return default
try:
return float(v)
except ValueError:
return default
def _env_int(name: str, default: int) -> int:
v = os.environ.get(name)
if v is None or not str(v).strip():
return default
try:
return int(str(v).strip())
except ValueError:
return default
def _env_bool(name: str, default: bool) -> bool:
v = os.environ.get(name)
if v is None or not str(v).strip():
return default
return str(v).strip().lower() in {"1", "true", "yes", "y", "on"}
# A page whose image objects cover at least this fraction of the page area is
# treated as scanned (image-dominated). Native digital pages measure ~0.0-0.05;
# scanned pages measure ~1.0+, so 0.5 leaves a wide safety margin.
SCAN_IMAGE_AREA_RATIO = _env_float("DOCREADER_PDF_SCAN_IMAGE_RATIO", 0.5)
# Below this many characters a page is considered to have no usable text layer.
SCAN_MIN_CHARS_PER_PAGE = _env_int("DOCREADER_PDF_SCAN_MIN_CHARS", 10)
# A near-empty-text page is only rendered as an image if it actually contains
# some image content (avoids rendering genuinely blank pages).
_LOW_TEXT_IMAGE_RATIO = 0.1
# --- Embedded figure extraction (text pages) ------------------------------
# Native pages can embed figures/charts. We surface them as image references so
# the Go App can OCR/caption them (docreader does not caption). Logos, icons,
# watermarks and tiny decorations are filtered out by size, page-area share and
# cross-page repetition.
EXTRACT_EMBEDDED_IMAGES = _env_bool("DOCREADER_PDF_EXTRACT_EMBEDDED_IMAGES", True)
# Minimum pixel width AND height for an embedded image to be kept.
EMBED_MIN_PIXELS = _env_int("DOCREADER_PDF_EMBED_MIN_PIXELS", 80)
# Minimum share of the page area for an embedded image to be kept.
EMBED_MIN_AREA_RATIO = _env_float("DOCREADER_PDF_EMBED_MIN_AREA_RATIO", 0.01)
# An identical image appearing on at least this fraction of text pages is
# treated as a running logo/watermark and dropped.
EMBED_REPEAT_PAGE_FRAC = _env_float("DOCREADER_PDF_EMBED_REPEAT_PAGE_FRAC", 0.5)
# Hard cap on the number of embedded images extracted per document.
EMBED_MAX_IMAGES = _env_int("DOCREADER_PDF_EMBED_MAX_IMAGES", 50)
# --- Layout-aware text extraction (native text pages) ---------------------
# Reconstruct reading order with a geometric XY-cut so multi-column pages are
# linearised column-by-column instead of line-interleaved.
LAYOUT_ORDERING = _env_bool("DOCREADER_PDF_LAYOUT_ORDERING", True)
# When glyphs are positioned without explicit space characters (common in OCR /
# search text layers), insert a space if the horizontal gap exceeds this
# multiple of the line's median glyph width.
WORD_GAP_WIDTH_RATIO = _env_float("DOCREADER_PDF_WORD_GAP_WIDTH_RATIO", 0.4)
# Promote visually larger lines to markdown headings (font-size proxy = rect
# height relative to the page's median line height).
DETECT_HEADINGS = _env_bool("DOCREADER_PDF_DETECT_HEADINGS", True)
# Drop invisible (render-mode 3), off-page and degenerate text — a cheap guard
# against hidden-text prompt injection and OCR artefacts.
FILTER_HIDDEN_TEXT = _env_bool("DOCREADER_PDF_FILTER_HIDDEN_TEXT", True)
# Narrow side strips (arXiv watermarks, page labels) narrower than this share of
# page width are dropped when they look like vertical / single-glyph noise.
MARGIN_COL_WIDTH_RATIO = _env_float("DOCREADER_PDF_MARGIN_COL_WIDTH_RATIO", 0.12)
# Minimum characters on a line before font-size heuristics may promote it to a
# markdown heading (avoids ``### C`` from margin glyphs).
MIN_HEADING_LINE_CHARS = _env_int("DOCREADER_PDF_MIN_HEADING_LINE_CHARS", 8)
# Strip pdfium placeholder glyphs (U+FFFE) and soft hyphens; remove axis/legend text
# from vector figures when a Figure caption is present on the page.
SANITIZE_PDF_TEXT = _env_bool("DOCREADER_PDF_SANITIZE_TEXT", True)
STRIP_CHART_TEXT_DEBRIS = _env_bool("DOCREADER_PDF_STRIP_CHART_DEBRIS", True)
# Render detected vector chart regions (no embedded bitmap) as JPEG for VLM/OCR.
RENDER_VECTOR_FIGURES = _env_bool("DOCREADER_PDF_RENDER_VECTOR_FIGURES", True)
MIN_CHART_REGION_CHARS = _env_int("DOCREADER_PDF_MIN_CHART_REGION_CHARS", 18)
MIN_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MIN_CHART_REGION_AREA", 0.015)
MAX_CHART_REGION_AREA_RATIO = _env_float("DOCREADER_PDF_MAX_CHART_REGION_AREA", 0.42)
MAX_FIGURE_HEIGHT_RATIO = _env_float("DOCREADER_PDF_MAX_FIGURE_HEIGHT_RATIO", 0.38)
# --- Force scanned mode ------------------------------------------------------
# When True, ALL PDF pages are rendered as images and routed through OCR/VLM,
# bypassing the automatic text/scanned page classification. Useful for PDFs
# with a low-quality or misleading text layer (web-print, scanned, image-heavy).
# Can be overridden per-upload via parser_engine_overrides.pdf_force_scanned.
FORCE_SCANNED_PDF = _env_bool("DOCREADER_PDF_FORCE_SCANNED", False)
# --- Source locators ---------------------------------------------------------
# Record where each paragraph of a native text page sits on the page, so a
# citation can open the PDF there and outline the paragraph. Costs one glyph
# pass per text page; disable to only record page numbers.
SOURCE_BOXES = _env_bool("DOCREADER_PDF_SOURCE_BOXES", True)
# pdfium / Adobe text layers often emit U+FFFE for missing hyphenation or ligatures.
_PDF_ARTIFACT_RE = re.compile(r"[\u00ad\u200b-\u200f\ufeff\ufffe\uffff]")
_PDF_ARTIFACT_JOIN_RE = re.compile(r"(\w)[\u00ad\ufffe](\w)")
_CHART_DEBRIS_LINE_RE = re.compile(
r"^(?:"
r"[\d\s.]+|"
r"\d{1,2}|"
r"\d+-layer|"
r"iter\.\s*\(1e4\)|"
r"(?:training|test)\s+error\s*\(%\)"
r")$",
re.IGNORECASE,
)
_CHART_LAYER_RE = re.compile(r"^\d+-layer$", re.IGNORECASE)
_FIGURE_CAPTION_RE = re.compile(r"^Figure\s+\d+\b", re.IGNORECASE)
_FIGURE_CAPTION_SEARCH_RE = re.compile(r"\bFigure\s+(\d+)\b", re.IGNORECASE)
_ARXIV_LINE_RE = re.compile(r"^arXiv:\s*\S+", re.IGNORECASE)
_PAGE_NUM_LINE_RE = re.compile(r"^\d{1,3}$")
_PAGE_CONTEXT_RE = re.compile(
r"(?:\bpage\b|\bp\.\s*\d+\b|\b\d+\s*(?:of|/)\s*\d+\b)",
re.IGNORECASE,
)
_CHART_CONTEXT_RE = re.compile(
r"(?:\bFigure\s+\d+\b|\biter\.\s*\(1e4\)|"
r"\b(?:training|test)\s+error\b|\b\d+-layer\b)",
re.IGNORECASE,
)
_NUMERIC_ROW_RE = re.compile(
r"^(?P<label>[^\d\n]+?)\s+"
r"(?P<value>[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?)$"
)
_NUMERIC_VALUE_LINE_RE = re.compile(
r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"
)
def _close_pdfium_resource(resource) -> None:
close = getattr(resource, "close", None)
if close:
close()
def _normalize_image_quality(quality: int) -> int:
return min(95, max(1, quality))
def _classify_page(image_area_ratio: float, text_len: int) -> str:
"""Classify a page as ``"scanned"`` or ``"text"``.
Image-area coverage is the primary signal; a sparse text layer combined with
some image content is the secondary signal.
"""
if image_area_ratio >= SCAN_IMAGE_AREA_RATIO:
return "scanned"
if text_len < SCAN_MIN_CHARS_PER_PAGE and image_area_ratio >= _LOW_TEXT_IMAGE_RATIO:
return "scanned"
return "text"
def _page_image_area_ratio(page, raw) -> float:
"""Return the fraction of the page area covered by image objects.
Overlapping images can push the ratio above 1.0; callers only compare it
against a threshold so that is harmless.
"""
width, height = page.get_size()
page_area = float(width) * float(height)
if page_area <= 0:
return 0.0
image_area = 0.0
for obj in page.get_objects():
try:
if obj.type == raw.FPDF_PAGEOBJ_IMAGE:
left, bottom, right, top = obj.get_bounds()
image_area += abs((right - left) * (top - bottom))
except Exception:
continue
return image_area / page_area
def _extract_page_text(page) -> str:
"""Plain top-to-bottom text extraction (fallback path)."""
textpage = None
try:
textpage = page.get_textpage()
return textpage.get_text_range()
finally:
_close_pdfium_resource(textpage)
def _sanitize_pdf_text(text: str) -> str:
"""Remove PDF text-layer placeholders and repair broken hyphenations."""
if not text:
return text
text = _PDF_ARTIFACT_RE.sub("", text)
text = _PDF_ARTIFACT_JOIN_RE.sub(r"\1\2", text)
return text
def _is_chart_debris_line(line: str) -> bool:
t = line.strip()
if not t:
return False
if _CHART_DEBRIS_LINE_RE.match(t):
return True
if _CHART_LAYER_RE.match(t):
return True
# Tick labels like "0 1 2 3 4 5 6 0"
if re.fullmatch(r"[\d\s.()-]+", t) and len(t) <= 24 and sum(c.isdigit() for c in t) >= 3:
return True
return False
def _strip_chart_text_debris(text: str) -> str:
"""Drop runs of axis/legend lines leaked from vector figures into the text layer."""
if not text:
return text
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
out: list = []
i = 0
while i < len(lines):
if _is_chart_debris_line(lines[i]):
j = i
while j < len(lines) and (
_is_chart_debris_line(lines[j]) or not lines[j].strip()
):
j += 1
# Numeric-only lines are also a common representation for table
# values. Only remove a run when a nearby chart/figure marker
# provides positive evidence that it is axis or legend text.
context_start = max(0, i - 1)
context_end = min(len(lines), j + 1)
has_chart_context = any(
_CHART_CONTEXT_RE.search(lines[k])
for k in range(context_start, context_end)
)
table_context = _tail_looks_like_numeric_table(
lines[max(0, i - 2) : j]
)
if j - i >= 3 and has_chart_context and not table_context:
i = j
continue
out.append(lines[i])
i += 1
return "\n".join(out)
def _is_probable_page_num_line(lines: list, index: int) -> bool:
"""Return whether a standalone number is likely a page marker.
A numeric-only line in the middle of a page is often a table value. The
old unconditional filter removed those values, so page-number cleanup is
deliberately limited to an explicitly labelled edge line or a line adjacent
to an arXiv header. Keeping an ambiguous number is safer than silently
losing document data.
"""
if index < 0 or index >= len(lines):
return False
if not _PAGE_NUM_LINE_RE.fullmatch(lines[index].strip()):
return False
nonempty = [i for i, line in enumerate(lines) if line.strip()]
if not nonempty:
return False
pos = nonempty.index(index)
neighbors = []
if pos:
neighbors.append(nonempty[pos - 1])
if pos + 1 < len(nonempty):
neighbors.append(nonempty[pos + 1])
if any(_ARXIV_LINE_RE.match(lines[i].strip()) for i in neighbors):
return True
# Keep an edge number unless a nearby line explicitly looks like a page
# label (``Page 3 of 8`` / ``3 / 8``). Ambiguous edge numbers can still be
# legitimate first/last values in a table.
if index in (nonempty[0], nonempty[-1]):
return any(_PAGE_CONTEXT_RE.search(lines[i]) for i in neighbors)
return False
def _strip_arxiv_and_page_num_lines(text: str) -> str:
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
kept: list = []
for i, ln in enumerate(lines):
t = ln.strip()
if _ARXIV_LINE_RE.match(t):
continue
if _is_probable_page_num_line(lines, i):
continue
if "arXiv:" in ln:
ln = re.sub(r"\s*arXiv:\s*\S+\s*(?:\[[^\]]+\])?\s*[^\n]*", "", ln).strip()
if not ln:
continue
kept.append(ln)
return "\n".join(kept)
def _strip_lines_above_figure_captions(text: str) -> str:
"""Remove diagram/chart label lines that sit immediately above a Figure caption."""
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
out: list = []
for ln in lines:
if _line_has_figure_caption(ln):
while out and _is_figure_interior_line(out[-1]):
# A table can legitimately end immediately before a figure
# caption. Preserve a tail with two labelled numeric rows;
# diagram labels do not have this row-shaped signal.
if _tail_looks_like_numeric_table(out):
break
out.pop()
out.append(ln)
else:
out.append(ln)
return "\n".join(out)
def _tail_looks_like_numeric_table(lines: list) -> bool:
"""Detect labelled or alternating label/value rows at the text tail."""
tail = [line for line in lines[-10:] if line.strip()]
rows = _structured_numeric_rows("\n".join(tail))
if rows and rows[-1][0] == len(tail) - 1:
return True
numeric = re.compile(
r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"
)
for index, header in enumerate(tail[:-2]):
if (
header.strip()
and any(char.isalpha() for char in header)
and all(numeric.fullmatch(value.strip()) for value in tail[index + 1 :])
):
return True
pairs = 0
index = len(tail) - 1
while index >= 1:
label = tail[index - 1].strip()
value = tail[index].strip()
if (
label
and any(char.isalpha() for char in label)
and len(label) <= 60
and numeric.fullmatch(value)
):
pairs += 1
index -= 2
else:
break
return pairs >= 2
def _is_body_paragraph_line(text: str) -> bool:
t = text.strip()
if len(t) < 48:
return False
return len(t.split()) >= 8
def _is_figure_interior_line(text: str) -> bool:
"""Short, non-body line directly above a Figure caption (diagram labels, ticks)."""
t = text.strip()
if not t or _FIGURE_CAPTION_RE.match(t):
return False
if _ARXIV_LINE_RE.match(t) or _PAGE_NUM_LINE_RE.match(t):
return True
if _is_body_paragraph_line(t):
return False
if _is_chart_debris_line(t):
return True
# Prose sentence above a figure (wrapped paragraph tail) — keep in text.
if t.endswith((".", "。", "!", "?", "!")) and len(t) >= 15:
return False
if len(t.split()) >= 7:
return False
if len(t) <= 40:
return True
return False
def _postprocess_pdf_text(text: str) -> str:
if SANITIZE_PDF_TEXT:
text = _sanitize_pdf_text(text)
text = _strip_arxiv_and_page_num_lines(text)
text = _strip_lines_above_figure_captions(text)
if STRIP_CHART_TEXT_DEBRIS:
text = _strip_chart_text_debris(text)
return text
def _char_looks_chart_axis_tick(ch: str) -> bool:
"""Axis tick / numeric chart labels only (not words like ``layer`` in diagrams)."""
t = ch.strip()
if not t:
return False
if len(t) == 1 and t in "0123456789.%()-":
return True
if _CHART_LAYER_RE.match(t):
return True
if re.fullmatch(r"iter\.\s*\(1e4\)", t, re.I):
return True
if re.fullmatch(r"(?:training|test)\s+error\s*\(%\)", t, re.I):
return True
return False
def _chars_bbox(char_list: list) -> tuple:
return (
min(c["x0"] for c in char_list),
min(c["y0"] for c in char_list),
max(c["x1"] for c in char_list),
max(c["y1"] for c in char_list),
)
def _bbox_area_ratio(bbox, page_w: float, page_h: float) -> float:
page_area = float(page_w) * float(page_h)
if page_area <= 0:
return 0.0
x0, y0, x1, y1 = bbox
return max(0.0, (x1 - x0) * (y1 - y0) / page_area)
def _chart_region_bbox(chars: list, page_w: float, page_h: float):
"""Bounding box of numeric chart axis labels (fallback when caption walk fails)."""
chart = [c for c in chars if _char_looks_chart_axis_tick(c["ch"])]
if len(chart) < MIN_CHART_REGION_CHARS:
return None
bbox = _chars_bbox(chart)
ratio = _bbox_area_ratio(bbox, page_w, page_h)
if ratio < MIN_CHART_REGION_AREA_RATIO or ratio > MAX_CHART_REGION_AREA_RATIO:
return None
x0, y0, x1, y1 = bbox
pad_x = max(8.0, (x1 - x0) * 0.08)
pad_y = max(8.0, (y1 - y0) * 0.08)
return (
max(0.0, x0 - pad_x),
max(0.0, y0 - pad_y),
min(page_w, x1 + pad_x),
min(page_h, y1 + pad_y),
)
def _expand_chart_bbox(bbox, page_w: float, page_h: float, margin_frac: float = 0.18):
x0, y0, x1, y1 = bbox
dx = (x1 - x0) * margin_frac
dy = (y1 - y0) * margin_frac
return (
max(0.0, x0 - dx),
max(0.0, y0 - dy),
min(page_w, x1 + dx),
min(page_h, y1 + dy),
)
def _render_page_clip_jpeg(page, bbox, scale: float, quality: int, max_edge: int) -> bytes:
"""Render a PDF page region to JPEG (bbox in PDF points, bottom-left origin)."""
left, bottom, right, top = bbox
scale_eff = _effective_scale(page, scale, max_edge)
bitmap = None
try:
bitmap = page.render(scale=scale_eff)
pil = bitmap.to_pil().convert("RGB")
finally:
_close_pdfium_resource(bitmap)
page_w, page_h = page.get_size()
x0 = int(left * scale_eff)
x1 = int(right * scale_eff)
y0 = int((page_h - top) * scale_eff)
y1 = int((page_h - bottom) * scale_eff)
if x1 <= x0 or y1 <= y0:
raise ValueError("degenerate clip bbox")
return _pil_to_jpeg_bytes(pil.crop((x0, y0, x1, y1)), quality)
def _pil_to_jpeg_bytes(pil, quality: int) -> bytes:
buf = io.BytesIO()
if pil.mode not in ("RGB", "L"):
pil = pil.convert("RGB")
pil.save(buf, format="JPEG", quality=quality, optimize=True)
return buf.getvalue()
def _group_lines_with_chars(chars: list) -> list:
"""Group glyphs into lines; each line includes its char list and bbox."""
if not chars:
return []
heights = [c["y1"] - c["y0"] for c in chars if c["y1"] > c["y0"]]
med_h = statistics.median(heights) if heights else 1.0
ordered = sorted(chars, key=lambda c: -(c["y0"] + c["y1"]) / 2)
groups: list = []
cur: list = []
ref = None
for c in ordered:
yc = (c["y0"] + c["y1"]) / 2
if ref is None or abs(yc - ref) <= 0.5 * med_h:
cur.append(c)
ref = yc if ref is None else ref
else:
groups.append(cur)
cur = [c]
ref = yc
if cur:
groups.append(cur)
lines: list = []
for grp in groups:
grp_sorted = sorted(grp, key=lambda c: c["x0"])
text = _join_line_glyphs(grp_sorted)
if not text:
continue
hs = [c["y1"] - c["y0"] for c in grp_sorted if c["y1"] > c["y0"]]
lines.append(
{
"text": text,
"h": statistics.median(hs) if hs else med_h,
"chars": grp_sorted,
"bbox": _chars_bbox(grp_sorted),
}
)
return lines
def _line_has_figure_caption(text: str) -> bool:
return bool(_FIGURE_CAPTION_SEARCH_RE.search((text or "").strip()))
def _bbox_above_caption(lines: list, cap_i: int, page_w: float, page_h: float):
"""Region above a Figure caption line (PDF coords, bottom-left origin)."""
cap_bbox = lines[cap_i]["bbox"]
cap_top = cap_bbox[3]
x0, x1 = cap_bbox[0], cap_bbox[2]
fig_h = page_h * min(MAX_FIGURE_HEIGHT_RATIO, 0.35)
y_bottom = cap_top
y_top = min(page_h, cap_top + fig_h)
for j in range(cap_i - 1, -1, -1):
t = lines[j]["text"]
b = lines[j]["bbox"]
if b[3] < y_bottom - 4:
continue
if b[1] > y_top + 4:
break
if _is_body_paragraph_line(t) and not _is_figure_interior_line(t):
break
if _is_figure_interior_line(t) or _is_chart_debris_line(t) or not t.strip():
x0 = min(x0, b[0])
x1 = max(x1, b[2])
y_top = max(y_top, min(page_h, b[3] + fig_h * 0.15))
min_h = page_h * 0.08
if y_top - y_bottom < min_h:
y_top = min(page_h, y_bottom + min_h)
margin_x = max(8.0, (x1 - x0) * 0.05)
return (
max(0.0, x0 - margin_x),
y_bottom,
min(page_w, x1 + margin_x),
y_top,
)
def _cap_bbox_height(bbox, page_h: float, cap_y_top: float) -> tuple:
"""Limit figure bbox height (PDF coords, bottom-left origin)."""
x0, y0, x1, y1 = bbox
max_top = min(y1, cap_y_top + page_h * MAX_FIGURE_HEIGHT_RATIO)
if max_top <= y0:
return bbox
return (x0, y0, x1, max_top)
def _inject_figure_markdown_before_captions(text: str, clips: list) -> str:
"""Place ``![...]()`` immediately before each Figure caption line in page text."""
if not clips:
return text
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
clip_idx = 0
for i, ln in enumerate(lines):
if clip_idx >= len(clips):
break
if not _line_has_figure_caption(ln):
continue
if i > 0 and lines[i - 1].lstrip().startswith("!["):
continue
ref_path = clips[clip_idx][0]
fname = os.path.basename(ref_path)
img_md = f""
lines[i] = f"{img_md}\n\n{ln}"
clip_idx += 1
return "\n".join(lines)
def _extract_vector_figure_clips(
page,
page_index: int,
plain_text: str,
raw,
base_name: str,
scale: float,
quality: int,
max_edge: int,
) -> list:
"""Render vector figure regions anchored at each ``Figure N.`` caption on the page.
Returns ``[(ref_path, b64, y_sort, caption_line), ...]`` for markdown injection.
"""
if not RENDER_VECTOR_FIGURES or not re.search(r"\bFigure\s+\d+", plain_text, re.I):
return []
textpage = None
try:
textpage = page.get_textpage()
chars, page_w = _page_chars(textpage, page, raw)
if not chars:
return []
page_h = page.get_size()[1]
lines = _merge_orphan_punctuation_lines(_group_lines_with_chars(chars))
caption_indices = [
i for i, ln in enumerate(lines) if _line_has_figure_caption(ln["text"])
]
if not caption_indices:
return []
results: list = []
for fig_idx, cap_i in enumerate(caption_indices):
cap_line = lines[cap_i]["text"].strip()
m = _FIGURE_CAPTION_SEARCH_RE.search(cap_line)
if m:
cap_line = cap_line[m.start() :].split("\n", 1)[0].strip()
bbox = _bbox_above_caption(lines, cap_i, page_w, page_h)
if bbox is None:
bbox = _chart_region_bbox(chars, page_w, page_h)
if bbox is None:
continue
ratio = _bbox_area_ratio(bbox, page_w, page_h)
if ratio > MAX_CHART_REGION_AREA_RATIO:
bbox = _cap_bbox_height(bbox, page_h, lines[cap_i]["bbox"][3])
ratio = _bbox_area_ratio(bbox, page_w, page_h)
if ratio > MAX_CHART_REGION_AREA_RATIO:
continue
if ratio < MIN_CHART_REGION_AREA_RATIO:
continue
bbox = _expand_chart_bbox(bbox, page_w, page_h, margin_frac=0.06)
jpeg = _render_page_clip_jpeg(page, bbox, scale, quality, max_edge)
fname = f"{base_name}_p{page_index + 1}_fig{fig_idx + 1}.jpg"
ref_path = f"images/{fname}"
results.append(
(
ref_path,
base64.b64encode(jpeg).decode("utf-8"),
bbox[3],
cap_line,
)
)
return results
except Exception:
logger.debug("vector figure clip failed on page %d", page_index, exc_info=True)
return []
finally:
_close_pdfium_resource(textpage)
def _collect_invisible_boxes(page, raw) -> list:
"""Bounding boxes of invisible (render-mode 3) text objects on the page."""
boxes: list = []
try:
for obj in page.get_objects():
if obj.type != raw.FPDF_PAGEOBJ_TEXT:
continue
try:
mode = raw.FPDFTextObj_GetTextRenderMode(obj.raw)
except Exception:
continue
if mode != raw.FPDF_TEXTRENDERMODE_INVISIBLE:
continue
try:
left, bottom, right, top = obj.get_bounds()
except Exception:
continue
boxes.append(
(min(left, right), min(bottom, top), max(left, right), max(bottom, top))
)
except Exception:
return []
return boxes
def _point_in_boxes(x: float, y: float, boxes: list) -> bool:
for x0, y0, x1, y1 in boxes:
if x0 <= x <= x1 and y0 <= y <= y1:
return True
return False
def _page_chars(
textpage, page, raw, loose: bool = False, *, return_filter_info: bool = False
) -> tuple:
"""Return ``(chars, page_width)`` with hidden/off-page glyphs filtered.
Working at the glyph level (instead of pdfium rect segments) keeps mixed
CJK + Latin/number lines in their true left-to-right order, which the
rect-level ``get_text_bounded`` API scrambles. ``loose`` boxes span the
font's ascent to descent, so every glyph of a line shares one height.
When ``return_filter_info`` is true, a third value reports whether any
source glyph was intentionally filtered as hidden or off-page. Callers
can then avoid falling back to the unfiltered plain text when a PDF
contains hidden or off-page text.
"""
n = textpage.count_chars()
if n <= 0:
return ([], 0.0, False) if return_filter_info else ([], 0.0)
# Glyph coordinates are unrotated PDF user space. get_size() reports
# rotated dimensions and would discard valid text on cropped/rotated pages.
try:
crop_left, crop_bottom, crop_right, crop_top = page.get_cropbox()
except Exception:
width, height = page.get_size()
crop_left, crop_bottom, crop_right, crop_top = 0, 0, width, height
width = crop_right - crop_left
invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else []
chars: list = []
source_count = 0
filtered_count = 0
for i in range(n):
try:
try:
left, bottom, right, top = textpage.get_charbox(i, loose=loose)
except TypeError:
# Keep compatibility with the small PDFium test doubles and
# older pypdfium2 releases that do not expose ``loose``.
left, bottom, right, top = textpage.get_charbox(i)
except Exception:
source_count += 1
continue
ch = textpage.get_text_range(i, 1)
if ch in ("\r", "\n"):
continue
source_count += 1
x0, x1 = (left, right) if left <= right else (right, left)
y0, y1 = (bottom, top) if bottom <= top else (top, bottom)
if FILTER_HIDDEN_TEXT:
if x1 < crop_left or x0 > crop_right or y1 < crop_bottom or y0 > crop_top:
filtered_count += 1
continue # off-page glyph
if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible):
filtered_count += 1
continue # covered by an invisible text object
chars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch, "i": i})
if return_filter_info:
return chars, width, filtered_count > 0
return chars, width
def _page_has_invisible_text(page, raw) -> bool:
"""Whether ``page`` contains text objects hidden from normal rendering."""
if not FILTER_HIDDEN_TEXT:
return False
try:
return bool(_collect_invisible_boxes(page, raw))
except Exception:
# A failed probe should not change the existing plain-text route. The
# full glyph-level probe still runs whenever layout extraction is used.
return False
def _page_has_filtered_text(page, raw) -> bool:
"""Whether layout filtering would discard hidden or off-page glyphs."""
if not FILTER_HIDDEN_TEXT:
return False
if _page_has_invisible_text(page, raw):
return True
textpage = None
try:
textpage = page.get_textpage()
_, _, filtered = _page_chars(textpage, page, raw, return_filter_info=True)
return filtered
except Exception:
return False
finally:
_close_pdfium_resource(textpage)
def _find_split(items: list, axis: str, min_gap: float):
"""Return a coordinate at the widest clean gap on ``axis`` ('x'), or None.
A "clean" gap means no item interval bridges it — i.e. a full-height column
gutter. Used to detect multi-column layouts.
"""
lo, hi = ("x0", "x1") if axis == "x" else ("y0", "y1")
intervals = sorted(((s[lo], s[hi]) for s in items), key=lambda iv: iv[0])
cur_end = intervals[0][1]
best_gap, best_cut = 0.0, None
for a, b in intervals[1:]:
gap = a - cur_end
if gap >= min_gap and gap > best_gap:
best_gap, best_cut = gap, cur_end + gap / 2
if b > cur_end:
cur_end = b
return best_cut
def _split_columns(chars: list, scale: float, width: float, depth: int = 0) -> list:
"""Split glyphs into reading-order columns at full-height gutters."""
if len(chars) <= 1 or depth > 10:
return [chars]
min_gap = max(scale * 2.5, width * 0.04)
cut = _find_split(chars, "x", min_gap)
if cut is None:
return [chars]
left = [c for c in chars if (c["x0"] + c["x1"]) / 2 < cut]
right = [c for c in chars if (c["x0"] + c["x1"]) / 2 >= cut]
if not left or not right:
return [chars]
return _split_columns(left, scale, width, depth + 1) + _split_columns(
right, scale, width, depth + 1
)
def _column_x_span(chars: list) -> float:
if not chars:
return 0.0
return max(c["x1"] for c in chars) - min(c["x0"] for c in chars)
def _column_single_line_fraction(lines: list) -> float:
if not lines:
return 0.0
single = sum(1 for ln in lines if len(ln["text"]) <= 2)
return single / len(lines)
def _is_artifact_column(chars: list, width: float) -> bool:
"""Detect margin strips and vertical watermarks (e.g. arXiv sidebar).
Docling / MinerU solve this with learned layout regions; here we use
geometry only: a narrow column whose lines are mostly one glyph tall is not
part of the reading order.
"""
if not chars or width <= 0:
return True
span = _column_x_span(chars)
if span <= 0:
return True
lines = _group_lines(chars)
single_frac = _column_single_line_fraction(lines)
narrow = span / width < MARGIN_COL_WIDTH_RATIO
if narrow and single_frac >= 0.45:
return True
ys = [(c["y0"] + c["y1"]) / 2 for c in chars]
y_span = max(ys) - min(ys)
# Vertical text: tall stack, narrow horizontal extent, mostly one char/line.
if y_span > span * 3.5 and len(chars) >= 8 and single_frac >= 0.35:
return True
return False
def _filter_reading_columns(chars: list, scale: float, width: float) -> list:
"""Split into columns and drop margin / watermark strips."""
cols = _split_columns(chars, scale, width)
table_pair = _find_numeric_table_columns(chars, scale, width, cols)
if table_pair is not None:
# Keep aligned table cells in one visual stream so rows such as
# ``Aster 124`` are not emitted as a label block followed by a value
# block, while still dropping unrelated margin/artifact columns.
# ``chars`` has already passed hidden/off-page filtering.
value_col, label_col = table_pair
table_cols = [
c
for c in cols
if not _is_artifact_column(c, width) or c is value_col or c is label_col
]
if table_cols:
return [[char for column in table_cols for char in column]]
return [chars]
kept = [c for c in cols if not _is_artifact_column(c, width)]
if kept:
return kept
# All columns looked like noise — keep the widest glyph set (main body).
if len(cols) > 1:
return [max(cols, key=_column_x_span)]
return cols
def _is_numeric_value_line(text: str) -> bool:
t = (text or "").strip()
return bool(
_NUMERIC_VALUE_LINE_RE.fullmatch(t)
or (
re.fullmatch(r"[\d\s./()%+-]+", t)
and any(char.isdigit() for char in t)
)
)
def _looks_like_numeric_table_columns(
chars: list, scale: float, width: float, columns: list | None = None
) -> bool:
"""Identify a narrow, aligned value column before artifact filtering.
A real two-column page should remain column-wise. A simple table often
looks different: one narrow column contains a header and mostly numeric
values, and its baselines line up with labels in a wider column. This
signal lets the caller choose the plain extractor only for that table-like
shape instead of turning every numeric mismatch into a page-wide fallback.
"""
return _find_numeric_table_columns(chars, scale, width, columns) is not None
def _find_numeric_table_columns(
chars: list, scale: float, width: float, columns: list | None = None
) -> tuple[list, list] | None:
"""Return the aligned value/label columns when a numeric table is found."""
if not chars or width <= 0:
return None
cols = columns if columns is not None else _split_columns(chars, scale, width)
if len(cols) < 2:
return None
for value_col in cols:
span_ratio = _column_x_span(value_col) / width
if span_ratio > 0.22:
continue
value_lines = _group_lines_with_chars(value_col)
if len(value_lines) < 2:
continue
numeric_lines = [
line for line in value_lines if _is_numeric_value_line(line["text"])
]
if not numeric_lines or len(numeric_lines) / len(value_lines) < 0.4:
continue
# Require a textual header/label in the narrow column. This keeps a
# vertical numeric watermark from being mistaken for a table.
text_lines = [
line
for line in value_lines
if not _is_numeric_value_line(line["text"])
and any(char.isalpha() for char in line["text"])
]
if not text_lines:
continue
value_ys = [
(line["bbox"][1] + line["bbox"][3]) / 2 for line in numeric_lines
]
for label_col in cols:
if label_col is value_col:
continue
label_lines = _group_lines_with_chars(label_col)
label_ys = [
(line["bbox"][1] + line["bbox"][3]) / 2
for line in label_lines
]
label_text_ys = [
(line["bbox"][1] + line["bbox"][3]) / 2
for line in label_lines
if any(char.isalpha() for char in line["text"])
]
tolerance = max(scale * 0.75, 2.0)
header_ys = [
(line["bbox"][1] + line["bbox"][3]) / 2 for line in text_lines
]
header_aligned = any(
abs(y - other_y) <= tolerance
for y in header_ys
for other_y in label_text_ys
)
if not header_aligned:
continue
aligned = sum(
any(abs(y - other_y) <= tolerance for other_y in label_ys)
for y in value_ys
)
# A table may contain an empty value cell. In that case the one
# remaining numeric row still carries enough evidence when its
# header and the label column share the same baseline.
required_alignment = (
1 if len(value_ys) == 1 else max(2, (len(value_ys) + 1) // 2)
)
if aligned >= required_alignment:
return value_col, label_col
return None
def _merge_orphan_punctuation_lines(lines: list) -> list:
"""Attach lines that are only punctuation to the previous visual line.
Many PDFs place ``.`` in figure labels or footnotes on a slightly different
baseline; grouping by y then leaves ``Figure 1`` and ``2:`` on separate lines.
"""
if not lines:
return []
merged: list = []
for ln in lines:
t = ln["text"].strip()
if (
merged
and t
and len(t) <= 4
and all(c in ".,;:!?…·" or c.isspace() for c in t)
):
suffix = "".join(t.split())
prev = merged[-1]["text"]
if suffix and prev and not prev.endswith((" ", "-")):
merged[-1]["text"] = prev + suffix
else:
merged[-1]["text"] = (prev + " " + t).strip()
continue
merged.append(dict(ln))
return merged
def _join_line_glyphs(ln_sorted: list) -> str:
"""Join a visual line's glyphs, inferring word spaces from horizontal gaps."""
if not ln_sorted:
return ""
widths = [c["x1"] - c["x0"] for c in ln_sorted if c["x1"] > c["x0"]]
med_w = statistics.median(widths) if widths else 1.0
gap_threshold = med_w * WORD_GAP_WIDTH_RATIO
parts: list[str] = []
for i, cur in enumerate(ln_sorted):
ch = cur["ch"]
if i == 0:
parts.append(ch)
continue
prev = ln_sorted[i - 1]
if ch.isspace() or prev["ch"].isspace():
if not ch.isspace() or (parts and not parts[-1].endswith(" ")):
parts.append(ch)
continue
gap = cur["x0"] - prev["x1"]
# PDF producers frequently position digits with a small kerning gap,
# especially in right-aligned table cells. Treating that gap as a
# word boundary turns ``124`` into ``1 24``. Keep the normal gap
# heuristic for genuinely separated numeric cells (larger gaps).
numeric_pair = (
prev["ch"] in "0123456789.,%+-"
and ch in "0123456789.,%+-"
and (prev["ch"].isdigit() or ch.isdigit())
)
if gap > gap_threshold and not (
numeric_pair and gap <= med_w * 1.25
):
parts.append(" ")
parts.append(ch)
return "".join(parts).strip()
def _group_lines(chars: list) -> list:
"""Group a column's glyphs into lines (top-to-bottom, glyphs sorted by x)."""
if not chars:
return []
heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]
med_h = statistics.median(heights) if heights else 1.0
ordered = sorted(chars, key=lambda c: -(c["y0"] + c["y1"]) / 2)
lines: list = []
cur: list = []
ref = None
for c in ordered:
yc = (c["y0"] + c["y1"]) / 2
if ref is None or abs(yc - ref) <= 0.5 * med_h:
cur.append(c)
ref = yc if ref is None else ref
else:
lines.append(cur)
cur = [c]
ref = yc
if cur:
lines.append(cur)
out: list = []
for ln in lines:
ln_sorted = sorted(ln, key=lambda c: c["x0"])
text = _join_line_glyphs(ln_sorted)
if not text:
continue
hs = [c["y1"] - c["y0"] for c in ln_sorted if c["y1"] - c["y0"] > 0]
out.append({"h": statistics.median(hs) if hs else med_h, "text": text})
return out
def _segments_to_markdown(lines: list) -> str:
"""Render merged lines to text, promoting visually large lines to headings."""
if not lines:
return ""
body = statistics.median([ln["h"] for ln in lines])
def level(ln) -> int:
txt = ln["text"]
if (
not DETECT_HEADINGS
or body <= 0
or len(txt) > 80
or len(txt) < MIN_HEADING_LINE_CHARS
):
return 0
if txt[-1:] in ".。!!??,,;;::":
return 0
r = ln["h"] / body
if r >= 2.0:
return 1
if r >= 1.6:
return 2
if r >= 1.35:
return 3
return 0
levels = [level(ln) for ln in lines]
# If too many lines qualify, the font sizes are too uniform/noisy to trust.
if sum(1 for x in levels if x) > max(1, int(0.4 * len(lines))):
levels = [0] * len(lines)
out = []
for ln, lv in zip(lines, levels):
out.append(("#" * lv + " " + ln["text"]) if lv else ln["text"])
return "\n".join(out)
def _chars_to_layout_markdown(chars: list, scale: float, width: float) -> str:
blocks: list = []
for col in _filter_reading_columns(chars, scale, width):
lines = _merge_orphan_punctuation_lines(_group_lines(col))
md = _segments_to_markdown(lines)
if md:
blocks.append(md)
return "\n".join(blocks)
def _layout_line_stats(text: str) -> tuple:
"""Return (line_count, single_char_line_count, punct_only_line_count)."""
lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
if not lines:
return 0, 0, 0
single = sum(1 for ln in lines if len(ln) <= 2)
punct_only = sum(
1
for ln in lines
if len(ln) <= 4 and re.fullmatch(r"[\s.,;:!?…·\-–—]+", ln)
)
return len(lines), single, punct_only
def _layout_garbled_line_fraction(text: str) -> float:
"""Share of lines that look like broken OCR (many 1–2 letter tokens)."""
lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
if not lines:
return 0.0
garbled = 0
for ln in lines:
words = ln.split()
if len(words) >= 6 and sum(1 for w in words if len(w) <= 2) / len(words) > 0.45:
garbled += 1
return garbled / len(lines)
def _numeric_row_values(text: str) -> list:
"""Return compact ``(line_index, label, value)`` rows from plain text.
This is intentionally conservative: a row must end in one numeric value,
have a short non-numeric label, and contain at least one letter. It is a
signal for choosing between extraction paths, not a table parser.
"""
rows: list = []
for index, line in enumerate((text or "").splitlines()):
match = _NUMERIC_ROW_RE.fullmatch(line.strip())
if not match:
continue
label = match.group("label").strip()
if (
not label
or len(label) > 60
or len(label.split()) > 4
or label[-1:] in ".,;:!?"
or not any(char.isalpha() for char in label)
):
continue
rows.append((index, label, match.group("value")))
return rows
def _structured_numeric_rows(text: str) -> list:
"""Return numeric rows only when at least two occur near each other."""
rows = _numeric_row_values(text)
if len(rows) < 2:
return []
for first, second in zip(rows, rows[1:]):
if second[0] - first[0] <= 3:
return rows
return []
def _normalized_numeric_value(value: str) -> str:
return value.replace(",", "").lower()
def _plain_is_well_formed(plain: str) -> bool:
"""True when pdfium plain text already has usable words and punctuation.
Academic PDFs (arXiv) and TOCs already expose a good text layer; running
geometric layout on them often destroys citations and words. Scanned books
with a poor text layer (no commas in refs, short glued tokens) still need
layout gap inference.
"""
plain = (plain or "").strip()
if not plain:
return False
if re.search(r"\[\w+,\s", plain):
return True
if plain.count(" . . ") >= 2:
return True
words = re.findall(r"\S+", plain)
if len(words) < 30:
return False
avg_len = sum(len(w) for w in words) / len(words)
return avg_len >= 5.0
def _should_prefer_plain(
plain: str, layout: str, *, allow_structured_rows: bool = True
) -> bool:
"""Fall back to pdfium plain text when layout reconstruction looks broken.
``allow_structured_rows`` is disabled by the page router unless geometry
identifies a narrow, aligned numeric table column. This prevents a real
two-column page from being replaced by its potentially interleaved plain
stream merely because it contains numbers.
"""
layout = (layout or "").strip()
plain = (plain or "").strip()
if not layout:
return True
if not plain:
return False
# Geometry must never change numeric evidence (1.5 -> 1 5, -5 -> 5).
# Compare a multiset so legitimate column reordering is still allowed.
numbers = re.compile(r"[+−-]?\d+(?:[.,:/]\d+)*(?:[%‰])?")
if Counter(numbers.findall(plain)) != Counter(numbers.findall(layout)):
return True
# A geometric split is useful for glued prose, but it can turn a simple
# table into one column of labels and another column of values. When the
# plain extractor exposes adjacent labelled numeric rows and the layout
# loses one of those values as an intact row, retain the plain version so
# the item/value correspondence survives.
plain_rows = _structured_numeric_rows(plain) if allow_structured_rows else []
if plain_rows:
layout_rows = _structured_numeric_rows(layout)
plain_values = Counter(_normalized_numeric_value(row[2]) for row in plain_rows)
layout_values = Counter(_normalized_numeric_value(row[2]) for row in layout_rows)
missing_values = sum((plain_values - layout_values).values())
if missing_values:
return True
if not missing_values and len(plain_rows) >= len(layout_rows):
# Both paths retained the values, but the plain stream keeps the
# table rows together with surrounding headings and prose. Prefer
# it when it is at least as structurally complete.
return True
n, single, punct_only = _layout_line_stats(layout)
if n == 0:
return True
if single / n >= 0.18 or punct_only / n >= 0.12:
return True
garbled = _layout_garbled_line_fraction(layout)
if garbled >= 0.20 and _layout_garbled_line_fraction(plain) < 0.08:
return True
if re.search(r"\[\w+,\s", plain) and re.search(
r"\[\w+\s+\w+\s+\d", layout
):
return True
# Title / lead sentence from plain should survive in layout.
for ln in plain.splitlines():
probe = ln.strip()
if len(probe) < 24:
continue
alnum = "".join(c for c in probe if c.isalnum())[:16]
if len(alnum) < 12:
continue
layout_alnum = "".join(c for c in layout if c.isalnum())
if alnum not in layout_alnum:
return True
break
return False
def _extract_layout_text(page, raw, *, return_metadata: bool = False):
"""Layout-aware extraction: reading order + headings + hidden-text filter.
Falls back to plain extraction on any failure so a single odd page never
breaks the document.
"""
textpage = None
filtered = False
table_hint = False
try:
textpage = page.get_textpage()
chars, width, filtered = _page_chars(
textpage, page, raw, return_filter_info=True
)
if not chars:
result = ""
else:
heights = [
c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0
]
scale = (statistics.median(heights) if heights else 1.0) or 1.0
table_hint = _looks_like_numeric_table_columns(chars, scale, width)
result = _chars_to_layout_markdown(chars, scale, width)
except Exception:
logger.debug("layout extraction failed; using plain text", exc_info=True)
result = _extract_page_text(page)
finally:
_close_pdfium_resource(textpage)
if return_metadata:
return result, filtered, table_hint
return result
# A visual line is cut where a gap wider than one glyph height coincides with
# a jump in the text stream: the gutter between two columns, not a word space.
LINE_SEGMENT_GAP_RATIO = 1.0
LINE_SEGMENT_STREAM_JUMP = 8
LINE_SEGMENT_MIN_CHARS = 3
def _split_line_at_gaps(line: dict) -> list:
"""Cut a visual line where it crosses from one column into another.
Narrow two-column gutters (ACL/IEEE papers leave ~16pt) and anything that
spans them (a title, a wide table) defeat the full-height gutter search,
so glyphs of both columns land in one line; each side must be placed and
boxed on its own. Columns are far apart in the text stream while glyphs of
one line are consecutive, which keeps wide justified word spaces intact.
"""
whole = [{"text": line["text"], "bbox": line["bbox"]}]
chars = [c for c in line["chars"] if not c["ch"].isspace()]
if len(chars) < 2 * LINE_SEGMENT_MIN_CHARS:
return whole
min_gap = max(max(c["y1"] - c["y0"] for c in chars), 1.0) * LINE_SEGMENT_GAP_RATIO
segments: list = [[chars[0]]]
for prev, cur in zip(chars, chars[1:]):
jump = abs(cur.get("i", 0) - prev.get("i", 0))
if cur["x0"] - prev["x1"] > min_gap and jump > LINE_SEGMENT_STREAM_JUMP:
segments.append([cur])
else:
segments[-1].append(cur)
if len(segments) == 1 or any(len(seg) < LINE_SEGMENT_MIN_CHARS for seg in segments):
return whole
out = []
for seg in segments:
text = _join_line_glyphs(seg)
if text.strip():
out.append({"text": text, "bbox": _chars_bbox(seg)})
return out or whole
def _page_line_boxes(page, raw) -> list:
"""Visual lines of a text page as ``{"text", "bbox"}`` in PDF points.
Lines are grouped per column so a two-column page does not fuse the
columns' glyphs into one line. Any failure yields no lines, which only
downgrades the page's locators to page numbers.
"""
textpage = None
try:
textpage = page.get_textpage()
# Tight glyph boxes put "a" and "d" at different heights, which splits
# one line of text into fragments by letter shape.
chars, width = _page_chars(textpage, page, raw, loose=True)
if not chars:
return []
heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]
scale = (statistics.median(heights) if heights else 1.0) or 1.0
lines = []
for col in _split_columns(chars, scale, width):
for ln in _group_lines_with_chars(col):
lines.extend(_split_line_at_gaps(ln))
return lines
except Exception:
logger.debug("line box extraction failed", exc_info=True)
return []
finally:
_close_pdfium_resource(textpage)
def _page_geometry(page):
try:
return PageGeometry(page.get_cropbox(), page.get_rotation())
except Exception:
logger.debug("page geometry unavailable", exc_info=True)
return None
def _effective_scale(page, scale: float, max_edge: int) -> float:
"""Reduce ``scale`` so the rendered long edge never exceeds ``max_edge`` px.
Some scanned PDFs declare enormous page boxes; rendering those at the raw
DPI scale produces 100+ MP JPEGs that exceed the gRPC message limit and are
far higher resolution than OCR needs.
"""
if max_edge <= 0:
return scale
width, height = page.get_size()
longest_pt = max(float(width), float(height))
if longest_pt <= 0:
return scale
return min(scale, max_edge / longest_pt)
def _render_page_to_jpeg(page, scale: float, quality: int, max_edge: int = 0) -> bytes:
bitmap = None
try:
bitmap = page.render(scale=_effective_scale(page, scale, max_edge))
img_obj = bitmap.to_pil()
if img_obj.mode != "RGB":
img_obj = img_obj.convert("RGB")
buf = io.BytesIO()
img_obj.save(buf, format="JPEG", quality=quality, optimize=True)
return buf.getvalue()
finally:
_close_pdfium_resource(bitmap)
# --- Parallel scanned-page rendering --------------------------------------
# pdfium is NOT thread-safe (concurrent get_page on one document crashes), so
# we parallelise across *processes*: each worker opens its own PdfDocument from
# a temp file and renders an assigned slice of pages. This turns the serial
# per-page render (the dominant cost for big scanned PDFs — hours on
# CPU-constrained containers) into a near-linear speedup.
# PDFium retains decoded page resources on some inputs until its document closes.
# ReadStream therefore restarts the renderer every fixed-size batch; the bound
# is independent of total page count while retaining useful parallel work.
PDF_STREAM_RENDER_BATCH_PAGES = 16
# Per-worker document handle, populated by the pool initializer.
_WORKER_RENDER_DOC = None
def _render_pool_init(pdf_path: str) -> None:
global _WORKER_RENDER_DOC
import pypdfium2 as pdfium
_WORKER_RENDER_DOC = pdfium.PdfDocument(pdf_path)
def _render_pool_task(args):
index, scale, quality, max_edge = args
page = _WORKER_RENDER_DOC[index]
try:
return index, _render_page_to_jpeg(page, scale, quality, max_edge)
finally:
_close_pdfium_resource(page)
def _select_mp_context():
"""Pick the safest available multiprocessing start method.
``forkserver`` forks workers from a clean, single-threaded server process,
avoiding the fork-in-a-multithreaded-process hazards of the gRPC server.
Falls back to ``fork`` and finally returns ``None`` (serial) when neither
is available (e.g. Windows/dev).
"""
import multiprocessing as mp
for method in ("forkserver", "fork"):
try:
return mp.get_context(method)
except ValueError:
continue
return None
def _render_pages_parallel(
content: bytes, indices: list, scale: float, quality: int, max_edge: int, workers: int
) -> dict | None:
"""Render ``indices`` in parallel. Returns ``{index: jpeg_bytes}`` or None.
Returns None to signal the caller to fall back to serial rendering (when
parallelism is disabled, only one page is requested, or no usable
multiprocessing start method exists).
"""
if workers <= 1 or len(indices) <= 1:
return None
ctx = _select_mp_context()
if ctx is None:
return None
import tempfile
from concurrent.futures import ProcessPoolExecutor
tmp_path = None
try:
with tempfile.NamedTemporaryFile(
prefix="docreader_render_", suffix=".pdf", delete=False
) as tmp:
tmp.write(content)
tmp_path = tmp.name
max_workers = min(workers, len(indices))
tasks = [(i, scale, quality, max_edge) for i in indices]
result: dict = {}
with ProcessPoolExecutor(
max_workers=max_workers,
mp_context=ctx,
initializer=_render_pool_init,
initargs=(tmp_path,),
) as ex:
for index, jpeg in ex.map(_render_pool_task, tasks, chunksize=4):
result[index] = jpeg
return result
except Exception:
logger.warning(
"parallel page rendering failed; falling back to serial",
exc_info=True,
)
return None
finally:
if tmp_path:
try:
os.unlink(tmp_path)
except OSError:
pass
def _write_pdf_temp_file(content: bytes) -> str:
import tempfile
path = None
try:
with tempfile.NamedTemporaryFile(
prefix="docreader_stream_", suffix=".pdf", delete=False
) as tmp:
path = tmp.name
tmp.write(content)
return path
except Exception:
if path is not None:
_unlink_pdf_temp_file(path)
raise
def _unlink_pdf_temp_file(path: str) -> None:
try:
os.unlink(path)
except FileNotFoundError:
pass
except OSError:
logger.warning("Failed to remove temporary PDF %s", path, exc_info=True)
def _iter_scanned_pages_from_open_pdf(
pdf,
indices: list[int],
base_name: str,
scale: float,
quality: int,
max_edge: int,
) -> Iterator[StreamedImage]:
"""Render one page at a time from an already-open PdfDocument."""
for index in indices:
with _PDFIUM_LOCK, parser_worker_limit(
"pdf_render", CONFIG.pdf_render_max_workers
):
page = pdf[index]
try:
image_bytes = _render_page_to_jpeg(page, scale, quality, max_edge)
finally:
_close_pdfium_resource(page)
yield StreamedImage(
ref_path=f"images/{base_name}_page_{index + 1}.jpg",
image_data=image_bytes,
)
def _iter_scanned_pages_serial_batch(
pdf_path: str,
indices: list[int],
base_name: str,
scale: float,
quality: int,
max_edge: int,
) -> Iterator[StreamedImage]:
import pypdfium2 as pdfium
with _PDFIUM_LOCK:
pdf = pdfium.PdfDocument(pdf_path)
try:
yield from _iter_scanned_pages_from_open_pdf(
pdf, indices, base_name, scale, quality, max_edge
)
finally:
with _PDFIUM_LOCK:
_close_pdfium_resource(pdf)
def _iter_scanned_pages_parallel_batch(
pdf_path: str,
indices: list[int],
base_name: str,
scale: float,
quality: int,
max_edge: int,
workers: int,
mp_context,
) -> Iterator[StreamedImage]:
from concurrent.futures import ProcessPoolExecutor
max_workers = min(workers, len(indices))
window_size = max_workers * 2
pending = deque()
executor = None
sent = False
fallback = False
# Hold the limiter until the executor has shut down, including when the
# consumer cancels mid-batch. Otherwise another request could start a new
# pool while the old worker processes still hold their PDFium documents.
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
try:
executor = ProcessPoolExecutor(
max_workers=max_workers,
mp_context=mp_context,
initializer=_render_pool_init,
initargs=(pdf_path,),
)
indices_iter = iter(indices)
for _ in range(window_size):
try:
index = next(indices_iter)
except StopIteration:
break
pending.append(
executor.submit(
_render_pool_task,
(index, scale, quality, max_edge),
)
)
while pending:
future = pending.popleft()
index, image_bytes = future.result()
try:
next_index = next(indices_iter)
except StopIteration:
pass
else:
pending.append(
executor.submit(
_render_pool_task,
(next_index, scale, quality, max_edge),
)
)
sent = True
yield StreamedImage(
ref_path=f"images/{base_name}_page_{index + 1}.jpg",
image_data=image_bytes,
)
except Exception:
if sent:
raise
logger.warning(
"parallel streamed PDF rendering failed before the first image; "
"falling back to serial rendering for this batch",
exc_info=True,
)
fallback = True
finally:
if executor is not None:
for future in pending:
future.cancel()
executor.shutdown(wait=True, cancel_futures=True)
pending.clear()
if fallback:
yield from _iter_scanned_pages_serial_batch(
pdf_path, indices, base_name, scale, quality, max_edge
)
def _iter_scanned_pages_from_path(
pdf_path: str,
indices: list[int],
base_name: str,
scale: float,
quality: int,
max_edge: int,
workers: int,
) -> Iterator[StreamedImage]:
"""Yield images in fixed-size batches to cap PDFium and result memory."""
for start in range(0, len(indices), PDF_STREAM_RENDER_BATCH_PAGES):
batch = indices[start : start + PDF_STREAM_RENDER_BATCH_PAGES]
mp_context = _select_mp_context() if workers > 1 and len(batch) > 1 else None
if workers > 1 and len(batch) > 1 and mp_context is not None:
yield from _iter_scanned_pages_parallel_batch(
pdf_path,
batch,
base_name,
scale,
quality,
max_edge,
workers,
mp_context,
)
else:
yield from _iter_scanned_pages_serial_batch(
pdf_path, batch, base_name, scale, quality, max_edge
)
def _render_scanned_pages(
pdf, content: bytes, indices: list, scale: float, quality: int, max_edge: int
) -> dict:
"""Render the given (scanned) page indices to JPEG bytes.
Tries process-parallel rendering first (big win for large scanned PDFs),
transparently falling back to serial rendering on the already-open ``pdf``
handle when parallelism is unavailable or fails.
"""
parallel = _render_pages_parallel(
content, indices, scale, quality, max_edge, CONFIG.pdf_render_parallelism
)
if parallel is not None:
return parallel
out: dict = {}
for i in indices:
page = pdf[i]
try:
out[i] = _render_page_to_jpeg(page, scale, quality, max_edge)
finally:
_close_pdfium_resource(page)
return out
def _encode_rendered_page_images(rendered: dict, base_name: str) -> dict:
"""Encode rendered pages while releasing each raw JPEG immediately.
``_render_scanned_pages`` returns raw JPEG bytes because the render workers
need that representation. Keeping the raw mapping alive while building a
second base64 mapping temporarily doubles the image memory for a scanned
document. Pop each page before encoding it so the parser retains only the
representation that the gRPC response needs.
"""
encoded = {}
for index in sorted(rendered):
image_bytes = rendered.pop(index)
ref_path = f"images/{base_name}_page_{index + 1}.jpg"
encoded[ref_path] = base64.b64encode(image_bytes).decode("utf-8")
return encoded
def _select_embedded_images(
meta: list,
num_text_pages: int,
*,
min_pixels: int = EMBED_MIN_PIXELS,
min_area_ratio: float = EMBED_MIN_AREA_RATIO,
repeat_frac: float = EMBED_REPEAT_PAGE_FRAC,
max_images: int = EMBED_MAX_IMAGES,
) -> list:
"""Decide which embedded-image candidates to keep (pure function).
``meta`` is a list of dicts with keys ``page``, ``width``, ``height``,
``area_ratio`` and ``hash``. Returns the indices (into ``meta``) to keep,
after filtering by size, page-area share, cross-page repetition (logos /
watermarks), exact in-page duplicates and a hard count cap.
"""
from collections import defaultdict
hash_pages = defaultdict(set)
for m in meta:
hash_pages[m["hash"]].add(m["page"])
repeat_threshold = max(2, int(num_text_pages * repeat_frac)) if num_text_pages else 2
banned = {h for h, pages in hash_pages.items() if len(pages) >= repeat_threshold}
kept: list = []
seen = set()
for idx, m in enumerate(meta):
if m["area_ratio"] < min_area_ratio:
continue
if m["width"] < min_pixels or m["height"] < min_pixels:
continue
if m["hash"] in banned:
continue
key = (m["page"], m["hash"])
if key in seen:
continue
seen.add(key)
kept.append(idx)
if len(kept) >= max_images:
break
return kept
def _decode_embedded_image_pil(obj):
"""Decode an embedded image the way a PDF viewer displays it.
``get_bitmap()`` returns the raw image plane and ignores any /SMask (soft
transparency mask). Figures exported from plotting tools often store a
black base plane with the visible content in the mask, so the raw plane
decodes to an all-black rectangle. ``get_bitmap(render=True)`` renders the
object through pdfium's imaging pipeline and carries the mask as an alpha
channel; compositing over white then yields what the page looks like
(JPEG cannot keep alpha, and pages render on white).
The ``PdfBitmap`` is kept alive until pixels are copied: ``to_pil()``
shares the pdfium buffer for RGBA/RGBX/L, and releasing the handle first
would use-after-free. Opaque RGB/L images keep their mode so grayscale
hashes and JPEGs stay single-channel.
"""
from PIL import Image
bitmap = None
try:
try:
bitmap = obj.get_bitmap(render=True)
except Exception:
logger.warning(
"get_bitmap(render=True) failed; falling back to raw decode",
exc_info=True,
)
_close_pdfium_resource(bitmap)
bitmap = obj.get_bitmap()
pil = bitmap.to_pil()
if pil.mode in ("RGBA", "LA", "PA"):
src_mode = pil.mode
rgba = pil.convert("RGBA")
if rgba.getchannel("A").getextrema()[0] < 255:
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
return Image.alpha_composite(background, rgba).convert("RGB")
if src_mode == "LA":
return rgba.convert("L")
return rgba.convert("RGB")
if pil.mode in ("RGB", "L"):
return pil.copy()
return pil.convert("RGB")
finally:
_close_pdfium_resource(bitmap)
def _extract_embedded_images(pdf, classes, raw, base_name: str, quality: int) -> dict:
"""Extract filtered embedded figures from native text pages.
Returns ``{page_index: [(ref_path, base64_jpeg, y_top), ...]}`` ordered so
callers can place figures after the page text in top-to-bottom order.
"""
import hashlib
text_indices = [i for i, c in enumerate(classes) if c == "text"]
if not text_indices:
return {}
candidates: list = []
meta: list = []
for i in text_indices:
page = pdf[i]
try:
width, height = page.get_size()
page_area = float(width) * float(height)
if page_area <= 0:
continue
for object_index, obj in enumerate(page.get_objects()):
if obj.type != raw.FPDF_PAGEOBJ_IMAGE:
continue
try:
left, bottom, right, top = obj.get_bounds()
except Exception:
continue
area_ratio = abs((right - left) * (top - bottom)) / page_area
if area_ratio < EMBED_MIN_AREA_RATIO:
continue # cheap skip before decoding (logos/decorations)
try:
pil = _decode_embedded_image_pil(obj)
except Exception:
continue
try:
content_hash = hashlib.md5(pil.tobytes()).hexdigest()
candidates.append((i, object_index, top))
meta.append(
{
"page": i,
"width": pil.width,
"height": pil.height,
"area_ratio": area_ratio,
"hash": content_hash,
}
)
finally:
pil.close()
finally:
_close_pdfium_resource(page)
kept_idx = _select_embedded_images(meta, len(text_indices))
if not kept_idx:
return {}
from collections import defaultdict
result: dict = defaultdict(list)
per_page_count: dict = defaultdict(int)
max_edge = CONFIG.pdf_render_max_edge
for idx in kept_idx:
# Reopen only selected figures so rejected candidates do not retain pixels.
page_i, object_index, y_top = candidates[idx]
page = pdf[page_i]
pil = None
try:
for index, obj in enumerate(page.get_objects()):
if index == object_index:
pil = _decode_embedded_image_pil(obj)
break
if pil is None:
raise ValueError("Selected embedded image is no longer available")
if pil.mode not in ("RGB", "L"):
converted = pil.convert("RGB")
pil.close()
pil = converted
if max_edge > 0 and max(pil.size) > max_edge:
ratio = max_edge / max(pil.size)
resized = pil.resize(
(max(1, int(pil.width * ratio)), max(1, int(pil.height * ratio)))
)
pil.close()
pil = resized
with io.BytesIO() as buf:
pil.save(buf, format="JPEG", quality=quality, optimize=True)
per_page_count[page_i] += 1
fname = f"{base_name}_p{page_i+1}_img{per_page_count[page_i]}.jpg"
ref_path = f"images/{fname}"
result[page_i].append(
(ref_path, base64.b64encode(buf.getvalue()).decode("utf-8"), y_top)
)
finally:
if pil is not None:
pil.close()
_close_pdfium_resource(page)
# Top-to-bottom within each page (PDF y grows upward, so larger y first).
for page_i in result:
result[page_i].sort(key=lambda item: item[2], reverse=True)
return result
def _strip_repeating_lines(texts: list, classes: list) -> list:
"""Remove running headers/footers that repeat across most text pages.
Conservative: only the first/last non-empty line of each text page is a
candidate, the line must be short, and it must appear on at least 60% of the
text pages (and there must be enough pages to judge). Mirrors DeepDoc's
cross-page "garbage set" idea without risking removal of real content.
"""
from collections import Counter
text_indices = [i for i, c in enumerate(classes) if c == "text"]
if len(text_indices) < 4:
return list(texts)
counter: Counter = Counter()
for i in text_indices:
lines = [ln.strip() for ln in texts[i].splitlines() if ln.strip()]
if not lines:
continue
for edge in {lines[0], lines[-1]}:
if len(edge) <= 80:
counter[edge] += 1
threshold = max(2, int(len(text_indices) * 0.6))
repeating = {line for line, count in counter.items() if count >= threshold}
if not repeating:
return list(texts)
cleaned = []
for i, text in enumerate(texts):
if classes[i] != "text":
cleaned.append(text)
continue
kept = [ln for ln in text.splitlines() if ln.strip() not in repeating]
cleaned.append("\n".join(kept))
return cleaned
class PDFScannedParser(BaseParser):
"""Render every PDF page to a JPEG image.
Used as a robust last-resort fallback and for image-only PDFs. The Go App
performs OCR on the extracted page images.
"""
def parse_into_text(self, content: bytes) -> Document:
import pypdfium2 as pdfium
images = {}
markdown_lines = []
base_name = os.path.splitext(self.file_name or "document")[0]
logger.info(
"PDFScannedParser: Rendering PDF pages to JPEG images for %s",
self.file_name,
)
try:
# pdfium lock first, then the render worker limit — same order as
# PDFParser._route so the two never deadlock against each other.
with _PDFIUM_LOCK, parser_worker_limit(
"pdf_render", CONFIG.pdf_render_max_workers
):
pdf = pdfium.PdfDocument(content)
try:
page_count = len(pdf)
scale = max(1, CONFIG.pdf_render_dpi) / 72
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
rendered = _render_scanned_pages(
pdf,
content,
list(range(page_count)),
scale,
quality,
CONFIG.pdf_render_max_edge,
)
finally:
_close_pdfium_resource(pdf)
source_blocks = []
offset = 0
images.update(_encode_rendered_page_images(rendered, base_name))
for i in range(page_count):
page_filename = f"{base_name}_page_{i+1}.jpg"
ref_path = f"images/{page_filename}"
line = f""
if markdown_lines:
offset += 2
markdown_lines.append(line)
source_blocks.append(
{
"start": offset,
"end": offset + len(line),
"locator": {"type": "pdf", "page": i + 1, "mapping": "exact"},
}
)
offset += len(line)
text = "\n\n".join(markdown_lines)
return Document(
content=text,
images=images,
metadata={
"image_source_type": "scanned_pdf",
"page_count": page_count,
},
source_blocks=source_blocks,
)
except Exception as e:
logger.exception("PDFScannedParser failed to parse PDF: %s", e)
raise e
def parse_stream(self, content: bytes) -> ParsedDocumentStream:
"""Return metadata first and lazily render each scanned page."""
import pypdfium2 as pdfium
base_name = os.path.splitext(self.file_name or "document")[0]
with _PDFIUM_LOCK:
pdf = pdfium.PdfDocument(content)
try:
page_count = len(pdf)
finally:
_close_pdfium_resource(pdf)
markdown_lines = []
source_blocks = []
offset = 0
for index in range(page_count):
page_filename = f"{base_name}_page_{index + 1}.jpg"
ref_path = f"images/{page_filename}"
line = f""
if markdown_lines:
offset += 2
markdown_lines.append(line)
source_blocks.append(
{
"start": offset,
"end": offset + len(line),
"locator": {"type": "pdf", "page": index + 1, "mapping": "exact"},
}
)
offset += len(line)
document = Document(
content="\n\n".join(markdown_lines),
metadata={"image_source_type": "scanned_pdf", "page_count": page_count},
source_blocks=source_blocks,
)
image_count = page_count
if page_count == 0:
return ParsedDocumentStream(document=document, image_count=0)
scale = max(1, CONFIG.pdf_render_dpi) / 72
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
indices = list(range(page_count))
pdf_path = _write_pdf_temp_file(content)
return ParsedDocumentStream(
document=document,
image_count=image_count,
images=_iter_scanned_pages_from_path(
pdf_path,
indices,
base_name,
scale,
quality,
CONFIG.pdf_render_max_edge,
CONFIG.pdf_render_parallelism,
),
_cleanup=lambda: _unlink_pdf_temp_file(pdf_path),
)
class PDFParser(BaseParser):
"""Per-page router between native text extraction and scanned rendering.
For each page:
* native text page -> keep its text layer (fast, pypdfium2)
* scanned page -> render to JPEG, tag ``image_source_type=scanned_pdf``
so the Go App OCRs it
Hybrid documents interleave both in reading order. On any unexpected error
the parser falls back to rendering all pages as images (safe last resort).
Force-scanned mode (``pdf_force_scanned=true`` override or
``DOCREADER_PDF_FORCE_SCANNED=true`` env) skips classification and
renders every page as an image.
"""
def __init__(self, file_name: str = "", file_type=None, **kwargs):
# Capture per-upload override before BaseParser consumes kwargs.
raw = kwargs.pop("pdf_force_scanned", None)
super().__init__(file_name=file_name, file_type=file_type, **kwargs)
# Priority: per-upload override > global env > default (False).
if raw is not None:
self._force_scanned = str(raw).strip().lower() in {
"1", "true", "yes", "y", "on",
}
else:
self._force_scanned = FORCE_SCANNED_PDF
def parse_into_text(self, content: bytes) -> Document:
# Force-scanned short-circuit: render every page as an image.
if self._force_scanned:
logger.info(
"PDFParser: force scanned mode enabled for %s",
self.file_name,
)
doc = PDFScannedParser(
file_name=self.file_name, file_type=self.file_type
).parse_into_text(content)
# Align metadata fields with automatic scanned route
page_count = doc.metadata.get("page_count", 0)
doc.metadata.update({
"scanned_page_count": page_count,
"text_page_count": 0,
"embedded_image_count": 0,
"vector_figure_count": 0,
})
return doc
try:
return self._route(content)
except Exception:
logger.exception(
"PDFParser: per-page routing failed for %s; "
"falling back to full image rendering",
self.file_name,
)
return PDFScannedParser(
file_name=self.file_name, file_type=self.file_type
).parse_into_text(content)
def parse_stream(self, content: bytes) -> ParsedDocumentStream:
"""Prepare PDF metadata, then render scanned pages only as consumed."""
if self._force_scanned:
stream = PDFScannedParser(
file_name=self.file_name, file_type=self.file_type
).parse_stream(content)
page_count = stream.document.metadata.get("page_count", 0)
stream.document.metadata.update(
{
"scanned_page_count": page_count,
"text_page_count": 0,
"embedded_image_count": 0,
"vector_figure_count": 0,
}
)
return stream
try:
with _PDFIUM_LOCK:
document, scanned_indices = self._route_locked(
content, include_scanned_images=False
)
except Exception:
logger.exception(
"PDFParser: per-page streaming preparation failed for %s; "
"falling back to full image rendering",
self.file_name,
)
return PDFScannedParser(
file_name=self.file_name, file_type=self.file_type
).parse_stream(content)
image_count = len(document.images or {}) + len(scanned_indices)
if not scanned_indices:
return ParsedDocumentStream(document=document, image_count=image_count)
scale = max(1, CONFIG.pdf_render_dpi) / 72
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
base_name = os.path.splitext(self.file_name or "document")[0]
# Use a temporary source file for every stream mode so each fixed-size
# batch can close its PdfDocument before the next batch starts. This
# bounds PDFium's decoded-page cache independently of document length.
pdf_path = _write_pdf_temp_file(content)
return ParsedDocumentStream(
document=document,
image_count=image_count,
images=_iter_scanned_pages_from_path(
pdf_path,
scanned_indices,
base_name,
scale,
quality,
CONFIG.pdf_render_max_edge,
CONFIG.pdf_render_parallelism,
),
_cleanup=lambda: _unlink_pdf_temp_file(pdf_path),
)
def _route(self, content: bytes) -> Document:
# Serialize all pdfium work: see _PDFIUM_LOCK. Holding it for the whole
# route (both the text pass and the render pass) is what prevents the
# concurrent-upload deadlock; the trailing markdown assembly is cheap
# pure-Python so keeping it inside the lock costs nothing meaningful.
with _PDFIUM_LOCK:
document, _scanned_indices = self._route_locked(content)
return document
def _route_locked(
self, content: bytes, *, include_scanned_images: bool = True
) -> tuple[Document, list[int]]:
import pypdfium2 as pdfium
import pypdfium2.raw as pdfium_r
base_name = os.path.splitext(self.file_name or "document")[0]
scale = max(1, CONFIG.pdf_render_dpi) / 72
quality = _normalize_image_quality(CONFIG.pdf_jpeg_quality)
pdf = pdfium.PdfDocument(content)
images: dict = {}
try:
page_count = len(pdf)
# Pass 1: cheap text extraction + image-area classification.
texts: list = []
classes: list = []
vector_clips: dict = {}
line_boxes: dict = {}
geometries: dict = {}
for i in range(page_count):
page = pdf[i]
try:
geometries[i] = _page_geometry(page)
plain = _extract_page_text(page)
ratio = _page_image_area_ratio(page, pdfium_r)
cls = _classify_page(ratio, len(plain.strip()))
# Layout reconstruction only pays off (and is only spent) on
# native text pages; scanned pages are rendered, not read.
if cls == "text" and LAYOUT_ORDERING:
plain_well_formed = _plain_is_well_formed(plain)
if plain_well_formed and not _page_has_filtered_text(
page, pdfium_r
):
text = plain
else:
layout, filtered_text, table_hint = _extract_layout_text(
page, pdfium_r, return_metadata=True
)
if filtered_text:
# The plain layer may contain hidden or
# off-page prompt-injection text. Once the
# layout path proves that filtering occurred,
# never re-introduce that layer as a fallback.
text = layout
elif layout and not _should_prefer_plain(
plain, layout, allow_structured_rows=table_hint
):
text = layout
else:
text = plain
else:
text = plain
if cls == "text":
clips = _extract_vector_figure_clips(
page,
i,
plain,
pdfium_r,
base_name,
scale,
quality,
CONFIG.pdf_render_max_edge,
)
if clips:
vector_clips[i] = clips
for ref_path, b64, _y, _cap in clips:
images[ref_path] = b64
text = _postprocess_pdf_text(text)
if cls == "text" and SOURCE_BOXES:
line_boxes[i] = _page_line_boxes(page, pdfium_r)
if cls == "text" and vector_clips.get(i):
text = _inject_figure_markdown_before_captions(
text, vector_clips[i]
)
finally:
_close_pdfium_resource(page)
texts.append(text)
classes.append(cls)
texts = _strip_repeating_lines(texts, classes)
scanned_indices = [i for i, c in enumerate(classes) if c == "scanned"]
# Pass 2: render only the scanned pages (heavy work, rate-limited).
# ReadStream defers this work until after its metadata frame and
# renders pages incrementally; unary Read keeps the materialized
# response path for compatibility.
if scanned_indices and include_scanned_images:
with parser_worker_limit("pdf_render", CONFIG.pdf_render_max_workers):
rendered = _render_scanned_pages(
pdf,
content,
scanned_indices,
scale,
quality,
CONFIG.pdf_render_max_edge,
)
images.update(_encode_rendered_page_images(rendered, base_name))
# Pass 3: extract embedded figures from native text pages so the Go
# App can OCR/caption them (logos/watermarks/tiny images filtered).
embedded: dict = {}
if EXTRACT_EMBEDDED_IMAGES:
embedded = _extract_embedded_images(
pdf, classes, pdfium_r, base_name, quality
)
for refs in embedded.values():
for ref_path, b64, _y in refs:
images[ref_path] = b64
finally:
_close_pdfium_resource(pdf)
# Assemble markdown in reading order, recording where each page's
# text lands so citations can point back at the page (and paragraph).
embedded_count = 0
vector_figure_count = 0
blocks = []
source_blocks = []
offset = 0
def place(part: str) -> int:
nonlocal offset
if blocks:
offset += 2 # the "\n\n" separator
start = offset
blocks.append(part)
offset += len(part)
return start
for i in range(page_count):
page_only = {"type": "pdf", "page": i + 1, "mapping": "exact"}
if classes[i] == "scanned":
page_filename = f"{base_name}_page_{i+1}.jpg"
part = f""
start = place(part)
source_blocks.append(
{"start": start, "end": start + len(part), "locator": page_only}
)
else:
stripped = texts[i].strip()
if stripped:
start = place(stripped)
source_blocks.extend(
page_blocks(
stripped,
start,
i + 1,
line_boxes.get(i) or [],
geometries.get(i),
)
)
vector_figure_count += len(vector_clips.get(i, []))
page_images = list(embedded.get(i, []))
page_images.sort(key=lambda item: item[2], reverse=True)
for ref_path, _b64, _y in page_images:
fname = os.path.basename(ref_path)
part = f""
start = place(part)
source_blocks.append(
{"start": start, "end": start + len(part), "locator": dict(page_only)}
)
embedded_count += 1
# Every part is already stripped, so the join needs no trimming and
# the recorded offsets stay exact.
content_text = "\n\n".join(blocks)
metadata = {
"page_count": page_count,
"scanned_page_count": len(scanned_indices),
"text_page_count": page_count - len(scanned_indices),
"embedded_image_count": embedded_count,
"vector_figure_count": vector_figure_count,
"image_source_type": "scanned_pdf" if scanned_indices else "pdf_text_layer",
}
logger.info(
"PDFParser: %s -> %d pages (%d scanned, %d text), "
"embedded_images=%d, content_len=%d",
self.file_name,
page_count,
len(scanned_indices),
page_count - len(scanned_indices),
embedded_count,
len(content_text),
)
document = Document(
content=content_text,
images=images,
metadata=metadata,
source_blocks=source_blocks,
)
return document, scanned_indices