Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(docreader): preserve numeric rows in PDF text extraction (#3228)

dvd233735eeb1f2db4
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.py
index fe7d2e6cc..c60187698 100644
--- a/docreader/parser/pdf_parser.py
+++ b/docreader/parser/pdf_parser.py
@@ -162,6 +162,22 @@ _FIGURE_CAPTION_RE = re.compile(r"^Figure\s+\d+\b", re.IGNORECASE)
_FIGURE_CAPTION_SEARCH_RE = re.compile(r"\bFigure\s+(\d+)\b", re.IGNORECASE)
_ARXIV_LINE_RE = re.compile(r"^arXiv:\s*\S+", re.IGNORECASE)
_PAGE_NUM_LINE_RE = re.compile(r"^\d{1,3}$")
+_PAGE_CONTEXT_RE = re.compile(
+ r"(?:\bpage\b|\bp\.\s*\d+\b|\b\d+\s*(?:of|/)\s*\d+\b)",
+ re.IGNORECASE,
+)
+_CHART_CONTEXT_RE = re.compile(
+ r"(?:\bFigure\s+\d+\b|\biter\.\s*\(1e4\)|"
+ r"\b(?:training|test)\s+error\b|\b\d+-layer\b)",
+ re.IGNORECASE,
+)
+_NUMERIC_ROW_RE = re.compile(
+ r"^(?P<label>[^\d\n]+?)\s+"
+ r"(?P<value>[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?)$"
+)
+_NUMERIC_VALUE_LINE_RE = re.compile(
+ r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"
+)
def _close_pdfium_resource(resource) -> None:
@@ -256,7 +272,19 @@ def _strip_chart_text_debris(text: str) -> str:
_is_chart_debris_line(lines[j]) or not lines[j].strip()
):
j += 1
- if j - i >= 3:
+ # Numeric-only lines are also a common representation for table
+ # values. Only remove a run when a nearby chart/figure marker
+ # provides positive evidence that it is axis or legend text.
+ context_start = max(0, i - 1)
+ context_end = min(len(lines), j + 1)
+ has_chart_context = any(
+ _CHART_CONTEXT_RE.search(lines[k])
+ for k in range(context_start, context_end)
+ )
+ table_context = _tail_looks_like_numeric_table(
+ lines[max(0, i - 2) : j]
+ )
+ if j - i >= 3 and has_chart_context and not table_context:
i = j
continue
out.append(lines[i])
@@ -264,14 +292,46 @@ def _strip_chart_text_debris(text: str) -> str:
return "\n".join(out)
+def _is_probable_page_num_line(lines: list, index: int) -> bool:
+ """Return whether a standalone number is likely a page marker.
+
+ A numeric-only line in the middle of a page is often a table value. The
+ old unconditional filter removed those values, so page-number cleanup is
+ deliberately limited to an explicitly labelled edge line or a line adjacent
+ to an arXiv header. Keeping an ambiguous number is safer than silently
+ losing document data.
+ """
+ if index < 0 or index >= len(lines):
+ return False
+ if not _PAGE_NUM_LINE_RE.fullmatch(lines[index].strip()):
+ return False
+ nonempty = [i for i, line in enumerate(lines) if line.strip()]
+ if not nonempty:
+ return False
+ pos = nonempty.index(index)
+ neighbors = []
+ if pos:
+ neighbors.append(nonempty[pos - 1])
+ if pos + 1 < len(nonempty):
+ neighbors.append(nonempty[pos + 1])
+ if any(_ARXIV_LINE_RE.match(lines[i].strip()) for i in neighbors):
+ return True
+ # Keep an edge number unless a nearby line explicitly looks like a page
+ # label (``Page 3 of 8`` / ``3 / 8``). Ambiguous edge numbers can still be
+ # legitimate first/last values in a table.
+ if index in (nonempty[0], nonempty[-1]):
+ return any(_PAGE_CONTEXT_RE.search(lines[i]) for i in neighbors)
+ return False
+
+
def _strip_arxiv_and_page_num_lines(text: str) -> str:
lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")
kept: list = []
- for ln in lines:
+ for i, ln in enumerate(lines):
t = ln.strip()
if _ARXIV_LINE_RE.match(t):
continue
- if _PAGE_NUM_LINE_RE.match(t):
+ if _is_probable_page_num_line(lines, i):
continue
if "arXiv:" in ln:
ln = re.sub(r"\s*arXiv:\s*\S+\s*(?:\[[^\]]+\])?\s*[^\n]*", "", ln).strip()
@@ -288,6 +348,11 @@ def _strip_lines_above_figure_captions(text: str) -> str:
for ln in lines:
if _line_has_figure_caption(ln):
while out and _is_figure_interior_line(out[-1]):
+ # A table can legitimately end immediately before a figure
+ # caption. Preserve a tail with two labelled numeric rows;
+ # diagram labels do not have this row-shaped signal.
+ if _tail_looks_like_numeric_table(out):
+ break
out.pop()
out.append(ln)
else:
@@ -295,6 +360,41 @@ def _strip_lines_above_figure_captions(text: str) -> str:
return "\n".join(out)
+def _tail_looks_like_numeric_table(lines: list) -> bool:
+ """Detect labelled or alternating label/value rows at the text tail."""
+ tail = [line for line in lines[-10:] if line.strip()]
+ rows = _structured_numeric_rows("\n".join(tail))
+ if rows and rows[-1][0] == len(tail) - 1:
+ return True
+
+ numeric = re.compile(
+ r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"
+ )
+ for index, header in enumerate(tail[:-2]):
+ if (
+ header.strip()
+ and any(char.isalpha() for char in header)
+ and all(numeric.fullmatch(value.strip()) for value in tail[index + 1 :])
+ ):
+ return True
+ pairs = 0
+ index = len(tail) - 1
+ while index >= 1:
+ label = tail[index - 1].strip()
+ value = tail[index].strip()
+ if (
+ label
+ and any(char.isalpha() for char in label)
+ and len(label) <= 60
+ and numeric.fullmatch(value)
+ ):
+ pairs += 1
+ index -= 2
+ else:
+ break
+ return pairs >= 2
+
+
def _is_body_paragraph_line(text: str) -> bool:
t = text.strip()
if len(t) < 48:
@@ -639,17 +739,24 @@ def _point_in_boxes(x: float, y: float, boxes: list) -> bool:
return False
-def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:
+def _page_chars(
+ textpage, page, raw, loose: bool = False, *, return_filter_info: bool = False
+) -> tuple:
"""Return ``(chars, page_width)`` with hidden/off-page glyphs filtered.
Working at the glyph level (instead of pdfium rect segments) keeps mixed
CJK + Latin/number lines in their true left-to-right order, which the
rect-level ``get_text_bounded`` API scrambles. ``loose`` boxes span the
font's ascent to descent, so every glyph of a line shares one height.
+
+ When ``return_filter_info`` is true, a third value reports whether any
+ source glyph was intentionally filtered as hidden or off-page. Callers
+ can then avoid falling back to the unfiltered plain text when a PDF
+ contains hidden or off-page text.
"""
n = textpage.count_chars()
if n <= 0:
- return [], 0.0
+ return ([], 0.0, False) if return_filter_info else ([], 0.0)
# Glyph coordinates are unrotated PDF user space. get_size() reports
# rotated dimensions and would discard valid text on cropped/rotated pages.
try:
@@ -661,25 +768,67 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:
invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else []
chars: list = []
+ source_count = 0
+ filtered_count = 0
for i in range(n):
try:
- left, bottom, right, top = textpage.get_charbox(i, loose=loose)
+ try:
+ left, bottom, right, top = textpage.get_charbox(i, loose=loose)
+ except TypeError:
+ # Keep compatibility with the small PDFium test doubles and
+ # older pypdfium2 releases that do not expose ``loose``.
+ left, bottom, right, top = textpage.get_charbox(i)
except Exception:
+ source_count += 1
continue
ch = textpage.get_text_range(i, 1)
if ch in ("\r", "\n"):
continue
+ source_count += 1
x0, x1 = (left, right) if left <= right else (right, left)
y0, y1 = (bottom, top) if bottom <= top else (top, bottom)
if FILTER_HIDDEN_TEXT:
if x1 < crop_left or x0 > crop_right or y1 < crop_bottom or y0 > crop_top:
+ filtered_count += 1
continue # off-page glyph
if invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible):
+ filtered_count += 1
continue # covered by an invisible text object
chars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch, "i": i})
+ if return_filter_info:
+ return chars, width, filtered_count > 0
return chars, width
+def _page_has_invisible_text(page, raw) -> bool:
+ """Whether ``page`` contains text objects hidden from normal rendering."""
+ if not FILTER_HIDDEN_TEXT:
+ return False
+ try:
+ return bool(_collect_invisible_boxes(page, raw))
+ except Exception:
+ # A failed probe should not change the existing plain-text route. The
+ # full glyph-level probe still runs whenever layout extraction is used.
+ return False
+
+
+def _page_has_filtered_text(page, raw) -> bool:
+ """Whether layout filtering would discard hidden or off-page glyphs."""
+ if not FILTER_HIDDEN_TEXT:
+ return False
+ if _page_has_invisible_text(page, raw):
+ return True
+ textpage = None
+ try:
+ textpage = page.get_textpage()
+ _, _, filtered = _page_chars(textpage, page, raw, return_filter_info=True)
+ return filtered
+ except Exception:
+ return False
+ finally:
+ _close_pdfium_resource(textpage)
+
+
def _find_split(items: list, axis: str, min_gap: float):
"""Return a coordinate at the widest clean gap on ``axis`` ('x'), or None.
@@ -757,6 +906,21 @@ def _is_artifact_column(chars: list, width: float) -> bool:
def _filter_reading_columns(chars: list, scale: float, width: float) -> list:
"""Split into columns and drop margin / watermark strips."""
cols = _split_columns(chars, scale, width)
+ table_pair = _find_numeric_table_columns(chars, scale, width, cols)
+ if table_pair is not None:
+ # Keep aligned table cells in one visual stream so rows such as
+ # ``Aster 124`` are not emitted as a label block followed by a value
+ # block, while still dropping unrelated margin/artifact columns.
+ # ``chars`` has already passed hidden/off-page filtering.
+ value_col, label_col = table_pair
+ table_cols = [
+ c
+ for c in cols
+ if not _is_artifact_column(c, width) or c is value_col or c is label_col
+ ]
+ if table_cols:
+ return [[char for column in table_cols for char in column]]
+ return [chars]
kept = [c for c in cols if not _is_artifact_column(c, width)]
if kept:
return kept
@@ -766,6 +930,106 @@ def _filter_reading_columns(chars: list, scale: float, width: float) -> list:
return cols
+def _is_numeric_value_line(text: str) -> bool:
+ t = (text or "").strip()
+ return bool(
+ _NUMERIC_VALUE_LINE_RE.fullmatch(t)
+ or (
+ re.fullmatch(r"[\d\s./()%+-]+", t)
+ and any(char.isdigit() for char in t)
+ )
+ )
+
+
+def _looks_like_numeric_table_columns(
+ chars: list, scale: float, width: float, columns: list | None = None
+) -> bool:
+ """Identify a narrow, aligned value column before artifact filtering.
+
+ A real two-column page should remain column-wise. A simple table often
+ looks different: one narrow column contains a header and mostly numeric
+ values, and its baselines line up with labels in a wider column. This
+ signal lets the caller choose the plain extractor only for that table-like
+ shape instead of turning every numeric mismatch into a page-wide fallback.
+ """
+ return _find_numeric_table_columns(chars, scale, width, columns) is not None
+
+
+def _find_numeric_table_columns(
+ chars: list, scale: float, width: float, columns: list | None = None
+) -> tuple[list, list] | None:
+ """Return the aligned value/label columns when a numeric table is found."""
+ if not chars or width <= 0:
+ return None
+ cols = columns if columns is not None else _split_columns(chars, scale, width)
+ if len(cols) < 2:
+ return None
+
+ for value_col in cols:
+ span_ratio = _column_x_span(value_col) / width
+ if span_ratio > 0.22:
+ continue
+ value_lines = _group_lines_with_chars(value_col)
+ if len(value_lines) < 2:
+ continue
+ numeric_lines = [
+ line for line in value_lines if _is_numeric_value_line(line["text"])
+ ]
+ if not numeric_lines or len(numeric_lines) / len(value_lines) < 0.4:
+ continue
+ # Require a textual header/label in the narrow column. This keeps a
+ # vertical numeric watermark from being mistaken for a table.
+ text_lines = [
+ line
+ for line in value_lines
+ if not _is_numeric_value_line(line["text"])
+ and any(char.isalpha() for char in line["text"])
+ ]
+ if not text_lines:
+ continue
+
+ value_ys = [
+ (line["bbox"][1] + line["bbox"][3]) / 2 for line in numeric_lines
+ ]
+ for label_col in cols:
+ if label_col is value_col:
+ continue
+ label_lines = _group_lines_with_chars(label_col)
+ label_ys = [
+ (line["bbox"][1] + line["bbox"][3]) / 2
+ for line in label_lines
+ ]
+ label_text_ys = [
+ (line["bbox"][1] + line["bbox"][3]) / 2
+ for line in label_lines
+ if any(char.isalpha() for char in line["text"])
+ ]
+ tolerance = max(scale * 0.75, 2.0)
+ header_ys = [
+ (line["bbox"][1] + line["bbox"][3]) / 2 for line in text_lines
+ ]
+ header_aligned = any(
+ abs(y - other_y) <= tolerance
+ for y in header_ys
+ for other_y in label_text_ys
+ )
+ if not header_aligned:
+ continue
+ aligned = sum(
+ any(abs(y - other_y) <= tolerance for other_y in label_ys)
+ for y in value_ys
+ )
+ # A table may contain an empty value cell. In that case the one
+ # remaining numeric row still carries enough evidence when its
+ # header and the label column share the same baseline.
+ required_alignment = (
+ 1 if len(value_ys) == 1 else max(2, (len(value_ys) + 1) // 2)
+ )
+ if aligned >= required_alignment:
+ return value_col, label_col
+ return None
+
+
def _merge_orphan_punctuation_lines(lines: list) -> list:
"""Attach lines that are only punctuation to the previous visual line.
@@ -813,7 +1077,19 @@ def _join_line_glyphs(ln_sorted: list) -> str:
if not ch.isspace() or (parts and not parts[-1].endswith(" ")):
parts.append(ch)
continue
- if cur["x0"] - prev["x1"] > gap_threshold:
+ gap = cur["x0"] - prev["x1"]
+ # PDF producers frequently position digits with a small kerning gap,
+ # especially in right-aligned table cells. Treating that gap as a
+ # word boundary turns ``124`` into ``1 24``. Keep the normal gap
+ # heuristic for genuinely separated numeric cells (larger gaps).
+ numeric_pair = (
+ prev["ch"] in "0123456789.,%+-"
+ and ch in "0123456789.,%+-"
+ and (prev["ch"].isdigit() or ch.isdigit())
+ )
+ if gap > gap_threshold and not (
+ numeric_pair and gap <= med_w * 1.25
+ ):
parts.append(" ")
parts.append(ch)
return "".join(parts).strip()
@@ -927,6 +1203,46 @@ def _layout_garbled_line_fraction(text: str) -> float:
return garbled / len(lines)
+def _numeric_row_values(text: str) -> list:
+ """Return compact ``(line_index, label, value)`` rows from plain text.
+
+ This is intentionally conservative: a row must end in one numeric value,
+ have a short non-numeric label, and contain at least one letter. It is a
+ signal for choosing between extraction paths, not a table parser.
+ """
+ rows: list = []
+ for index, line in enumerate((text or "").splitlines()):
+ match = _NUMERIC_ROW_RE.fullmatch(line.strip())
+ if not match:
+ continue
+ label = match.group("label").strip()
+ if (
+ not label
+ or len(label) > 60
+ or len(label.split()) > 4
+ or label[-1:] in ".,;:!?"
+ or not any(char.isalpha() for char in label)
+ ):
+ continue
+ rows.append((index, label, match.group("value")))
+ return rows
+
+
+def _structured_numeric_rows(text: str) -> list:
+ """Return numeric rows only when at least two occur near each other."""
+ rows = _numeric_row_values(text)
+ if len(rows) < 2:
+ return []
+ for first, second in zip(rows, rows[1:]):
+ if second[0] - first[0] <= 3:
+ return rows
+ return []
+
+
+def _normalized_numeric_value(value: str) -> str:
+ return value.replace(",", "").lower()
+
+
def _plain_is_well_formed(plain: str) -> bool:
"""True when pdfium plain text already has usable words and punctuation.
@@ -949,19 +1265,48 @@ def _plain_is_well_formed(plain: str) -> bool:
return avg_len >= 5.0
-def _should_prefer_plain(plain: str, layout: str) -> bool:
- """Fall back to pdfium plain text when layout reconstruction looks broken."""
+def _should_prefer_plain(
+ plain: str, layout: str, *, allow_structured_rows: bool = True
+) -> bool:
+ """Fall back to pdfium plain text when layout reconstruction looks broken.
+
+ ``allow_structured_rows`` is disabled by the page router unless geometry
+ identifies a narrow, aligned numeric table column. This prevents a real
+ two-column page from being replaced by its potentially interleaved plain
+ stream merely because it contains numbers.
+ """
layout = (layout or "").strip()
plain = (plain or "").strip()
if not layout:
return True
if not plain:
return False
+
# Geometry must never change numeric evidence (1.5 -> 1 5, -5 -> 5).
# Compare a multiset so legitimate column reordering is still allowed.
numbers = re.compile(r"[+−-]?\d+(?:[.,:/]\d+)*(?:[%‰])?")
if Counter(numbers.findall(plain)) != Counter(numbers.findall(layout)):
return True
+
+ # A geometric split is useful for glued prose, but it can turn a simple
+ # table into one column of labels and another column of values. When the
+ # plain extractor exposes adjacent labelled numeric rows and the layout
+ # loses one of those values as an intact row, retain the plain version so
+ # the item/value correspondence survives.
+ plain_rows = _structured_numeric_rows(plain) if allow_structured_rows else []
+ if plain_rows:
+ layout_rows = _structured_numeric_rows(layout)
+ plain_values = Counter(_normalized_numeric_value(row[2]) for row in plain_rows)
+ layout_values = Counter(_normalized_numeric_value(row[2]) for row in layout_rows)
+ missing_values = sum((plain_values - layout_values).values())
+ if missing_values:
+ return True
+ if not missing_values and len(plain_rows) >= len(layout_rows):
+ # Both paths retained the values, but the plain stream keeps the
+ # table rows together with surrounding headings and prose. Prefer
+ # it when it is at least as structurally complete.
+ return True
+
n, single, punct_only = _layout_line_stats(layout)
if n == 0:
return True
@@ -989,26 +1334,37 @@ def _should_prefer_plain(plain: str, layout: str) -> bool:
return False
-def _extract_layout_text(page, raw) -> str:
+def _extract_layout_text(page, raw, *, return_metadata: bool = False):
"""Layout-aware extraction: reading order + headings + hidden-text filter.
Falls back to plain extraction on any failure so a single odd page never
breaks the document.
"""
textpage = None
+ filtered = False
+ table_hint = False
try:
textpage = page.get_textpage()
- chars, width = _page_chars(textpage, page, raw)
+ chars, width, filtered = _page_chars(
+ textpage, page, raw, return_filter_info=True
+ )
if not chars:
- return ""
- heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]
- scale = (statistics.median(heights) if heights else 1.0) or 1.0
- return _chars_to_layout_markdown(chars, scale, width)
+ result = ""
+ else:
+ heights = [
+ c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0
+ ]
+ scale = (statistics.median(heights) if heights else 1.0) or 1.0
+ table_hint = _looks_like_numeric_table_columns(chars, scale, width)
+ result = _chars_to_layout_markdown(chars, scale, width)
except Exception:
logger.debug("layout extraction failed; using plain text", exc_info=True)
- return _extract_page_text(page)
+ result = _extract_page_text(page)
finally:
_close_pdfium_resource(textpage)
+ if return_metadata:
+ return result, filtered, table_hint
+ return result
# A visual line is cut where a gap wider than one glyph height coincides with
@@ -1972,11 +2328,24 @@ class PDFParser(BaseParser):
# Layout reconstruction only pays off (and is only spent) on
# native text pages; scanned pages are rendered, not read.
if cls == "text" and LAYOUT_ORDERING:
- if _plain_is_well_formed(plain):
+ plain_well_formed = _plain_is_well_formed(plain)
+ if plain_well_formed and not _page_has_filtered_text(
+ page, pdfium_r
+ ):
text = plain
else:
- layout = _extract_layout_text(page, pdfium_r)
- if layout and not _should_prefer_plain(plain, layout):
+ layout, filtered_text, table_hint = _extract_layout_text(
+ page, pdfium_r, return_metadata=True
+ )
+ if filtered_text:
+ # The plain layer may contain hidden or
+ # off-page prompt-injection text. Once the
+ # layout path proves that filtering occurred,
+ # never re-introduce that layer as a fallback.
+ text = layout
+ elif layout and not _should_prefer_plain(
+ plain, layout, allow_structured_rows=table_hint
+ ):
text = layout
else:
text = plain
diff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.py
index 248214029..b560db89c 100644
--- a/docreader/tests/test_pdf_router.py
+++ b/docreader/tests/test_pdf_router.py
@@ -15,14 +15,19 @@ from docreader.parser.pdf_parser import (
_filter_reading_columns,
_group_lines,
_is_artifact_column,
+ _chars_to_layout_markdown,
_join_line_glyphs,
+ _looks_like_numeric_table_columns,
_merge_orphan_punctuation_lines,
+ _page_chars,
_point_in_boxes,
+ _postprocess_pdf_text,
_segments_to_markdown,
_select_embedded_images,
_should_prefer_plain,
_split_columns,
_strip_repeating_lines,
+ _tail_looks_like_numeric_table,
)
@@ -374,6 +379,23 @@ class ReadingOrderTest(unittest.TestCase):
chars = [_char("A", 100, 110, 700, 712), _char("B", 110, 120, 700, 712)]
self.assertEqual(_join_line_glyphs(chars), "AB")
+ def test_join_line_glyphs_keeps_kerning_gaps_inside_numbers(self):
+ # Right-aligned PDF table cells often add a small gap between digit
+ # glyphs. It is not a word boundary.
+ chars = [
+ _char("1", 0, 4, 0, 10),
+ _char("2", 7, 11, 0, 10),
+ _char("4", 14, 18, 0, 10),
+ ]
+ self.assertEqual(_join_line_glyphs(chars), "124")
+
+ def test_join_line_glyphs_keeps_large_numeric_cell_gap(self):
+ chars = [
+ _char("1", 0, 4, 0, 10),
+ _char("2", 20, 24, 0, 10),
+ ]
+ self.assertEqual(_join_line_glyphs(chars), "1 2")
+
class HeadingDetectionTest(unittest.TestCase):
def test_promotes_large_line_to_heading(self):
@@ -400,8 +422,78 @@ class HiddenTextFilterTest(unittest.TestCase):
self.assertTrue(_point_in_boxes(5.0, 5.0, boxes))
self.assertFalse(_point_in_boxes(20.0, 5.0, boxes))
+ def test_unreadable_box_does_not_claim_hidden_text_was_filtered(self):
+ from types import SimpleNamespace
+
+ def charbox(index, **kwargs):
+ if index == 0:
+ raise ValueError("unreadable glyph box")
+ return 10.0, 10.0, 20.0, 20.0
+
+ textpage = SimpleNamespace(
+ count_chars=lambda: 2,
+ get_charbox=charbox,
+ get_text_range=lambda index, length: "a",
+ )
+ page = SimpleNamespace(get_size=lambda: (100.0, 100.0), get_objects=lambda: [])
+ raw = SimpleNamespace(FPDF_PAGEOBJ_TEXT=1)
+ chars, _, filtered = _page_chars(textpage, page, raw, return_filter_info=True)
+ self.assertEqual(len(chars), 1)
+ self.assertFalse(filtered)
+ textpage.count_chars = lambda: 0
+ self.assertEqual(_page_chars(textpage, page, raw, return_filter_info=True), ([], 0.0, False))
+
+ def test_page_chars_reports_filtered_off_page_glyphs(self):
+ class FakeTextPage:
+ chars = [
+ ("visible", (10.0, 10.0, 20.0, 20.0)),
+ ("hidden", (-20.0, 10.0, -10.0, 20.0)),
+ ]
+
+ def count_chars(self):
+ return len(self.chars)
+
+ def get_charbox(self, index):
+ return self.chars[index][1]
+
+ def get_text_range(self, index, length=1):
+ return self.chars[index][0][0]
+
+ class FakePage:
+ def get_size(self):
+ return 100.0, 100.0
+
+ def get_objects(self):
+ return []
+
+ class FakeRaw:
+ FPDF_PAGEOBJ_TEXT = 1
+
+ chars, width, filtered = _page_chars(
+ FakeTextPage(), FakePage(), FakeRaw(), return_filter_info=True
+ )
+ self.assertEqual(width, 100.0)
+ self.assertEqual(len(chars), 1)
+ self.assertTrue(filtered)
+
class MarginColumnFilterTest(unittest.TestCase):
+ @staticmethod
+ def _table_chars(rows):
+ chars = []
+ for i, (label, value) in enumerate(rows):
+ y = 700 - i * 40
+ for j, ch in enumerate(label):
+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, y, y + 10))
+ if value is not None:
+ for j, ch in enumerate(value):
+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, y, y + 10))
+ for j, ch in enumerate("ITEM"):
+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, 740, 750))
+ for j, ch in enumerate("QUANTITY"):
+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, 740, 750))
+ return chars
+
def test_drops_narrow_vertical_margin_column(self):
# Mimics arXiv sidebar: narrow x span, one glyph per line.
margin = [
@@ -423,6 +515,61 @@ class MarginColumnFilterTest(unittest.TestCase):
cols = _filter_reading_columns(left + right, scale=12.0, width=600.0)
self.assertEqual(len(cols), 2)
+ def test_numeric_table_hint_requires_aligned_value_column(self):
+ # A narrow quantity column aligned with labels is a table, not a
+ # margin watermark. The hint is consumed before artifact filtering.
+ chars = []
+ for i, label in enumerate(("ITEM", "Aster", "Willow")):
+ y = 700 - i * 40
+ for j, ch in enumerate(label):
+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, y, y + 10))
+ for i, value in enumerate(("QUANTITY", "124", "237")):
+ y = 700 - i * 40
+ for j, ch in enumerate(value):
+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, y, y + 10))
+ self.assertTrue(
+ _looks_like_numeric_table_columns(chars, scale=10.0, width=600.0)
+ )
+
+ def test_numeric_table_keeps_rows_and_drops_unrelated_artifact_column(self):
+ chars = self._table_chars([("Aster", "124"), ("Willow", "237")])
+ chars.extend(
+ _char(c, 20, 28, 500 - i * 14, 512 - i * 14)
+ for i, c in enumerate("0202luJ22")
+ )
+ text = _chars_to_layout_markdown(chars, scale=10.0, width=600.0)
+ self.assertIn("Aster 124", text)
+ self.assertIn("Willow 237", text)
+ self.assertNotIn("0202luJ22", text)
+
+ def test_numeric_table_hint_does_not_match_two_text_columns(self):
+ left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)]
+ right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)]
+ self.assertFalse(
+ _looks_like_numeric_table_columns(left + right, scale=12.0, width=600.0)
+ )
+
+ def test_layout_keeps_repeated_values_and_empty_cells_in_rows(self):
+ repeated = _chars_to_layout_markdown(
+ self._table_chars(
+ [("Aster", "1"), ("Willow", "1")]
+ ),
+ scale=10.0,
+ width=600.0,
+ )
+ self.assertIn("Aster 1", repeated)
+ self.assertIn("Willow 1", repeated)
+
+ with_empty = _chars_to_layout_markdown(
+ self._table_chars(
+ [("Aster", None), ("Willow", "237")]
+ ),
+ scale=10.0,
+ width=600.0,
+ )
+ self.assertIn("Aster", with_empty)
+ self.assertIn("Willow 237", with_empty)
+
class PunctuationMergeTest(unittest.TestCase):
def test_merges_orphan_periods(self):
@@ -488,6 +635,59 @@ class PdfTextSanitizeTest(unittest.TestCase):
self.assertNotIn("arXiv:", out)
self.assertIn("Body text.", out)
+ def test_preserves_numeric_body_run_without_chart_context(self):
+ raw = "Quantity\n124\n237\n68\n91\nEnd of inventory."
+ out = _postprocess_pdf_text(raw)
+ for value in ("124", "237", "68", "91"):
+ self.assertIn(value, out)
+
+ def test_preserves_numeric_body_run_with_chart_context_when_header_is_present(self):
+ raw = "Quantity\n124\n237\n68\n91\ntraining error"
+ out = _postprocess_pdf_text(raw)
+ for value in ("124", "237", "68", "91"):
+ self.assertIn(value, out)
+
+ def test_numeric_header_and_rows_are_a_table_tail(self):
+ self.assertTrue(_tail_looks_like_numeric_table(["Body", "Quantity", "124", "237"]))
+
+ def test_table_signal_must_reach_the_current_nonempty_tail(self):
+ for rows in (
+ ["Aster 1", "Willow 2"],
+ ["Quantity", "124", "237"],
+ ["Aster", "124", "Willow", "237"],
+ ):
+ with self.subTest(rows=rows):
+ self.assertTrue(_tail_looks_like_numeric_table(rows + [""]))
+ self.assertFalse(_tail_looks_like_numeric_table(rows + ["identity"]))
+ out = _postprocess_pdf_text("\n".join(rows + ["identity", "Figure 1. Diagram", "After"]))
+ self.assertNotIn("identity", out)
+ self.assertIn("\n".join(rows), out)
+
+ def test_preserves_repeated_single_digit_values(self):
+ from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+ raw = "ITEM QUANTITY\nAster 1\nWillow 1\nHazel 1"
+ out = _postprocess_pdf_text(raw)
+ self.assertIn("Aster 1", out)
+ self.assertIn("Willow 1", out)
+ self.assertIn("Hazel 1", out)
+
+ def test_keeps_ambiguous_middle_page_number(self):
+ from docreader.parser.pdf_parser import _postprocess_pdf_text
+
+ out = _postprocess_pdf_text("Body\n124\nMore body\n2")
+ self.assertIn("124", out)
+ # An unlabelled edge number is ambiguous too, so retain it. The
+ # existing arXiv-header regression covers the explicit cleanup case.
+ self.assertIn("\n2", out)
+
+ def test_keeps_numeric_table_before_figure_caption(self):
+ raw = "Header\nAster 124\nWillow 237\nFigure 1. Chart\nAfter"
+ out = _postprocess_pdf_text(raw)
+ self.assertIn("Aster 124", out)
+ self.assertIn("Willow 237", out)
+ self.assertIn("Figure 1. Chart", out)
+
class PlainWellFormedTest(unittest.TestCase):
def test_academic_plain_skips_layout(self):
@@ -516,6 +716,23 @@ class LayoutQualityFallbackTest(unittest.TestCase):
layout = "Hello world"
self.assertFalse(_should_prefer_plain(plain, layout))
+ def test_prefers_plain_when_layout_splits_numeric_table_columns(self):
+ plain = "ITEM QUANTITY\nAster 124\nWillow 237"
+ layout = "ITEM\nAster\nWillow\nQUANTITY\n1 24\n237"
+ self.assertTrue(_should_prefer_plain(plain, layout))
+
+ def test_prefers_plain_when_layout_substitutes_a_numeric_value(self):
+ plain = "ITEM QUANTITY\nAster 124\nWillow 237"
+ layout = "ITEM QUANTITY\nAster 124\nWillow 999"
+ self.assertTrue(_should_prefer_plain(plain, layout))
+
+ def test_does_not_prefer_plain_numeric_rows_without_table_hint(self):
+ plain = "Method 2020\nDataset 2021"
+ layout = "Method\nDataset\n2020\n2021"
+ self.assertFalse(
+ _should_prefer_plain(plain, layout, allow_structured_rows=False)
+ )
+
class ResNetPaperFigureTest(unittest.TestCase):
"""Regression: ResNet PDF (arXiv:1512.03385) vector figures and captions."""
@@ -631,6 +848,38 @@ class ScanEnglishDictLayoutTest(unittest.TestCase):
class PDFRouterIntegrationTest(unittest.TestCase):
+ def test_well_formed_plain_text_does_not_restore_outside_cropbox_text(self):
+ import pypdfium2 as pdfium
+
+ from docreader.parser.pdf_parser import _extract_page_text, _plain_is_well_formed
+ from docreader.tests.test_source_locator_pdf import pdf_fixture
+
+ visible = "Visible source evidence remains inside crop boundary"
+ outside = "Outside-note"
+ content = pdf_fixture(
+ [
+ {
+ "crop": "/CropBox [100 100 500 700]",
+ "lines": [(visible, 150, y) for y in (600, 560, 520, 480, 440)]
+ + [(outside, 10, 300)],
+ }
+ ]
+ )
+ with pdfium.PdfDocument(content) as pdf:
+ page = pdf[0]
+ try:
+ plain = _extract_page_text(page)
+ finally:
+ page.close()
+ self.assertIn(outside, plain)
+ self.assertTrue(_plain_is_well_formed(plain))
+
+ doc = PDFParser(
+ file_name="cropped-text.pdf", file_type="pdf", pdf_force_scanned=False
+ ).parse_into_text(content)
+ self.assertIn(visible, doc.content)
+ self.assertNotIn(outside, doc.content)
+
def test_image_only_pdf_routes_to_scanned(self):
pdf_bytes = _make_image_only_pdf(2)
doc = PDFParser(file_name="imgonly.pdf", file_type="pdf").parse_into_text(