Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): preserve numeric rows in PDF text extraction (#3228)
dvd233735eeb1f2db4
diff --git a/docreader/parser/pdf_parser.py b/docreader/parser/pdf_parser.pyindex fe7d2e6cc..c60187698 100644--- a/docreader/parser/pdf_parser.py+++ b/docreader/parser/pdf_parser.py@@ -162,6 +162,22 @@ _FIGURE_CAPTION_RE = re.compile(r"^Figure\s+\d+\b", re.IGNORECASE)_FIGURE_CAPTION_SEARCH_RE = re.compile(r"\bFigure\s+(\d+)\b", re.IGNORECASE)_ARXIV_LINE_RE = re.compile(r"^arXiv:\s*\S+", re.IGNORECASE)_PAGE_NUM_LINE_RE = re.compile(r"^\d{1,3}$")+_PAGE_CONTEXT_RE = re.compile(+ r"(?:\bpage\b|\bp\.\s*\d+\b|\b\d+\s*(?:of|/)\s*\d+\b)",+ re.IGNORECASE,+)+_CHART_CONTEXT_RE = re.compile(+ r"(?:\bFigure\s+\d+\b|\biter\.\s*\(1e4\)|"+ r"\b(?:training|test)\s+error\b|\b\d+-layer\b)",+ re.IGNORECASE,+)+_NUMERIC_ROW_RE = re.compile(+ r"^(?P<label>[^\d\n]+?)\s+"+ r"(?P<value>[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?)$"+)+_NUMERIC_VALUE_LINE_RE = re.compile(+ r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"+)def _close_pdfium_resource(resource) -> None:@@ -256,7 +272,19 @@ def _strip_chart_text_debris(text: str) -> str:_is_chart_debris_line(lines[j]) or not lines[j].strip()):j += 1- if j - i >= 3:+ # Numeric-only lines are also a common representation for table+ # values. Only remove a run when a nearby chart/figure marker+ # provides positive evidence that it is axis or legend text.+ context_start = max(0, i - 1)+ context_end = min(len(lines), j + 1)+ has_chart_context = any(+ _CHART_CONTEXT_RE.search(lines[k])+ for k in range(context_start, context_end)+ )+ table_context = _tail_looks_like_numeric_table(+ lines[max(0, i - 2) : j]+ )+ if j - i >= 3 and has_chart_context and not table_context:i = jcontinueout.append(lines[i])@@ -264,14 +292,46 @@ def _strip_chart_text_debris(text: str) -> str:return "\n".join(out)+def _is_probable_page_num_line(lines: list, index: int) -> bool:+ """Return whether a standalone number is likely a page marker.++ A numeric-only line in the middle of a page is often a table value. The+ old unconditional filter removed those values, so page-number cleanup is+ deliberately limited to an explicitly labelled edge line or a line adjacent+ to an arXiv header. Keeping an ambiguous number is safer than silently+ losing document data.+ """+ if index < 0 or index >= len(lines):+ return False+ if not _PAGE_NUM_LINE_RE.fullmatch(lines[index].strip()):+ return False+ nonempty = [i for i, line in enumerate(lines) if line.strip()]+ if not nonempty:+ return False+ pos = nonempty.index(index)+ neighbors = []+ if pos:+ neighbors.append(nonempty[pos - 1])+ if pos + 1 < len(nonempty):+ neighbors.append(nonempty[pos + 1])+ if any(_ARXIV_LINE_RE.match(lines[i].strip()) for i in neighbors):+ return True+ # Keep an edge number unless a nearby line explicitly looks like a page+ # label (``Page 3 of 8`` / ``3 / 8``). Ambiguous edge numbers can still be+ # legitimate first/last values in a table.+ if index in (nonempty[0], nonempty[-1]):+ return any(_PAGE_CONTEXT_RE.search(lines[i]) for i in neighbors)+ return False++def _strip_arxiv_and_page_num_lines(text: str) -> str:lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n")kept: list = []- for ln in lines:+ for i, ln in enumerate(lines):t = ln.strip()if _ARXIV_LINE_RE.match(t):continue- if _PAGE_NUM_LINE_RE.match(t):+ if _is_probable_page_num_line(lines, i):continueif "arXiv:" in ln:ln = re.sub(r"\s*arXiv:\s*\S+\s*(?:\[[^\]]+\])?\s*[^\n]*", "", ln).strip()@@ -288,6 +348,11 @@ def _strip_lines_above_figure_captions(text: str) -> str:for ln in lines:if _line_has_figure_caption(ln):while out and _is_figure_interior_line(out[-1]):+ # A table can legitimately end immediately before a figure+ # caption. Preserve a tail with two labelled numeric rows;+ # diagram labels do not have this row-shaped signal.+ if _tail_looks_like_numeric_table(out):+ breakout.pop()out.append(ln)else:@@ -295,6 +360,41 @@ def _strip_lines_above_figure_captions(text: str) -> str:return "\n".join(out)+def _tail_looks_like_numeric_table(lines: list) -> bool:+ """Detect labelled or alternating label/value rows at the text tail."""+ tail = [line for line in lines[-10:] if line.strip()]+ rows = _structured_numeric_rows("\n".join(tail))+ if rows and rows[-1][0] == len(tail) - 1:+ return True++ numeric = re.compile(+ r"^[+-]?(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?%?$"+ )+ for index, header in enumerate(tail[:-2]):+ if (+ header.strip()+ and any(char.isalpha() for char in header)+ and all(numeric.fullmatch(value.strip()) for value in tail[index + 1 :])+ ):+ return True+ pairs = 0+ index = len(tail) - 1+ while index >= 1:+ label = tail[index - 1].strip()+ value = tail[index].strip()+ if (+ label+ and any(char.isalpha() for char in label)+ and len(label) <= 60+ and numeric.fullmatch(value)+ ):+ pairs += 1+ index -= 2+ else:+ break+ return pairs >= 2++def _is_body_paragraph_line(text: str) -> bool:t = text.strip()if len(t) < 48:@@ -639,17 +739,24 @@ def _point_in_boxes(x: float, y: float, boxes: list) -> bool:return False-def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:+def _page_chars(+ textpage, page, raw, loose: bool = False, *, return_filter_info: bool = False+) -> tuple:"""Return ``(chars, page_width)`` with hidden/off-page glyphs filtered.Working at the glyph level (instead of pdfium rect segments) keeps mixedCJK + Latin/number lines in their true left-to-right order, which therect-level ``get_text_bounded`` API scrambles. ``loose`` boxes span thefont's ascent to descent, so every glyph of a line shares one height.++ When ``return_filter_info`` is true, a third value reports whether any+ source glyph was intentionally filtered as hidden or off-page. Callers+ can then avoid falling back to the unfiltered plain text when a PDF+ contains hidden or off-page text."""n = textpage.count_chars()if n <= 0:- return [], 0.0+ return ([], 0.0, False) if return_filter_info else ([], 0.0)# Glyph coordinates are unrotated PDF user space. get_size() reports# rotated dimensions and would discard valid text on cropped/rotated pages.try:@@ -661,25 +768,67 @@ def _page_chars(textpage, page, raw, loose: bool = False) -> tuple:invisible = _collect_invisible_boxes(page, raw) if FILTER_HIDDEN_TEXT else []chars: list = []+ source_count = 0+ filtered_count = 0for i in range(n):try:- left, bottom, right, top = textpage.get_charbox(i, loose=loose)+ try:+ left, bottom, right, top = textpage.get_charbox(i, loose=loose)+ except TypeError:+ # Keep compatibility with the small PDFium test doubles and+ # older pypdfium2 releases that do not expose ``loose``.+ left, bottom, right, top = textpage.get_charbox(i)except Exception:+ source_count += 1continuech = textpage.get_text_range(i, 1)if ch in ("\r", "\n"):continue+ source_count += 1x0, x1 = (left, right) if left <= right else (right, left)y0, y1 = (bottom, top) if bottom <= top else (top, bottom)if FILTER_HIDDEN_TEXT:if x1 < crop_left or x0 > crop_right or y1 < crop_bottom or y0 > crop_top:+ filtered_count += 1continue # off-page glyphif invisible and _point_in_boxes((x0 + x1) / 2, (y0 + y1) / 2, invisible):+ filtered_count += 1continue # covered by an invisible text objectchars.append({"x0": x0, "y0": y0, "x1": x1, "y1": y1, "ch": ch, "i": i})+ if return_filter_info:+ return chars, width, filtered_count > 0return chars, width+def _page_has_invisible_text(page, raw) -> bool:+ """Whether ``page`` contains text objects hidden from normal rendering."""+ if not FILTER_HIDDEN_TEXT:+ return False+ try:+ return bool(_collect_invisible_boxes(page, raw))+ except Exception:+ # A failed probe should not change the existing plain-text route. The+ # full glyph-level probe still runs whenever layout extraction is used.+ return False+++def _page_has_filtered_text(page, raw) -> bool:+ """Whether layout filtering would discard hidden or off-page glyphs."""+ if not FILTER_HIDDEN_TEXT:+ return False+ if _page_has_invisible_text(page, raw):+ return True+ textpage = None+ try:+ textpage = page.get_textpage()+ _, _, filtered = _page_chars(textpage, page, raw, return_filter_info=True)+ return filtered+ except Exception:+ return False+ finally:+ _close_pdfium_resource(textpage)++def _find_split(items: list, axis: str, min_gap: float):"""Return a coordinate at the widest clean gap on ``axis`` ('x'), or None.@@ -757,6 +906,21 @@ def _is_artifact_column(chars: list, width: float) -> bool:def _filter_reading_columns(chars: list, scale: float, width: float) -> list:"""Split into columns and drop margin / watermark strips."""cols = _split_columns(chars, scale, width)+ table_pair = _find_numeric_table_columns(chars, scale, width, cols)+ if table_pair is not None:+ # Keep aligned table cells in one visual stream so rows such as+ # ``Aster 124`` are not emitted as a label block followed by a value+ # block, while still dropping unrelated margin/artifact columns.+ # ``chars`` has already passed hidden/off-page filtering.+ value_col, label_col = table_pair+ table_cols = [+ c+ for c in cols+ if not _is_artifact_column(c, width) or c is value_col or c is label_col+ ]+ if table_cols:+ return [[char for column in table_cols for char in column]]+ return [chars]kept = [c for c in cols if not _is_artifact_column(c, width)]if kept:return kept@@ -766,6 +930,106 @@ def _filter_reading_columns(chars: list, scale: float, width: float) -> list:return cols+def _is_numeric_value_line(text: str) -> bool:+ t = (text or "").strip()+ return bool(+ _NUMERIC_VALUE_LINE_RE.fullmatch(t)+ or (+ re.fullmatch(r"[\d\s./()%+-]+", t)+ and any(char.isdigit() for char in t)+ )+ )+++def _looks_like_numeric_table_columns(+ chars: list, scale: float, width: float, columns: list | None = None+) -> bool:+ """Identify a narrow, aligned value column before artifact filtering.++ A real two-column page should remain column-wise. A simple table often+ looks different: one narrow column contains a header and mostly numeric+ values, and its baselines line up with labels in a wider column. This+ signal lets the caller choose the plain extractor only for that table-like+ shape instead of turning every numeric mismatch into a page-wide fallback.+ """+ return _find_numeric_table_columns(chars, scale, width, columns) is not None+++def _find_numeric_table_columns(+ chars: list, scale: float, width: float, columns: list | None = None+) -> tuple[list, list] | None:+ """Return the aligned value/label columns when a numeric table is found."""+ if not chars or width <= 0:+ return None+ cols = columns if columns is not None else _split_columns(chars, scale, width)+ if len(cols) < 2:+ return None++ for value_col in cols:+ span_ratio = _column_x_span(value_col) / width+ if span_ratio > 0.22:+ continue+ value_lines = _group_lines_with_chars(value_col)+ if len(value_lines) < 2:+ continue+ numeric_lines = [+ line for line in value_lines if _is_numeric_value_line(line["text"])+ ]+ if not numeric_lines or len(numeric_lines) / len(value_lines) < 0.4:+ continue+ # Require a textual header/label in the narrow column. This keeps a+ # vertical numeric watermark from being mistaken for a table.+ text_lines = [+ line+ for line in value_lines+ if not _is_numeric_value_line(line["text"])+ and any(char.isalpha() for char in line["text"])+ ]+ if not text_lines:+ continue++ value_ys = [+ (line["bbox"][1] + line["bbox"][3]) / 2 for line in numeric_lines+ ]+ for label_col in cols:+ if label_col is value_col:+ continue+ label_lines = _group_lines_with_chars(label_col)+ label_ys = [+ (line["bbox"][1] + line["bbox"][3]) / 2+ for line in label_lines+ ]+ label_text_ys = [+ (line["bbox"][1] + line["bbox"][3]) / 2+ for line in label_lines+ if any(char.isalpha() for char in line["text"])+ ]+ tolerance = max(scale * 0.75, 2.0)+ header_ys = [+ (line["bbox"][1] + line["bbox"][3]) / 2 for line in text_lines+ ]+ header_aligned = any(+ abs(y - other_y) <= tolerance+ for y in header_ys+ for other_y in label_text_ys+ )+ if not header_aligned:+ continue+ aligned = sum(+ any(abs(y - other_y) <= tolerance for other_y in label_ys)+ for y in value_ys+ )+ # A table may contain an empty value cell. In that case the one+ # remaining numeric row still carries enough evidence when its+ # header and the label column share the same baseline.+ required_alignment = (+ 1 if len(value_ys) == 1 else max(2, (len(value_ys) + 1) // 2)+ )+ if aligned >= required_alignment:+ return value_col, label_col+ return None++def _merge_orphan_punctuation_lines(lines: list) -> list:"""Attach lines that are only punctuation to the previous visual line.@@ -813,7 +1077,19 @@ def _join_line_glyphs(ln_sorted: list) -> str:if not ch.isspace() or (parts and not parts[-1].endswith(" ")):parts.append(ch)continue- if cur["x0"] - prev["x1"] > gap_threshold:+ gap = cur["x0"] - prev["x1"]+ # PDF producers frequently position digits with a small kerning gap,+ # especially in right-aligned table cells. Treating that gap as a+ # word boundary turns ``124`` into ``1 24``. Keep the normal gap+ # heuristic for genuinely separated numeric cells (larger gaps).+ numeric_pair = (+ prev["ch"] in "0123456789.,%+-"+ and ch in "0123456789.,%+-"+ and (prev["ch"].isdigit() or ch.isdigit())+ )+ if gap > gap_threshold and not (+ numeric_pair and gap <= med_w * 1.25+ ):parts.append(" ")parts.append(ch)return "".join(parts).strip()@@ -927,6 +1203,46 @@ def _layout_garbled_line_fraction(text: str) -> float:return garbled / len(lines)+def _numeric_row_values(text: str) -> list:+ """Return compact ``(line_index, label, value)`` rows from plain text.++ This is intentionally conservative: a row must end in one numeric value,+ have a short non-numeric label, and contain at least one letter. It is a+ signal for choosing between extraction paths, not a table parser.+ """+ rows: list = []+ for index, line in enumerate((text or "").splitlines()):+ match = _NUMERIC_ROW_RE.fullmatch(line.strip())+ if not match:+ continue+ label = match.group("label").strip()+ if (+ not label+ or len(label) > 60+ or len(label.split()) > 4+ or label[-1:] in ".,;:!?"+ or not any(char.isalpha() for char in label)+ ):+ continue+ rows.append((index, label, match.group("value")))+ return rows+++def _structured_numeric_rows(text: str) -> list:+ """Return numeric rows only when at least two occur near each other."""+ rows = _numeric_row_values(text)+ if len(rows) < 2:+ return []+ for first, second in zip(rows, rows[1:]):+ if second[0] - first[0] <= 3:+ return rows+ return []+++def _normalized_numeric_value(value: str) -> str:+ return value.replace(",", "").lower()++def _plain_is_well_formed(plain: str) -> bool:"""True when pdfium plain text already has usable words and punctuation.@@ -949,19 +1265,48 @@ def _plain_is_well_formed(plain: str) -> bool:return avg_len >= 5.0-def _should_prefer_plain(plain: str, layout: str) -> bool:- """Fall back to pdfium plain text when layout reconstruction looks broken."""+def _should_prefer_plain(+ plain: str, layout: str, *, allow_structured_rows: bool = True+) -> bool:+ """Fall back to pdfium plain text when layout reconstruction looks broken.++ ``allow_structured_rows`` is disabled by the page router unless geometry+ identifies a narrow, aligned numeric table column. This prevents a real+ two-column page from being replaced by its potentially interleaved plain+ stream merely because it contains numbers.+ """layout = (layout or "").strip()plain = (plain or "").strip()if not layout:return Trueif not plain:return False+# Geometry must never change numeric evidence (1.5 -> 1 5, -5 -> 5).# Compare a multiset so legitimate column reordering is still allowed.numbers = re.compile(r"[+−-]?\d+(?:[.,:/]\d+)*(?:[%‰])?")if Counter(numbers.findall(plain)) != Counter(numbers.findall(layout)):return True++ # A geometric split is useful for glued prose, but it can turn a simple+ # table into one column of labels and another column of values. When the+ # plain extractor exposes adjacent labelled numeric rows and the layout+ # loses one of those values as an intact row, retain the plain version so+ # the item/value correspondence survives.+ plain_rows = _structured_numeric_rows(plain) if allow_structured_rows else []+ if plain_rows:+ layout_rows = _structured_numeric_rows(layout)+ plain_values = Counter(_normalized_numeric_value(row[2]) for row in plain_rows)+ layout_values = Counter(_normalized_numeric_value(row[2]) for row in layout_rows)+ missing_values = sum((plain_values - layout_values).values())+ if missing_values:+ return True+ if not missing_values and len(plain_rows) >= len(layout_rows):+ # Both paths retained the values, but the plain stream keeps the+ # table rows together with surrounding headings and prose. Prefer+ # it when it is at least as structurally complete.+ return True+n, single, punct_only = _layout_line_stats(layout)if n == 0:return True@@ -989,26 +1334,37 @@ def _should_prefer_plain(plain: str, layout: str) -> bool:return False-def _extract_layout_text(page, raw) -> str:+def _extract_layout_text(page, raw, *, return_metadata: bool = False):"""Layout-aware extraction: reading order + headings + hidden-text filter.Falls back to plain extraction on any failure so a single odd page neverbreaks the document."""textpage = None+ filtered = False+ table_hint = Falsetry:textpage = page.get_textpage()- chars, width = _page_chars(textpage, page, raw)+ chars, width, filtered = _page_chars(+ textpage, page, raw, return_filter_info=True+ )if not chars:- return ""- heights = [c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0]- scale = (statistics.median(heights) if heights else 1.0) or 1.0- return _chars_to_layout_markdown(chars, scale, width)+ result = ""+ else:+ heights = [+ c["y1"] - c["y0"] for c in chars if c["y1"] - c["y0"] > 0+ ]+ scale = (statistics.median(heights) if heights else 1.0) or 1.0+ table_hint = _looks_like_numeric_table_columns(chars, scale, width)+ result = _chars_to_layout_markdown(chars, scale, width)except Exception:logger.debug("layout extraction failed; using plain text", exc_info=True)- return _extract_page_text(page)+ result = _extract_page_text(page)finally:_close_pdfium_resource(textpage)+ if return_metadata:+ return result, filtered, table_hint+ return result# A visual line is cut where a gap wider than one glyph height coincides with@@ -1972,11 +2328,24 @@ class PDFParser(BaseParser):# Layout reconstruction only pays off (and is only spent) on# native text pages; scanned pages are rendered, not read.if cls == "text" and LAYOUT_ORDERING:- if _plain_is_well_formed(plain):+ plain_well_formed = _plain_is_well_formed(plain)+ if plain_well_formed and not _page_has_filtered_text(+ page, pdfium_r+ ):text = plainelse:- layout = _extract_layout_text(page, pdfium_r)- if layout and not _should_prefer_plain(plain, layout):+ layout, filtered_text, table_hint = _extract_layout_text(+ page, pdfium_r, return_metadata=True+ )+ if filtered_text:+ # The plain layer may contain hidden or+ # off-page prompt-injection text. Once the+ # layout path proves that filtering occurred,+ # never re-introduce that layer as a fallback.+ text = layout+ elif layout and not _should_prefer_plain(+ plain, layout, allow_structured_rows=table_hint+ ):text = layoutelse:text = plaindiff --git a/docreader/tests/test_pdf_router.py b/docreader/tests/test_pdf_router.pyindex 248214029..b560db89c 100644--- a/docreader/tests/test_pdf_router.py+++ b/docreader/tests/test_pdf_router.py@@ -15,14 +15,19 @@ from docreader.parser.pdf_parser import (_filter_reading_columns,_group_lines,_is_artifact_column,+ _chars_to_layout_markdown,_join_line_glyphs,+ _looks_like_numeric_table_columns,_merge_orphan_punctuation_lines,+ _page_chars,_point_in_boxes,+ _postprocess_pdf_text,_segments_to_markdown,_select_embedded_images,_should_prefer_plain,_split_columns,_strip_repeating_lines,+ _tail_looks_like_numeric_table,)@@ -374,6 +379,23 @@ class ReadingOrderTest(unittest.TestCase):chars = [_char("A", 100, 110, 700, 712), _char("B", 110, 120, 700, 712)]self.assertEqual(_join_line_glyphs(chars), "AB")+ def test_join_line_glyphs_keeps_kerning_gaps_inside_numbers(self):+ # Right-aligned PDF table cells often add a small gap between digit+ # glyphs. It is not a word boundary.+ chars = [+ _char("1", 0, 4, 0, 10),+ _char("2", 7, 11, 0, 10),+ _char("4", 14, 18, 0, 10),+ ]+ self.assertEqual(_join_line_glyphs(chars), "124")++ def test_join_line_glyphs_keeps_large_numeric_cell_gap(self):+ chars = [+ _char("1", 0, 4, 0, 10),+ _char("2", 20, 24, 0, 10),+ ]+ self.assertEqual(_join_line_glyphs(chars), "1 2")+class HeadingDetectionTest(unittest.TestCase):def test_promotes_large_line_to_heading(self):@@ -400,8 +422,78 @@ class HiddenTextFilterTest(unittest.TestCase):self.assertTrue(_point_in_boxes(5.0, 5.0, boxes))self.assertFalse(_point_in_boxes(20.0, 5.0, boxes))+ def test_unreadable_box_does_not_claim_hidden_text_was_filtered(self):+ from types import SimpleNamespace++ def charbox(index, **kwargs):+ if index == 0:+ raise ValueError("unreadable glyph box")+ return 10.0, 10.0, 20.0, 20.0++ textpage = SimpleNamespace(+ count_chars=lambda: 2,+ get_charbox=charbox,+ get_text_range=lambda index, length: "a",+ )+ page = SimpleNamespace(get_size=lambda: (100.0, 100.0), get_objects=lambda: [])+ raw = SimpleNamespace(FPDF_PAGEOBJ_TEXT=1)+ chars, _, filtered = _page_chars(textpage, page, raw, return_filter_info=True)+ self.assertEqual(len(chars), 1)+ self.assertFalse(filtered)+ textpage.count_chars = lambda: 0+ self.assertEqual(_page_chars(textpage, page, raw, return_filter_info=True), ([], 0.0, False))++ def test_page_chars_reports_filtered_off_page_glyphs(self):+ class FakeTextPage:+ chars = [+ ("visible", (10.0, 10.0, 20.0, 20.0)),+ ("hidden", (-20.0, 10.0, -10.0, 20.0)),+ ]++ def count_chars(self):+ return len(self.chars)++ def get_charbox(self, index):+ return self.chars[index][1]++ def get_text_range(self, index, length=1):+ return self.chars[index][0][0]++ class FakePage:+ def get_size(self):+ return 100.0, 100.0++ def get_objects(self):+ return []++ class FakeRaw:+ FPDF_PAGEOBJ_TEXT = 1++ chars, width, filtered = _page_chars(+ FakeTextPage(), FakePage(), FakeRaw(), return_filter_info=True+ )+ self.assertEqual(width, 100.0)+ self.assertEqual(len(chars), 1)+ self.assertTrue(filtered)+class MarginColumnFilterTest(unittest.TestCase):+ @staticmethod+ def _table_chars(rows):+ chars = []+ for i, (label, value) in enumerate(rows):+ y = 700 - i * 40+ for j, ch in enumerate(label):+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, y, y + 10))+ if value is not None:+ for j, ch in enumerate(value):+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, y, y + 10))+ for j, ch in enumerate("ITEM"):+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, 740, 750))+ for j, ch in enumerate("QUANTITY"):+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, 740, 750))+ return chars+def test_drops_narrow_vertical_margin_column(self):# Mimics arXiv sidebar: narrow x span, one glyph per line.margin = [@@ -423,6 +515,61 @@ class MarginColumnFilterTest(unittest.TestCase):cols = _filter_reading_columns(left + right, scale=12.0, width=600.0)self.assertEqual(len(cols), 2)+ def test_numeric_table_hint_requires_aligned_value_column(self):+ # A narrow quantity column aligned with labels is a table, not a+ # margin watermark. The hint is consumed before artifact filtering.+ chars = []+ for i, label in enumerate(("ITEM", "Aster", "Willow")):+ y = 700 - i * 40+ for j, ch in enumerate(label):+ chars.append(_char(ch, 50 + j * 8, 56 + j * 8, y, y + 10))+ for i, value in enumerate(("QUANTITY", "124", "237")):+ y = 700 - i * 40+ for j, ch in enumerate(value):+ chars.append(_char(ch, 500 + j * 8, 506 + j * 8, y, y + 10))+ self.assertTrue(+ _looks_like_numeric_table_columns(chars, scale=10.0, width=600.0)+ )++ def test_numeric_table_keeps_rows_and_drops_unrelated_artifact_column(self):+ chars = self._table_chars([("Aster", "124"), ("Willow", "237")])+ chars.extend(+ _char(c, 20, 28, 500 - i * 14, 512 - i * 14)+ for i, c in enumerate("0202luJ22")+ )+ text = _chars_to_layout_markdown(chars, scale=10.0, width=600.0)+ self.assertIn("Aster 124", text)+ self.assertIn("Willow 237", text)+ self.assertNotIn("0202luJ22", text)++ def test_numeric_table_hint_does_not_match_two_text_columns(self):+ left = [_char("L", 50, 150, 700 - i * 12, 712 - i * 12) for i in range(4)]+ right = [_char("R", 400, 500, 700 - i * 12, 712 - i * 12) for i in range(4)]+ self.assertFalse(+ _looks_like_numeric_table_columns(left + right, scale=12.0, width=600.0)+ )++ def test_layout_keeps_repeated_values_and_empty_cells_in_rows(self):+ repeated = _chars_to_layout_markdown(+ self._table_chars(+ [("Aster", "1"), ("Willow", "1")]+ ),+ scale=10.0,+ width=600.0,+ )+ self.assertIn("Aster 1", repeated)+ self.assertIn("Willow 1", repeated)++ with_empty = _chars_to_layout_markdown(+ self._table_chars(+ [("Aster", None), ("Willow", "237")]+ ),+ scale=10.0,+ width=600.0,+ )+ self.assertIn("Aster", with_empty)+ self.assertIn("Willow 237", with_empty)+class PunctuationMergeTest(unittest.TestCase):def test_merges_orphan_periods(self):@@ -488,6 +635,59 @@ class PdfTextSanitizeTest(unittest.TestCase):self.assertNotIn("arXiv:", out)self.assertIn("Body text.", out)+ def test_preserves_numeric_body_run_without_chart_context(self):+ raw = "Quantity\n124\n237\n68\n91\nEnd of inventory."+ out = _postprocess_pdf_text(raw)+ for value in ("124", "237", "68", "91"):+ self.assertIn(value, out)++ def test_preserves_numeric_body_run_with_chart_context_when_header_is_present(self):+ raw = "Quantity\n124\n237\n68\n91\ntraining error"+ out = _postprocess_pdf_text(raw)+ for value in ("124", "237", "68", "91"):+ self.assertIn(value, out)++ def test_numeric_header_and_rows_are_a_table_tail(self):+ self.assertTrue(_tail_looks_like_numeric_table(["Body", "Quantity", "124", "237"]))++ def test_table_signal_must_reach_the_current_nonempty_tail(self):+ for rows in (+ ["Aster 1", "Willow 2"],+ ["Quantity", "124", "237"],+ ["Aster", "124", "Willow", "237"],+ ):+ with self.subTest(rows=rows):+ self.assertTrue(_tail_looks_like_numeric_table(rows + [""]))+ self.assertFalse(_tail_looks_like_numeric_table(rows + ["identity"]))+ out = _postprocess_pdf_text("\n".join(rows + ["identity", "Figure 1. Diagram", "After"]))+ self.assertNotIn("identity", out)+ self.assertIn("\n".join(rows), out)++ def test_preserves_repeated_single_digit_values(self):+ from docreader.parser.pdf_parser import _postprocess_pdf_text++ raw = "ITEM QUANTITY\nAster 1\nWillow 1\nHazel 1"+ out = _postprocess_pdf_text(raw)+ self.assertIn("Aster 1", out)+ self.assertIn("Willow 1", out)+ self.assertIn("Hazel 1", out)++ def test_keeps_ambiguous_middle_page_number(self):+ from docreader.parser.pdf_parser import _postprocess_pdf_text++ out = _postprocess_pdf_text("Body\n124\nMore body\n2")+ self.assertIn("124", out)+ # An unlabelled edge number is ambiguous too, so retain it. The+ # existing arXiv-header regression covers the explicit cleanup case.+ self.assertIn("\n2", out)++ def test_keeps_numeric_table_before_figure_caption(self):+ raw = "Header\nAster 124\nWillow 237\nFigure 1. Chart\nAfter"+ out = _postprocess_pdf_text(raw)+ self.assertIn("Aster 124", out)+ self.assertIn("Willow 237", out)+ self.assertIn("Figure 1. Chart", out)+class PlainWellFormedTest(unittest.TestCase):def test_academic_plain_skips_layout(self):@@ -516,6 +716,23 @@ class LayoutQualityFallbackTest(unittest.TestCase):layout = "Hello world"self.assertFalse(_should_prefer_plain(plain, layout))+ def test_prefers_plain_when_layout_splits_numeric_table_columns(self):+ plain = "ITEM QUANTITY\nAster 124\nWillow 237"+ layout = "ITEM\nAster\nWillow\nQUANTITY\n1 24\n237"+ self.assertTrue(_should_prefer_plain(plain, layout))++ def test_prefers_plain_when_layout_substitutes_a_numeric_value(self):+ plain = "ITEM QUANTITY\nAster 124\nWillow 237"+ layout = "ITEM QUANTITY\nAster 124\nWillow 999"+ self.assertTrue(_should_prefer_plain(plain, layout))++ def test_does_not_prefer_plain_numeric_rows_without_table_hint(self):+ plain = "Method 2020\nDataset 2021"+ layout = "Method\nDataset\n2020\n2021"+ self.assertFalse(+ _should_prefer_plain(plain, layout, allow_structured_rows=False)+ )+class ResNetPaperFigureTest(unittest.TestCase):"""Regression: ResNet PDF (arXiv:1512.03385) vector figures and captions."""@@ -631,6 +848,38 @@ class ScanEnglishDictLayoutTest(unittest.TestCase):class PDFRouterIntegrationTest(unittest.TestCase):+ def test_well_formed_plain_text_does_not_restore_outside_cropbox_text(self):+ import pypdfium2 as pdfium++ from docreader.parser.pdf_parser import _extract_page_text, _plain_is_well_formed+ from docreader.tests.test_source_locator_pdf import pdf_fixture++ visible = "Visible source evidence remains inside crop boundary"+ outside = "Outside-note"+ content = pdf_fixture(+ [+ {+ "crop": "/CropBox [100 100 500 700]",+ "lines": [(visible, 150, y) for y in (600, 560, 520, 480, 440)]+ + [(outside, 10, 300)],+ }+ ]+ )+ with pdfium.PdfDocument(content) as pdf:+ page = pdf[0]+ try:+ plain = _extract_page_text(page)+ finally:+ page.close()+ self.assertIn(outside, plain)+ self.assertTrue(_plain_is_well_formed(plain))++ doc = PDFParser(+ file_name="cropped-text.pdf", file_type="pdf", pdf_force_scanned=False+ ).parse_into_text(content)+ self.assertIn(visible, doc.content)+ self.assertNotIn(outside, doc.content)+def test_image_only_pdf_routes_to_scanned(self):pdf_bytes = _make_image_only_pdf(2)doc = PDFParser(file_name="imgonly.pdf", file_type="pdf").parse_into_text(