Skip to content

A fast, helpful, and open-source document parser

4.4 KB
Raw Blame
"""E2E tests for pool mode (process-isolated workers with hard timeouts)."""

import math
import zlib
from pathlib import Path

import pytest

from liteparse import LiteParse, ParseError, ParseTimeoutError

REPO_ROOT = Path(__file__).resolve().parents[3]
SAMPLE_PDF = REPO_ROOT / "integration_tests_data" / "sample.pdf"

CFG = dict(quiet=True, ocr_enabled=False, output_format="text")


def _grid_pdf(n_objects: int) -> bytes:
    """One page with a dense grid of tiny text objects.

    Parse time is superlinear in the object count (PDFium's text-page
    assembly), which makes a small file arbitrarily slow to parse — the same
    shape as the rogue documents that stall prod pipelines, used here to
    exercise the timeout path.
    """
    cols = int(math.sqrt(n_objects)) or 1
    parts = [
        f"BT /F1 2 Tf {20 + (i % cols) * 2.5:.1f} "
        f"{770 - (i // cols) * 2.5:.1f} Td (x) Tj ET"
        for i in range(n_objects)
    ]
    stream = zlib.compress("\n".join(parts).encode())
    width = int(40 + cols * 2.5)
    height = int(40 + (n_objects // cols) * 2.5)
    objects = [
        b"<< /Type /Catalog /Pages 2 0 R >>",
        b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>",
        b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 %d %d] "
        b"/Resources << /Font << /F1 4 0 R >> >> /Contents 5 0 R >>"
        % (width, height),
        b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>",
        b"<< /Length %d /Filter /FlateDecode >>\nstream\n%s\nendstream"
        % (len(stream), stream),
    ]
    out = bytearray(b"%PDF-1.4\n")
    offsets = []
    for i, body in enumerate(objects, start=1):
        offsets.append(len(out))
        out += b"%d 0 obj\n" % i + body + b"\nendobj\n"
    xref_pos = len(out)
    out += b"xref\n0 %d\n" % (len(objects) + 1)
    out += b"0000000000 65535 f \n"
    for off in offsets:
        out += b"%010d 00000 n \n" % off
    out += b"trailer\n<< /Size %d /Root 1 0 R >>\nstartxref\n%d\n%%%%EOF\n" % (
        len(objects) + 1,
        xref_pos,
    )
    return bytes(out)


@pytest.fixture
def sample_pdf() -> Path:
    if not SAMPLE_PDF.exists():
        pytest.skip(f"Test document not found: {SAMPLE_PDF}")
    return SAMPLE_PDF


def test_pool_matches_in_process(sample_pdf):
    expected = LiteParse(**CFG).parse(sample_pdf)
    with LiteParse(**CFG, pool_size=2) as pooled:
        from_path = pooled.parse(sample_pdf)
        from_bytes = pooled.parse(sample_pdf.read_bytes())
    assert from_path.text == expected.text
    assert from_bytes.text == expected.text
    assert len(from_path.pages) == len(expected.pages)


def test_parse_timeout_requires_pool():
    with pytest.raises(ValueError, match="pool_size"):
        LiteParse(parse_timeout=5)


def test_timeout_kills_worker_and_pool_recovers(sample_pdf):
    slow_doc = _grid_pdf(64000)  # parses in seconds, not before the deadline
    with LiteParse(**CFG, pool_size=1, parse_timeout=0.5) as pooled:
        pooled.warm_up()
        with pytest.raises(ParseTimeoutError) as excinfo:
            pooled.parse(slow_doc)
        assert excinfo.value.timeout == 0.5
        assert "bytes" in excinfo.value.source
        # The replacement worker must serve subsequent parses.
        result = pooled.parse(sample_pdf)
        assert result.text


def test_timeout_names_the_file(tmp_path):
    slow_path = tmp_path / "rogue.pdf"
    slow_path.write_bytes(_grid_pdf(64000))
    # The doc parses in ~0.35s, so a 0.5s budget only ever tripped because
    # worker spawn + `import liteparse` were charged to it. Pick a timeout the
    # parse itself blows through, so the assertion does not depend on how slow
    # process startup happens to be on the machine running the suite.
    with LiteParse(**CFG, pool_size=1, parse_timeout=0.1) as pooled:
        with pytest.raises(ParseTimeoutError) as excinfo:
            pooled.parse(slow_path)
    assert excinfo.value.source == str(slow_path.absolute())


def test_parse_error_propagates_from_worker():
    with LiteParse(**CFG, pool_size=1) as pooled:
        with pytest.raises(ParseError):
            pooled.parse(b"%PDF-1.4 not actually a pdf")


def test_close_shuts_down_workers(sample_pdf):
    pooled = LiteParse(**CFG, pool_size=2)
    pooled.parse(sample_pdf)
    pooled.close()
    workers = list(pooled._pool._workers)
    assert workers == []
    with pytest.raises(ParseError, match="closed"):
        pooled.parse(sample_pdf)
    pooled.close()  # idempotent