Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

16.1 KB
Raw Blame
"""Decompression caps for the OpenAlex snapshot partition streaming.

``iter_partitions`` streams each downloaded ``.gz`` partition line by
line and ``json.loads`` per line. Every bound today is on the
*compressed* side (``MAX_RESPONSE_SIZE`` caps the body at 1 GB) — gzip
expands ~1032:1, and nothing bounds the decompressed side:

* a hostile partition with one enormous line is buffered whole by the
  line iterator before ``json.loads`` ever sees it (memory spike), and
* a partition whose decompressed stream expands far past what its
  manifest entry declared streams unbounded. ``meta.content_length``
  is checked by #6438, but only against the *compressed* bytes
  received — it counts gz bytes, so it detects a truncated download
  and says nothing about how far those bytes expand.

The upstream is a public S3 bucket reached over HTTPS — a compromised
bucket, CDN, or MITM'd redirect hop controls the bytes. These tests pin
two caps:

* an absolute per-line character cap (over-long lines are skipped as
  malformed *without ever being materialized whole*), and
* a per-partition decompressed-size bound anchored to the manifest's
  declared ``meta.content_length`` (the compressed size, scaled by an
  expansion allowance and clamped to an absolute ceiling) — a stream
  expanding far past the declaration aborts the partition, whatever
  its line structure (fail closed, like the manifest URL allowlist
  already does).
"""

import gzip
import io
import json
import random
from unittest.mock import MagicMock

import pytest

from local_deep_research.journal_quality.data_sources._openalex_common import (
    _partition_decompressed_cap,
    iter_partitions,
)

_ENTRY = "s3://openalex/data/jsonl/sources/part_0.gz"


def _response(content: bytes) -> MagicMock:
    resp = MagicMock()
    resp.raise_for_status = MagicMock()
    resp.content = content
    return resp


def _gz_raw(payload: bytes) -> bytes:
    buf = io.BytesIO()
    with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
        gz.write(payload)
    return buf.getvalue()


def _drain(entries, tmp_path, **kwargs):
    safe_get = MagicMock(return_value=_response(kwargs.pop("body")))
    out = []
    for _idx, _total, records in iter_partitions(
        entries,
        tmp_path,
        file_prefix="cap",
        label="test",
        safe_get=safe_get,
        **kwargs,
    ):
        out.extend(records)
    return out


_REALISTIC_RECORDS = 12_000


def _realistic_partition(n: int = _REALISTIC_RECORDS) -> tuple[bytes, bytes]:
    """Deterministic OpenAlex-shaped payload at the live upper density.

    A real sources record is ~1 KB of mostly boilerplate — 38 keys,
    most of them ``null``/``false``/``[]``, with the source id repeated
    three times — so gzip crushes a partition hard: a census of every
    live partition (2026-09-24, taken from the gzip ISIZE trailers) put
    the 196 sources partitions at a median 14.23x and a maximum 18.69x
    decompressed bytes per compressed byte, and the 80 institutions
    partitions at up to 12.17x.

    This fixture mirrors that record shape, with per-record variation
    (display name, alternate titles, ids, counts) tuned so gzip cannot
    crush it past the live tail: it lands at ~18.3:1, just under the
    densest live partition. That density is the point — a slacker
    fixture would let an expansion allowance that false-aborts real
    partitions pass this file's control test, which is exactly how the
    original 16x allowance shipped. The test asserts the ratio.
    """
    rng = random.Random(6436)
    words = (
        "advance analysis archive article author bias citation cluster "
        "dataset discipline editor faculty gateway hypothesis index "
        "journal knowledge library method network ontology publication "
        "query repository science taxonomy university venue workflow"
    ).split()
    lines = []
    for i in range(n):
        source_id = 100_000_000 + (i * 7919) % 90_000_000
        lines.append(
            json.dumps(
                {
                    "id": f"https://openalex.org/S{source_id}",
                    "issn_l": None,
                    "issn": [],
                    "display_name": " ".join(
                        rng.choice(words) for _ in range(6)
                    ),
                    "host_organization": None,
                    "host_organization_name": None,
                    "host_organization_lineage": [],
                    "works_count": rng.randrange(50),
                    "oa_works_count": 0,
                    "cited_by_count": rng.randrange(200),
                    "summary_stats": {
                        "2yr_mean_citedness": 0.0,
                        "h_index": 0,
                        "i10_index": 0,
                    },
                    "is_oa": False,
                    "is_in_doaj": False,
                    "is_in_doaj_since_year": None,
                    "is_high_oa_rate": False,
                    "is_high_oa_rate_since_year": None,
                    "is_in_scielo": False,
                    "is_ojs": False,
                    "is_core": False,
                    "listed_in": [],
                    "is_preprint_repository": False,
                    "oa_flip_year": None,
                    "first_publication_year": None,
                    "last_publication_year": None,
                    "ids": {
                        "openalex": f"https://openalex.org/S{source_id}",
                        "issn_l": None,
                        "issn": None,
                        "mag": str(source_id),
                        "wikidata": None,
                    },
                    "homepage_url": None,
                    "apc_prices": [],
                    "apc_usd": None,
                    "apc_usd_by_year": None,
                    "country_code": None,
                    "societies": [],
                    "alternate_titles": [
                        " ".join(rng.choice(words) for _ in range(5))
                        for _ in range(3)
                    ],
                    "type": "journal",
                    "topics": [],
                    "topic_share": [],
                    "counts_by_year": [
                        {
                            "year": year,
                            "works_count": rng.randrange(20),
                            "cited_by_count": rng.randrange(100),
                        }
                        for year in (2026, 2025)
                    ],
                    "works_api_url": (
                        "https://api.openalex.org/works?filter="
                        f"primary_location.source.id:S{source_id}"
                    ),
                    "updated_date": "2026-02-09T16:10:06.000Z",
                    "created_date": None,
                }
            )
        )
    payload = "\n".join(lines).encode() + b"\n"
    return payload, _gz_raw(payload)


def test_overlong_line_is_skipped_not_yielded(tmp_path):
    """A single huge JSON line must not survive as a record.

    Given a partition whose one line is a valid JSON object just past
    the default 10 M-character line cap (no ``max_line_chars`` is passed
    here — the production default is what is under test), When streamed,
    Then the line is skipped as malformed (never yielded) and the small
    lines around it still parse — without materializing the huge line.
    """
    # Just past the default 10M-char cap; real OpenAlex records are a
    # few KB, so anything this size is hostile padding.
    huge = b'{"pad": "' + b"A" * 10_000_001 + b'"}'
    body = _gz_raw(
        b'{"id": "ok-before"}\n' + huge + b"\n" + b'{"id": "ok-after"}\n'
    )

    records = _drain(
        [{"url": _ENTRY}],
        tmp_path,
        body=body,
    )

    assert [r["id"] for r in records] == ["ok-before", "ok-after"]


def test_partition_exceeding_declared_content_length_aborts(tmp_path):
    """A stream expanding far past the manifest declaration must abort.

    Given a manifest entry declaring its own (truthful) compressed size
    while the partition decompresses to megabytes of valid JSONL — an
    expansion of ~74:1, far past the 32x allowance — When streamed,
    Then the partition fails closed with a ValueError naming the
    decompression bound — the same refuse-the-whole-fetch posture the
    manifest URL allowlist already takes.

    The declaration has to be truthful: #6438 verifies
    ``meta.content_length`` against the bytes actually received and
    refuses a mismatch before anything is decompressed, so a fabricated
    small declaration would abort on that transport check instead of
    reaching the bound this test exists to pin. Anchoring on the real
    compressed size is also the stronger fixture — it is what the live
    manifest declares.
    """
    lines = b"\n".join(
        json.dumps({"id": f"S{n}", "pad": "x" * 200}).encode()
        for n in range(20_000)
    )  # ~4.5 MB decompressed
    body = _gz_raw(lines)  # ~60 KB compressed -> a ~3 MB bound

    with pytest.raises(ValueError, match="decompression"):
        _drain(
            [{"url": _ENTRY, "meta": {"content_length": len(body)}}],
            tmp_path,
            body=body,
        )


def test_partition_declaring_compressed_size_streams(tmp_path):
    """Control: a partition at the live density must stream, not abort.

    Given a manifest entry declaring ``meta.content_length`` equal to
    the partition's *compressed* gz size (what the live manifest
    actually declares) with the payload expanding at the live fleet's
    upper tail — ~18.3 characters per compressed byte, just under the
    densest live sources partition's 18.67 — When streamed, Then every
    record parses: the anchored cap must admit legitimate partitions.

    This is the guard on the expansion allowance, so the fixture's own
    density is asserted first. At this density the cap admits the
    partition only from 18x upward; the shipped 32x clears it with ~1.7x
    to spare, while the 16x this file first pinned would abort here —
    as it did against the real ``sources`` fleet.
    """
    payload, body = _realistic_partition()
    ratio = len(payload) / len(body)
    # Pin the fixture's density. A fixture that drifted back down to the
    # ~9:1 of a mostly-random payload would pass for allowances that
    # false-abort real partitions, which is the bug this control exists
    # to catch.
    assert 17 <= ratio <= 20, f"fixture density drifted to {ratio:.2f}:1"
    assert len(payload) <= len(body) * 32
    # The control only constrains the allowance if the fixture is large
    # enough that the 1 MiB slack cannot rescue a too-small multiplier:
    # a 17x allowance must still refuse it. Shrinking _REALISTIC_RECORDS
    # would silently weaken this test without tripping the ratio guard.
    assert len(payload) > len(body) * 17 + 1024**2

    records = _drain(
        [{"url": _ENTRY, "meta": {"content_length": len(body)}}],
        tmp_path,
        body=body,
    )

    assert len(records) == _REALISTIC_RECORDS
    assert records[0]["id"] == "https://openalex.org/S100000000"


def test_all_overlong_line_bomb_aborts(tmp_path):
    """A bomb made only of over-long lines must abort, not stream.

    Given a partition whose every line exceeds the line cap — so every
    line takes the skip path and is discarded in chunks — with its own
    truthful compressed size declared (~7 KB, anchoring a ~1.27 MB
    bound; see the note on #6438's transport check above), When
    streamed, Then the discarded chunks still count toward the bound
    and the partition aborts: the skip path is not an uncounted bypass.
    """
    # 101-char physical lines; with max_line_chars=64 the first
    # readline(65) never sees the newline, so every line is over-long.
    payload = (b"A" * 100 + b"\n") * 20_000  # ~2 MB decompressed
    body = _gz_raw(payload)

    with pytest.raises(ValueError, match="decompression"):
        _drain(
            [{"url": _ENTRY, "meta": {"content_length": len(body)}}],
            tmp_path,
            body=body,
            max_line_chars=64,
        )


def test_discard_loop_chunks_count_toward_the_partition_cap(tmp_path):
    """Discarded remainder chunks must themselves trip the bound.

    Given a partition of only a handful of lines, each millions of
    characters past the line cap — so the outer loop's first-chunk
    counting accumulates almost nothing (5 x 65 chars) — with its own
    truthful compressed size declared (~10 KB, anchoring a ~1.36 MB
    bound; see the note on #6438's transport check above), When
    streamed, Then the discard loop's own chunk counting trips the
    bound and the partition aborts: without the inner-loop counting,
    such a bomb streams through uncounted.
    """
    # 5 lines x ~2 MB each: only the inner discard loop's chunks can
    # carry seen_chars past the cap (len(body) * 32 + 1 MiB slack).
    payload = (b"A" * 2_000_000 + b"\n") * 5
    body = _gz_raw(payload)

    with pytest.raises(ValueError, match="decompression"):
        _drain(
            [{"url": _ENTRY, "meta": {"content_length": len(body)}}],
            tmp_path,
            body=body,
            max_line_chars=64,
        )


def test_hostile_content_length_declaration_is_clamped():
    """A hostile huge declaration cannot buy a bigger bomb budget.

    Given a manifest entry declaring ``content_length`` of 10^12 bytes
    — a pure ``multiple-of-declaration`` anchor would put the bound at
    ~32 TB, strictly worse than the 4 GiB fallback a meta-less entry
    gets — When the cap is computed, Then it is clamped to the 4 GiB
    absolute ceiling, while a sane declaration still anchors at 32x
    the compressed size plus 1 MB slack.

    The allowance is 32 because the live fleet's densest partition
    expands 18.69x (census of all 276 live partitions, 2026-09-24,
    from their gzip ISIZE trailers): 32 clears that by ~1.7x, and still
    aborts every bomb fixture in this file (74:1, 293:1, 1024:1).
    """
    assert (
        _partition_decompressed_cap(
            {"url": _ENTRY, "meta": {"content_length": 10**12}}
        )
        == 4 * 1024**3
    )
    assert (
        _partition_decompressed_cap(
            {"url": _ENTRY, "meta": {"content_length": 1000}}
        )
        == 1000 * 32 + 1024**2
    )


def test_bool_content_length_does_not_anchor_a_tiny_cap():
    """A boolean declaration is not a size.

    Given a manifest entry whose ``content_length`` is ``True`` — which
    ``isinstance(declared, int)`` accepts, since ``bool`` subclasses
    ``int`` — When the cap is computed, Then the 4 GiB fallback applies
    rather than an anchor of ``1 * 32 + 1 MiB``, which would abort every
    real partition as a "bomb" under a hostile or malformed manifest.
    The transport-side ``content_length`` check already excludes
    ``bool`` this way; the two must agree on what counts as declared.
    """
    assert (
        _partition_decompressed_cap(
            {"url": _ENTRY, "meta": {"content_length": True}}
        )
        == 4 * 1024**3
    )
    # ``False`` is a falsy int and already took the fallback path; it
    # must keep doing so for the same reason.
    assert (
        _partition_decompressed_cap(
            {"url": _ENTRY, "meta": {"content_length": False}}
        )
        == 4 * 1024**3
    )


def test_live_sources_partition_cap_clears_the_real_stream():
    """Pinned regression against today's live snapshot.

    ``files[0]`` of the live sources manifest
    (``s3://openalex/data/jsonl/sources/.../part_0000.gz``, fetched
    2026-09-24) declares ``content_length`` 560,290 compressed bytes and
    decompresses to 10,461,631 characters — 18.67 per compressed byte
    (18.69 by the gzip ISIZE trailer, which counts bytes), the densest
    of the 196 live sources partitions (median 14.23; the 80
    institutions partitions top out at 12.17). The 16x allowance this
    file first pinned put the cap at 10,013,216 and aborted it, and
    ``OpenAlexSource.required`` is True, so that failed every
    journal-quality download. The cap must clear the real stream.
    """
    cap = _partition_decompressed_cap(
        {"url": _ENTRY, "meta": {"content_length": 560_290}}
    )

    assert cap == 560_290 * 32 + 1024**2  # 18,977,856 characters
    assert cap > 10_461_631  # the measured stream, with ~1.8x to spare