~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Decompression caps for the OpenAlex snapshot partition streaming.
``iter_partitions`` streams each downloaded ``.gz`` partition line by
line and ``json.loads`` per line. Every bound today is on the
*compressed* side (``MAX_RESPONSE_SIZE`` caps the body at 1 GB) — gzip
expands ~1032:1, and nothing bounds the decompressed side:
* a hostile partition with one enormous line is buffered whole by the
line iterator before ``json.loads`` ever sees it (memory spike), and
* a partition whose decompressed stream expands far past what its
manifest entry declared streams unbounded. ``meta.content_length``
is checked by #6438, but only against the *compressed* bytes
received — it counts gz bytes, so it detects a truncated download
and says nothing about how far those bytes expand.
The upstream is a public S3 bucket reached over HTTPS — a compromised
bucket, CDN, or MITM'd redirect hop controls the bytes. These tests pin
two caps:
* an absolute per-line character cap (over-long lines are skipped as
malformed *without ever being materialized whole*), and
* a per-partition decompressed-size bound anchored to the manifest's
declared ``meta.content_length`` (the compressed size, scaled by an
expansion allowance and clamped to an absolute ceiling) — a stream
expanding far past the declaration aborts the partition, whatever
its line structure (fail closed, like the manifest URL allowlist
already does).
"""
import gzip
import io
import json
import random
from unittest.mock import MagicMock
import pytest
from local_deep_research.journal_quality.data_sources._openalex_common import (
_partition_decompressed_cap,
iter_partitions,
)
_ENTRY = "s3://openalex/data/jsonl/sources/part_0.gz"
def _response(content: bytes) -> MagicMock:
resp = MagicMock()
resp.raise_for_status = MagicMock()
resp.content = content
return resp
def _gz_raw(payload: bytes) -> bytes:
buf = io.BytesIO()
with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
gz.write(payload)
return buf.getvalue()
def _drain(entries, tmp_path, **kwargs):
safe_get = MagicMock(return_value=_response(kwargs.pop("body")))
out = []
for _idx, _total, records in iter_partitions(
entries,
tmp_path,
file_prefix="cap",
label="test",
safe_get=safe_get,
**kwargs,
):
out.extend(records)
return out
_REALISTIC_RECORDS = 12_000
def _realistic_partition(n: int = _REALISTIC_RECORDS) -> tuple[bytes, bytes]:
"""Deterministic OpenAlex-shaped payload at the live upper density.
A real sources record is ~1 KB of mostly boilerplate — 38 keys,
most of them ``null``/``false``/``[]``, with the source id repeated
three times — so gzip crushes a partition hard: a census of every
live partition (2026-09-24, taken from the gzip ISIZE trailers) put
the 196 sources partitions at a median 14.23x and a maximum 18.69x
decompressed bytes per compressed byte, and the 80 institutions
partitions at up to 12.17x.
This fixture mirrors that record shape, with per-record variation
(display name, alternate titles, ids, counts) tuned so gzip cannot
crush it past the live tail: it lands at ~18.3:1, just under the
densest live partition. That density is the point — a slacker
fixture would let an expansion allowance that false-aborts real
partitions pass this file's control test, which is exactly how the
original 16x allowance shipped. The test asserts the ratio.
"""
rng = random.Random(6436)
words = (
"advance analysis archive article author bias citation cluster "
"dataset discipline editor faculty gateway hypothesis index "
"journal knowledge library method network ontology publication "
"query repository science taxonomy university venue workflow"
).split()
lines = []
for i in range(n):
source_id = 100_000_000 + (i * 7919) % 90_000_000
lines.append(
json.dumps(
{
"id": f"https://openalex.org/S{source_id}",
"issn_l": None,
"issn": [],
"display_name": " ".join(
rng.choice(words) for _ in range(6)
),
"host_organization": None,
"host_organization_name": None,
"host_organization_lineage": [],
"works_count": rng.randrange(50),
"oa_works_count": 0,
"cited_by_count": rng.randrange(200),
"summary_stats": {
"2yr_mean_citedness": 0.0,
"h_index": 0,
"i10_index": 0,
},
"is_oa": False,
"is_in_doaj": False,
"is_in_doaj_since_year": None,
"is_high_oa_rate": False,
"is_high_oa_rate_since_year": None,
"is_in_scielo": False,
"is_ojs": False,
"is_core": False,
"listed_in": [],
"is_preprint_repository": False,
"oa_flip_year": None,
"first_publication_year": None,
"last_publication_year": None,
"ids": {
"openalex": f"https://openalex.org/S{source_id}",
"issn_l": None,
"issn": None,
"mag": str(source_id),
"wikidata": None,
},
"homepage_url": None,
"apc_prices": [],
"apc_usd": None,
"apc_usd_by_year": None,
"country_code": None,
"societies": [],
"alternate_titles": [
" ".join(rng.choice(words) for _ in range(5))
for _ in range(3)
],
"type": "journal",
"topics": [],
"topic_share": [],
"counts_by_year": [
{
"year": year,
"works_count": rng.randrange(20),
"cited_by_count": rng.randrange(100),
}
for year in (2026, 2025)
],
"works_api_url": (
"https://api.openalex.org/works?filter="
f"primary_location.source.id:S{source_id}"
),
"updated_date": "2026-02-09T16:10:06.000Z",
"created_date": None,
}
)
)
payload = "\n".join(lines).encode() + b"\n"
return payload, _gz_raw(payload)
def test_overlong_line_is_skipped_not_yielded(tmp_path):
"""A single huge JSON line must not survive as a record.
Given a partition whose one line is a valid JSON object just past
the default 10 M-character line cap (no ``max_line_chars`` is passed
here — the production default is what is under test), When streamed,
Then the line is skipped as malformed (never yielded) and the small
lines around it still parse — without materializing the huge line.
"""
# Just past the default 10M-char cap; real OpenAlex records are a
# few KB, so anything this size is hostile padding.
huge = b'{"pad": "' + b"A" * 10_000_001 + b'"}'
body = _gz_raw(
b'{"id": "ok-before"}\n' + huge + b"\n" + b'{"id": "ok-after"}\n'
)
records = _drain(
[{"url": _ENTRY}],
tmp_path,
body=body,
)
assert [r["id"] for r in records] == ["ok-before", "ok-after"]
def test_partition_exceeding_declared_content_length_aborts(tmp_path):
"""A stream expanding far past the manifest declaration must abort.
Given a manifest entry declaring its own (truthful) compressed size
while the partition decompresses to megabytes of valid JSONL — an
expansion of ~74:1, far past the 32x allowance — When streamed,
Then the partition fails closed with a ValueError naming the
decompression bound — the same refuse-the-whole-fetch posture the
manifest URL allowlist already takes.
The declaration has to be truthful: #6438 verifies
``meta.content_length`` against the bytes actually received and
refuses a mismatch before anything is decompressed, so a fabricated
small declaration would abort on that transport check instead of
reaching the bound this test exists to pin. Anchoring on the real
compressed size is also the stronger fixture — it is what the live
manifest declares.
"""
lines = b"\n".join(
json.dumps({"id": f"S{n}", "pad": "x" * 200}).encode()
for n in range(20_000)
) # ~4.5 MB decompressed
body = _gz_raw(lines) # ~60 KB compressed -> a ~3 MB bound
with pytest.raises(ValueError, match="decompression"):
_drain(
[{"url": _ENTRY, "meta": {"content_length": len(body)}}],
tmp_path,
body=body,
)
def test_partition_declaring_compressed_size_streams(tmp_path):
"""Control: a partition at the live density must stream, not abort.
Given a manifest entry declaring ``meta.content_length`` equal to
the partition's *compressed* gz size (what the live manifest
actually declares) with the payload expanding at the live fleet's
upper tail — ~18.3 characters per compressed byte, just under the
densest live sources partition's 18.67 — When streamed, Then every
record parses: the anchored cap must admit legitimate partitions.
This is the guard on the expansion allowance, so the fixture's own
density is asserted first. At this density the cap admits the
partition only from 18x upward; the shipped 32x clears it with ~1.7x
to spare, while the 16x this file first pinned would abort here —
as it did against the real ``sources`` fleet.
"""
payload, body = _realistic_partition()
ratio = len(payload) / len(body)
# Pin the fixture's density. A fixture that drifted back down to the
# ~9:1 of a mostly-random payload would pass for allowances that
# false-abort real partitions, which is the bug this control exists
# to catch.
assert 17 <= ratio <= 20, f"fixture density drifted to {ratio:.2f}:1"
assert len(payload) <= len(body) * 32
# The control only constrains the allowance if the fixture is large
# enough that the 1 MiB slack cannot rescue a too-small multiplier:
# a 17x allowance must still refuse it. Shrinking _REALISTIC_RECORDS
# would silently weaken this test without tripping the ratio guard.
assert len(payload) > len(body) * 17 + 1024**2
records = _drain(
[{"url": _ENTRY, "meta": {"content_length": len(body)}}],
tmp_path,
body=body,
)
assert len(records) == _REALISTIC_RECORDS
assert records[0]["id"] == "https://openalex.org/S100000000"
def test_all_overlong_line_bomb_aborts(tmp_path):
"""A bomb made only of over-long lines must abort, not stream.
Given a partition whose every line exceeds the line cap — so every
line takes the skip path and is discarded in chunks — with its own
truthful compressed size declared (~7 KB, anchoring a ~1.27 MB
bound; see the note on #6438's transport check above), When
streamed, Then the discarded chunks still count toward the bound
and the partition aborts: the skip path is not an uncounted bypass.
"""
# 101-char physical lines; with max_line_chars=64 the first
# readline(65) never sees the newline, so every line is over-long.
payload = (b"A" * 100 + b"\n") * 20_000 # ~2 MB decompressed
body = _gz_raw(payload)
with pytest.raises(ValueError, match="decompression"):
_drain(
[{"url": _ENTRY, "meta": {"content_length": len(body)}}],
tmp_path,
body=body,
max_line_chars=64,
)
def test_discard_loop_chunks_count_toward_the_partition_cap(tmp_path):
"""Discarded remainder chunks must themselves trip the bound.
Given a partition of only a handful of lines, each millions of
characters past the line cap — so the outer loop's first-chunk
counting accumulates almost nothing (5 x 65 chars) — with its own
truthful compressed size declared (~10 KB, anchoring a ~1.36 MB
bound; see the note on #6438's transport check above), When
streamed, Then the discard loop's own chunk counting trips the
bound and the partition aborts: without the inner-loop counting,
such a bomb streams through uncounted.
"""
# 5 lines x ~2 MB each: only the inner discard loop's chunks can
# carry seen_chars past the cap (len(body) * 32 + 1 MiB slack).
payload = (b"A" * 2_000_000 + b"\n") * 5
body = _gz_raw(payload)
with pytest.raises(ValueError, match="decompression"):
_drain(
[{"url": _ENTRY, "meta": {"content_length": len(body)}}],
tmp_path,
body=body,
max_line_chars=64,
)
def test_hostile_content_length_declaration_is_clamped():
"""A hostile huge declaration cannot buy a bigger bomb budget.
Given a manifest entry declaring ``content_length`` of 10^12 bytes
— a pure ``multiple-of-declaration`` anchor would put the bound at
~32 TB, strictly worse than the 4 GiB fallback a meta-less entry
gets — When the cap is computed, Then it is clamped to the 4 GiB
absolute ceiling, while a sane declaration still anchors at 32x
the compressed size plus 1 MB slack.
The allowance is 32 because the live fleet's densest partition
expands 18.69x (census of all 276 live partitions, 2026-09-24,
from their gzip ISIZE trailers): 32 clears that by ~1.7x, and still
aborts every bomb fixture in this file (74:1, 293:1, 1024:1).
"""
assert (
_partition_decompressed_cap(
{"url": _ENTRY, "meta": {"content_length": 10**12}}
)
== 4 * 1024**3
)
assert (
_partition_decompressed_cap(
{"url": _ENTRY, "meta": {"content_length": 1000}}
)
== 1000 * 32 + 1024**2
)
def test_bool_content_length_does_not_anchor_a_tiny_cap():
"""A boolean declaration is not a size.
Given a manifest entry whose ``content_length`` is ``True`` — which
``isinstance(declared, int)`` accepts, since ``bool`` subclasses
``int`` — When the cap is computed, Then the 4 GiB fallback applies
rather than an anchor of ``1 * 32 + 1 MiB``, which would abort every
real partition as a "bomb" under a hostile or malformed manifest.
The transport-side ``content_length`` check already excludes
``bool`` this way; the two must agree on what counts as declared.
"""
assert (
_partition_decompressed_cap(
{"url": _ENTRY, "meta": {"content_length": True}}
)
== 4 * 1024**3
)
# ``False`` is a falsy int and already took the fallback path; it
# must keep doing so for the same reason.
assert (
_partition_decompressed_cap(
{"url": _ENTRY, "meta": {"content_length": False}}
)
== 4 * 1024**3
)
def test_live_sources_partition_cap_clears_the_real_stream():
"""Pinned regression against today's live snapshot.
``files[0]`` of the live sources manifest
(``s3://openalex/data/jsonl/sources/.../part_0000.gz``, fetched
2026-09-24) declares ``content_length`` 560,290 compressed bytes and
decompresses to 10,461,631 characters — 18.67 per compressed byte
(18.69 by the gzip ISIZE trailer, which counts bytes), the densest
of the 196 live sources partitions (median 14.23; the 80
institutions partitions top out at 12.17). The 16x allowance this
file first pinned put the cap at 10,013,216 and aborted it, and
``OpenAlexSource.required`` is True, so that failed every
journal-quality download. The cap must clear the real stream.
"""
cap = _partition_decompressed_cap(
{"url": _ENTRY, "meta": {"content_length": 560_290}}
)
assert cap == 560_290 * 32 + 1024**2 # 18,977,856 characters
assert cap > 10_461_631 # the measured stream, with ~1.8x to spare