~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): verify manifest-declared md5 for OpenAlex partitions (#6438)
LearningCircuitc71adbcc45d3
diff --git a/changelog.d/6438.security.md b/changelog.d/6438.security.mdnew file mode 100644index 0000000000..d492a9a6b0--- /dev/null+++ b/changelog.d/6438.security.md@@ -0,0 +1 @@+OpenAlex snapshot partitions now get a transport-corruption check: when a manifest entry declares `meta.content_length` and/or a 32-hex `meta.md5`, the downloaded partition is compared against them before being written to disk. A size or hash mismatch aborts that refresh (the previous snapshot is left in place; nothing partial is persisted), logging the partition and the mismatched values. This is a corruption/truncation guard, not an authentication control: the declared length and digest come from the same manifest fetch as the bytes they describe, so a party able to substitute a partition can substitute those values too — what this catches is a garbled or truncated transfer, not a compromised origin. `content_length` is present on every partition in OpenAlex's current manifests, so that half is active today; `md5` is not currently populated on any live OpenAlex manifest, so that half stays dormant until upstream adds it. A manifest that omits a field, or declares a digest that isn't a plain or quoted 32-character hex string, is treated as if that field weren't declared — a malformed digest logs a warning naming the partition (never the value) and the partition downloads normally.diff --git a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py b/src/local_deep_research/journal_quality/data_sources/_openalex_common.pyindex 0d369cbddd..4e0de7598b 100644--- a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py+++ b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py@@ -13,6 +13,7 @@ lifecycle.from __future__ import annotationsimport gzip+import hashlibimport jsonfrom pathlib import Pathfrom typing import Callable, Iterator, Tuple@@ -74,6 +75,36 @@ def validate_manifest_entries(entries: list[dict], label: str) -> None:_PARTITION_MAX_RETRIES = 5_PARTITION_BACKOFF_SECONDS = (2, 5, 10, 20, 40)+_MD5_HEX_LENGTH = 32+_HEX_DIGIT_CHARS = frozenset("0123456789abcdefABCDEF")+++def _normalize_md5(declared: object) -> str | None:+ """Return a lowercase 32-hex digest from a manifest ``meta.md5`` value.++ Accepts a bare hex digest or the same digest wrapped in one matching+ pair of double quotes (the S3 ETag wire form for a non-multipart+ object). Returns ``None`` for anything else — a multipart ETag+ (``<hex>-<n>``), a base64 digest, the wrong length, non-hex+ characters, or a non-string value — so the caller can skip+ verification for this partition instead of comparing bytes against+ a value that was never a valid md5 hex digest.+ """+ if not isinstance(declared, str):+ return None+ candidate = declared.strip()+ if (+ len(candidate) == _MD5_HEX_LENGTH + 2+ and candidate[0] == '"'+ and candidate[-1] == '"'+ ):+ candidate = candidate[1:-1]+ if len(candidate) == _MD5_HEX_LENGTH and all(+ c in _HEX_DIGIT_CHARS for c in candidate+ ):+ return candidate.lower()+ return None+def iter_partitions(entries: list[dict],@@ -127,6 +158,14 @@ def iter_partitions(longer schedule than the generic ``safe_get_with_retries``(1, 2, 4) so we ride out a sustained S3 blip instead ofburning all retries inside the same bad window.++ Raises:+ ValueError: If a partition's declared ``meta.content_length``+ doesn't match the number of bytes actually received, or+ its declared ``meta.md5`` (once normalized to a bare+ digest) doesn't match the digest of the bytes received —+ either aborts before anything is written to disk, so the+ previous snapshot is left in place."""malformed_total = 0total_parts = len(entries)@@ -173,6 +212,81 @@ def iter_partitions(backoff_times=backoff_times,)resp.raise_for_status()+ meta = entry.get("meta") or {}++ # Transport-corruption check when the manifest declares an+ # expected size: compressed bytes whose length doesn't match+ # meta.content_length were truncated or corrupted in+ # transit — refuse before writing them to disk. This is not+ # an authentication check: the declared length comes from+ # the same manifest fetch as the bytes it describes, so it+ # can't detect a hostile origin, only a garbled download. A+ # declared value that isn't a plain integer (a string,+ # float, or bool) is logged and skipped rather than treated+ # as a mismatch.+ declared_length = meta.get("content_length")+ if declared_length is not None:+ if isinstance(declared_length, int) and not isinstance(+ declared_length, bool+ ):+ actual_length = len(resp.content)+ if actual_length != declared_length:+ raise ValueError(+ f"{label} partition {idx}: content_length "+ f"mismatch — manifest declares "+ f"{declared_length} bytes but received "+ f"{actual_length} bytes; "+ "refusing possibly corrupted or truncated "+ "partition"+ )+ else:+ shape = (+ f"{len(declared_length)}-character string"+ if isinstance(declared_length, str)+ else type(declared_length).__name__+ )+ logger.warning(+ f"{label} partition {idx}: manifest "+ f"content_length is not an integer ({shape}); "+ "skipping the size check for this partition"+ )++ # Transport-corruption check when the manifest declares a+ # digest: bytes that hash differently than meta.md5 were+ # altered or corrupted in transit — refuse before writing+ # them to disk. Like the content_length check above, this+ # is not an authentication check: the digest comes from the+ # same manifest fetch as the bytes it certifies, so a party+ # able to substitute the partition can substitute the+ # declared md5 too. A declared value that isn't a 32-hex+ # digest (plain or ETag-quoted) is logged and skipped+ # rather than treated as a mismatch.+ declared_md5 = meta.get("md5")+ if declared_md5 is not None:+ normalized_md5 = _normalize_md5(declared_md5)+ if normalized_md5 is None:+ shape = (+ f"{len(declared_md5)}-character string"+ if isinstance(declared_md5, str)+ else type(declared_md5).__name__+ )+ logger.warning(+ f"{label} partition {idx}: manifest md5 is not "+ f"a 32-character hex digest ({shape}); skipping "+ "the integrity check for this partition"+ )+ else:+ actual_md5 = hashlib.md5(+ resp.content, usedforsecurity=False+ ).hexdigest()+ if actual_md5 != normalized_md5:+ raise ValueError(+ f"{label} partition {idx}: md5 mismatch — "+ f"manifest declares {normalized_md5!r} but "+ f"received {actual_md5!r}; refusing possibly "+ "corrupted partition"+ )+tmp_part.write_bytes(resp.content)# Unpin the compressed body; the suspended frame would# otherwise hold it until the caller drains the partition.diff --git a/tests/security/test_journal_partition_md5_verification.py b/tests/security/test_journal_partition_md5_verification.pynew file mode 100644index 0000000000..0610066edc--- /dev/null+++ b/tests/security/test_journal_partition_md5_verification.py@@ -0,0 +1,313 @@+"""Transport-corruption verification for OpenAlex snapshot partitions.++The journal-quality dataset fetchers already defend a compromised+manifest with a URL prefix allowlist (pinned separately). What they did+not check is whether the bytes actually received match what the+manifest itself describes: OpenAlex manifest entries can carry+``meta.content_length`` (present on every entry in the live snapshot)+and ``meta.md5`` (absent from every live entry today, but part of the+documented manifest format) per partition, and the fetcher did not+compare either to the downloaded bytes.++This is a transport-corruption / truncation guard, not an+authentication control, and the tests below verify it as one: the+reference length and digest are fetched from the same manifest, over+the same channel, as the bytes they describe, so a party able to+substitute the partition can substitute those reference values too.+What this catches is a bit-flipped or truncated transfer — a real and+far more likely failure mode for a ~10 MB streamed download than a+hostile origin.++These tests pin: a declared ``content_length`` or ``md5`` that doesn't+match the bytes received aborts the fetch (the previous snapshot is+kept, nothing partial is persisted); a matching declaration, or no+declaration at all, streams unchanged; and a declared ``md5`` that+isn't a plain or ETag-quoted 32-hex digest is logged and skipped+rather than compared or treated as fatal.+"""++import gzip+import hashlib+import io+from unittest.mock import MagicMock++import pytest++from local_deep_research.journal_quality.data_sources._openalex_common import (+ iter_partitions,+)++_ENTRY = "s3://openalex/data/jsonl/sources/part_0.gz"+++def _response(content: bytes) -> MagicMock:+ resp = MagicMock()+ resp.raise_for_status = MagicMock()+ resp.content = content+ return resp+++def _gz(payload: bytes) -> bytes:+ buf = io.BytesIO()+ with gzip.GzipFile(fileobj=buf, mode="wb") as gz:+ gz.write(payload)+ return buf.getvalue()+++def _drain(entry, tmp_path, body):+ safe_get = MagicMock(return_value=_response(body))+ out = []+ for _idx, _total, records in iter_partitions(+ [entry],+ tmp_path,+ file_prefix="journal",+ label="test",+ safe_get=safe_get,+ ):+ out.extend(records)+ return out+++def _md5(data: bytes) -> str:+ return hashlib.md5(data, usedforsecurity=False).hexdigest()+++def _big_records(n: int) -> list:+ """``n`` distinct JSON lines with varied (not merely repeated) content.++ Varied content keeps gzip from crushing the body down to a few+ dozen bytes. The compressed body must stay above 1 KiB so a+ truncated or partial-hash read is distinguishable from the real+ thing — a tiny fixture would let a "hash/compare only the first N+ bytes" bug pass every test silently.+ """+ return [+ b'{"id": "S%d", "h": %d, "issn": "%04d-%04d"}'+ % (i, (i * 37) % 9973, (i * 101) % 10000, (i * 233) % 10000)+ for i in range(n)+ ]+++_BIG_PAYLOAD = b"\n".join(_big_records(150)) + b"\n"+_BODY = _gz(_BIG_PAYLOAD)+assert len(_BODY) > 1024, (+ "fixture body must compress to more than 1 KiB so truncation is "+ f"distinguishable from the full body; got {len(_BODY)} bytes"+)+_IDS = [f"S{i}" for i in range(150)]+++def test_partition_failing_declared_md5_is_refused(tmp_path):+ """Content hashing differently than the manifest says must abort.++ Asserts the exact message, not a coarse ``match="md5"`` — a mutant+ that raises *a* ValueError for the wrong reason (e.g. hashing a+ truncated slice of the body, or reporting the wrong pair of+ digests) must not be able to slip past by coincidentally+ mentioning "md5" somewhere. It does not by itself prove the+ comparison checks the full digest rather than a prefix — see+ ``test_partition_failing_declared_md5_with_shared_prefix_is_refused``+ below for that.+ """+ wrong_md5 = _md5(b"totally different bytes")+ entry = {"url": _ENTRY, "meta": {"md5": wrong_md5}}+ actual_md5 = _md5(_BODY)++ with pytest.raises(ValueError) as exc_info:+ _drain(entry, tmp_path, _BODY)++ assert str(exc_info.value) == (+ f"test partition 0: md5 mismatch — manifest declares "+ f"{wrong_md5!r} but received {actual_md5!r}; refusing possibly "+ "corrupted partition"+ )+++def test_partition_failing_declared_md5_with_shared_prefix_is_refused(+ tmp_path,+):+ """A wrong digest sharing the real one's leading hex characters+ must still be refused.++ Pins full-digest comparison against a mutant that compares only a+ prefix (e.g. ``actual_md5[:8] != normalized_md5[:8]``): built to+ differ from the real digest only in its last character, this value+ would look like a match to any such prefix-only comparison and be+ wrongly accepted instead of refused.+ """+ actual_md5 = _md5(_BODY)+ flipped_last = "0" if actual_md5[-1] != "0" else "1"+ wrong_md5 = actual_md5[:-1] + flipped_last+ assert wrong_md5 != actual_md5+ assert wrong_md5[:8] == actual_md5[:8]+ entry = {"url": _ENTRY, "meta": {"md5": wrong_md5}}++ with pytest.raises(ValueError) as exc_info:+ _drain(entry, tmp_path, _BODY)++ assert str(exc_info.value) == (+ f"test partition 0: md5 mismatch — manifest declares "+ f"{wrong_md5!r} but received {actual_md5!r}; refusing possibly "+ "corrupted partition"+ )+++def test_partition_matching_declared_md5_streams(tmp_path):+ """Content matching the declaration streams normally.++ Uses the >1 KiB ``_BODY`` fixture: a mutant that hashes only+ ``resp.content[:1024]`` would compute a different digest than the+ one declared here (over the full body) and wrongly raise.+ """+ entry = {"url": _ENTRY, "meta": {"md5": _md5(_BODY)}}++ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+++def test_manifest_without_md5_streams_unchanged(tmp_path):+ """No declaration, no verification — current behaviour preserved."""+ records = _drain({"url": _ENTRY}, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+++def test_partition_quoted_uppercase_declared_md5_streams(+ tmp_path, loguru_caplog_full+):+ """A quoted, uppercase ETag-style digest is accepted and matched.++ ``meta.md5`` isn't populated on any live OpenAlex manifest today,+ but AWS's own ETag wire form wraps a non-multipart object's digest+ in double quotes; accept that shape case-insensitively rather than+ treating it as malformed the day upstream starts declaring one.++ Asserts the malformed-shape warning is absent, not just that the+ records come through unchanged — the malformed-skip path streams+ every record too, so ``== _IDS`` alone can't distinguish "digest+ verified" from "quote-stripping broke and this was silently+ skipped instead."+ """+ quoted = f'"{_md5(_BODY).upper()}"'+ entry = {"url": _ENTRY, "meta": {"md5": quoted}}++ with loguru_caplog_full.at_level("WARNING"):+ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+ assert "not a 32-character hex digest" not in loguru_caplog_full.text+++def test_partition_malformed_declared_md5_is_skipped_with_warning(+ tmp_path, loguru_caplog_full+):+ """A multipart ETag (``<hex>-<n>``) isn't a 32-hex digest.++ It must not be compared byte-for-byte against a plain md5 (it+ would never match, hard-aborting every fetch the day upstream+ declares this shape) and must not be silently ignored either — log+ a warning identifying the partition and the malformed shape, then+ continue downloading as if no digest were declared.+ """+ malformed = f"{_md5(_BODY)}-3"+ entry = {"url": _ENTRY, "meta": {"md5": malformed}}++ with loguru_caplog_full.at_level("WARNING"):+ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+ assert (+ "test partition 0: manifest md5 is not a 32-character hex "+ "digest (34-character string); skipping the integrity check "+ "for this partition"+ ) in loguru_caplog_full.text+ # The malformed value itself must never be logged, only its shape.+ assert malformed not in loguru_caplog_full.text+++def test_partition_non_string_declared_md5_is_skipped_with_warning(+ tmp_path, loguru_caplog_full+):+ """A non-string ``meta.md5`` (e.g. a JSON number) is skipped, not fatal.++ The warning names the value's type, never the value — the same+ code path has to handle whatever shape a malformed manifest sends.+ """+ entry = {"url": _ENTRY, "meta": {"md5": 123456}}++ with loguru_caplog_full.at_level("WARNING"):+ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+ assert (+ "test partition 0: manifest md5 is not a 32-character hex "+ "digest (int); skipping the integrity check for this partition"+ ) in loguru_caplog_full.text+++ "delta",+ [-100, 100],+ ids=[+ "declared-shorter-than-received",+ "declared-longer-than-received",+ ],+)+def test_partition_content_length_mismatch_is_refused(tmp_path, delta):+ """A declared content_length that doesn't match the bytes received+ must abort in both directions, checked (and raised) before the+ costlier md5 comparison runs.++ A shorter declaration (``delta`` negative, the only direction+ previously tested here) catches a corrupted transfer padded with+ extra bytes; a longer declaration (``delta`` positive) catches the+ opposite and more likely failure — a truncated download that fell+ short of what the manifest promised.+ """+ wrong_length = len(_BODY) + delta+ entry = {"url": _ENTRY, "meta": {"content_length": wrong_length}}++ with pytest.raises(ValueError) as exc_info:+ _drain(entry, tmp_path, _BODY)++ assert str(exc_info.value) == (+ f"test partition 0: content_length mismatch — manifest "+ f"declares {wrong_length} bytes but received {len(_BODY)} "+ "bytes; refusing possibly corrupted or truncated partition"+ )+++def test_partition_content_length_match_streams(tmp_path):+ """A correctly declared content_length streams normally."""+ entry = {"url": _ENTRY, "meta": {"content_length": len(_BODY)}}++ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+++def test_partition_malformed_content_length_is_skipped_with_warning(+ tmp_path, loguru_caplog_full+):+ """A string content_length isn't silently ignored.++ Even one that spells the correct length right, only as the wrong+ type, must still warn and continue downloading — mirroring how a+ malformed ``meta.md5`` is handled — rather than disabling the size+ check in total silence.+ """+ declared = str(len(_BODY))+ entry = {"url": _ENTRY, "meta": {"content_length": declared}}++ with loguru_caplog_full.at_level("WARNING"):+ records = _drain(entry, tmp_path, _BODY)++ assert [r["id"] for r in records] == _IDS+ assert (+ f"test partition 0: manifest content_length is not an integer "+ f"({len(declared)}-character string); skipping the size check "+ "for this partition"+ ) in loguru_caplog_full.text+ # The malformed value itself must never be logged, only its shape.+ assert declared not in loguru_caplog_full.text