Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): verify manifest-declared md5 for OpenAlex partitions (#6438)

LearningCircuitc71adbcc45d3
diff --git a/changelog.d/6438.security.md b/changelog.d/6438.security.md
new file mode 100644
index 0000000000..d492a9a6b0
--- /dev/null
+++ b/changelog.d/6438.security.md
@@ -0,0 +1 @@
+OpenAlex snapshot partitions now get a transport-corruption check: when a manifest entry declares `meta.content_length` and/or a 32-hex `meta.md5`, the downloaded partition is compared against them before being written to disk. A size or hash mismatch aborts that refresh (the previous snapshot is left in place; nothing partial is persisted), logging the partition and the mismatched values. This is a corruption/truncation guard, not an authentication control: the declared length and digest come from the same manifest fetch as the bytes they describe, so a party able to substitute a partition can substitute those values too — what this catches is a garbled or truncated transfer, not a compromised origin. `content_length` is present on every partition in OpenAlex's current manifests, so that half is active today; `md5` is not currently populated on any live OpenAlex manifest, so that half stays dormant until upstream adds it. A manifest that omits a field, or declares a digest that isn't a plain or quoted 32-character hex string, is treated as if that field weren't declared — a malformed digest logs a warning naming the partition (never the value) and the partition downloads normally.
diff --git a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
index 0d369cbddd..4e0de7598b 100644
--- a/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
+++ b/src/local_deep_research/journal_quality/data_sources/_openalex_common.py
@@ -13,6 +13,7 @@ lifecycle.
from __future__ import annotations
import gzip
+import hashlib
import json
from pathlib import Path
from typing import Callable, Iterator, Tuple
@@ -74,6 +75,36 @@ def validate_manifest_entries(entries: list[dict], label: str) -> None:
_PARTITION_MAX_RETRIES = 5
_PARTITION_BACKOFF_SECONDS = (2, 5, 10, 20, 40)
+_MD5_HEX_LENGTH = 32
+_HEX_DIGIT_CHARS = frozenset("0123456789abcdefABCDEF")
+
+
+def _normalize_md5(declared: object) -> str | None:
+ """Return a lowercase 32-hex digest from a manifest ``meta.md5`` value.
+
+ Accepts a bare hex digest or the same digest wrapped in one matching
+ pair of double quotes (the S3 ETag wire form for a non-multipart
+ object). Returns ``None`` for anything else — a multipart ETag
+ (``<hex>-<n>``), a base64 digest, the wrong length, non-hex
+ characters, or a non-string value — so the caller can skip
+ verification for this partition instead of comparing bytes against
+ a value that was never a valid md5 hex digest.
+ """
+ if not isinstance(declared, str):
+ return None
+ candidate = declared.strip()
+ if (
+ len(candidate) == _MD5_HEX_LENGTH + 2
+ and candidate[0] == '"'
+ and candidate[-1] == '"'
+ ):
+ candidate = candidate[1:-1]
+ if len(candidate) == _MD5_HEX_LENGTH and all(
+ c in _HEX_DIGIT_CHARS for c in candidate
+ ):
+ return candidate.lower()
+ return None
+
def iter_partitions(
entries: list[dict],
@@ -127,6 +158,14 @@ def iter_partitions(
longer schedule than the generic ``safe_get_with_retries``
(1, 2, 4) so we ride out a sustained S3 blip instead of
burning all retries inside the same bad window.
+
+ Raises:
+ ValueError: If a partition's declared ``meta.content_length``
+ doesn't match the number of bytes actually received, or
+ its declared ``meta.md5`` (once normalized to a bare
+ digest) doesn't match the digest of the bytes received —
+ either aborts before anything is written to disk, so the
+ previous snapshot is left in place.
"""
malformed_total = 0
total_parts = len(entries)
@@ -173,6 +212,81 @@ def iter_partitions(
backoff_times=backoff_times,
)
resp.raise_for_status()
+ meta = entry.get("meta") or {}
+
+ # Transport-corruption check when the manifest declares an
+ # expected size: compressed bytes whose length doesn't match
+ # meta.content_length were truncated or corrupted in
+ # transit — refuse before writing them to disk. This is not
+ # an authentication check: the declared length comes from
+ # the same manifest fetch as the bytes it describes, so it
+ # can't detect a hostile origin, only a garbled download. A
+ # declared value that isn't a plain integer (a string,
+ # float, or bool) is logged and skipped rather than treated
+ # as a mismatch.
+ declared_length = meta.get("content_length")
+ if declared_length is not None:
+ if isinstance(declared_length, int) and not isinstance(
+ declared_length, bool
+ ):
+ actual_length = len(resp.content)
+ if actual_length != declared_length:
+ raise ValueError(
+ f"{label} partition {idx}: content_length "
+ f"mismatch — manifest declares "
+ f"{declared_length} bytes but received "
+ f"{actual_length} bytes; "
+ "refusing possibly corrupted or truncated "
+ "partition"
+ )
+ else:
+ shape = (
+ f"{len(declared_length)}-character string"
+ if isinstance(declared_length, str)
+ else type(declared_length).__name__
+ )
+ logger.warning(
+ f"{label} partition {idx}: manifest "
+ f"content_length is not an integer ({shape}); "
+ "skipping the size check for this partition"
+ )
+
+ # Transport-corruption check when the manifest declares a
+ # digest: bytes that hash differently than meta.md5 were
+ # altered or corrupted in transit — refuse before writing
+ # them to disk. Like the content_length check above, this
+ # is not an authentication check: the digest comes from the
+ # same manifest fetch as the bytes it certifies, so a party
+ # able to substitute the partition can substitute the
+ # declared md5 too. A declared value that isn't a 32-hex
+ # digest (plain or ETag-quoted) is logged and skipped
+ # rather than treated as a mismatch.
+ declared_md5 = meta.get("md5")
+ if declared_md5 is not None:
+ normalized_md5 = _normalize_md5(declared_md5)
+ if normalized_md5 is None:
+ shape = (
+ f"{len(declared_md5)}-character string"
+ if isinstance(declared_md5, str)
+ else type(declared_md5).__name__
+ )
+ logger.warning(
+ f"{label} partition {idx}: manifest md5 is not "
+ f"a 32-character hex digest ({shape}); skipping "
+ "the integrity check for this partition"
+ )
+ else:
+ actual_md5 = hashlib.md5(
+ resp.content, usedforsecurity=False
+ ).hexdigest()
+ if actual_md5 != normalized_md5:
+ raise ValueError(
+ f"{label} partition {idx}: md5 mismatch — "
+ f"manifest declares {normalized_md5!r} but "
+ f"received {actual_md5!r}; refusing possibly "
+ "corrupted partition"
+ )
+
tmp_part.write_bytes(resp.content)
# Unpin the compressed body; the suspended frame would
# otherwise hold it until the caller drains the partition.
diff --git a/tests/security/test_journal_partition_md5_verification.py b/tests/security/test_journal_partition_md5_verification.py
new file mode 100644
index 0000000000..0610066edc
--- /dev/null
+++ b/tests/security/test_journal_partition_md5_verification.py
@@ -0,0 +1,313 @@
+"""Transport-corruption verification for OpenAlex snapshot partitions.
+
+The journal-quality dataset fetchers already defend a compromised
+manifest with a URL prefix allowlist (pinned separately). What they did
+not check is whether the bytes actually received match what the
+manifest itself describes: OpenAlex manifest entries can carry
+``meta.content_length`` (present on every entry in the live snapshot)
+and ``meta.md5`` (absent from every live entry today, but part of the
+documented manifest format) per partition, and the fetcher did not
+compare either to the downloaded bytes.
+
+This is a transport-corruption / truncation guard, not an
+authentication control, and the tests below verify it as one: the
+reference length and digest are fetched from the same manifest, over
+the same channel, as the bytes they describe, so a party able to
+substitute the partition can substitute those reference values too.
+What this catches is a bit-flipped or truncated transfer — a real and
+far more likely failure mode for a ~10 MB streamed download than a
+hostile origin.
+
+These tests pin: a declared ``content_length`` or ``md5`` that doesn't
+match the bytes received aborts the fetch (the previous snapshot is
+kept, nothing partial is persisted); a matching declaration, or no
+declaration at all, streams unchanged; and a declared ``md5`` that
+isn't a plain or ETag-quoted 32-hex digest is logged and skipped
+rather than compared or treated as fatal.
+"""
+
+import gzip
+import hashlib
+import io
+from unittest.mock import MagicMock
+
+import pytest
+
+from local_deep_research.journal_quality.data_sources._openalex_common import (
+ iter_partitions,
+)
+
+_ENTRY = "s3://openalex/data/jsonl/sources/part_0.gz"
+
+
+def _response(content: bytes) -> MagicMock:
+ resp = MagicMock()
+ resp.raise_for_status = MagicMock()
+ resp.content = content
+ return resp
+
+
+def _gz(payload: bytes) -> bytes:
+ buf = io.BytesIO()
+ with gzip.GzipFile(fileobj=buf, mode="wb") as gz:
+ gz.write(payload)
+ return buf.getvalue()
+
+
+def _drain(entry, tmp_path, body):
+ safe_get = MagicMock(return_value=_response(body))
+ out = []
+ for _idx, _total, records in iter_partitions(
+ [entry],
+ tmp_path,
+ file_prefix="journal",
+ label="test",
+ safe_get=safe_get,
+ ):
+ out.extend(records)
+ return out
+
+
+def _md5(data: bytes) -> str:
+ return hashlib.md5(data, usedforsecurity=False).hexdigest()
+
+
+def _big_records(n: int) -> list:
+ """``n`` distinct JSON lines with varied (not merely repeated) content.
+
+ Varied content keeps gzip from crushing the body down to a few
+ dozen bytes. The compressed body must stay above 1 KiB so a
+ truncated or partial-hash read is distinguishable from the real
+ thing — a tiny fixture would let a "hash/compare only the first N
+ bytes" bug pass every test silently.
+ """
+ return [
+ b'{"id": "S%d", "h": %d, "issn": "%04d-%04d"}'
+ % (i, (i * 37) % 9973, (i * 101) % 10000, (i * 233) % 10000)
+ for i in range(n)
+ ]
+
+
+_BIG_PAYLOAD = b"\n".join(_big_records(150)) + b"\n"
+_BODY = _gz(_BIG_PAYLOAD)
+assert len(_BODY) > 1024, (
+ "fixture body must compress to more than 1 KiB so truncation is "
+ f"distinguishable from the full body; got {len(_BODY)} bytes"
+)
+_IDS = [f"S{i}" for i in range(150)]
+
+
+def test_partition_failing_declared_md5_is_refused(tmp_path):
+ """Content hashing differently than the manifest says must abort.
+
+ Asserts the exact message, not a coarse ``match="md5"`` — a mutant
+ that raises *a* ValueError for the wrong reason (e.g. hashing a
+ truncated slice of the body, or reporting the wrong pair of
+ digests) must not be able to slip past by coincidentally
+ mentioning "md5" somewhere. It does not by itself prove the
+ comparison checks the full digest rather than a prefix — see
+ ``test_partition_failing_declared_md5_with_shared_prefix_is_refused``
+ below for that.
+ """
+ wrong_md5 = _md5(b"totally different bytes")
+ entry = {"url": _ENTRY, "meta": {"md5": wrong_md5}}
+ actual_md5 = _md5(_BODY)
+
+ with pytest.raises(ValueError) as exc_info:
+ _drain(entry, tmp_path, _BODY)
+
+ assert str(exc_info.value) == (
+ f"test partition 0: md5 mismatch — manifest declares "
+ f"{wrong_md5!r} but received {actual_md5!r}; refusing possibly "
+ "corrupted partition"
+ )
+
+
+def test_partition_failing_declared_md5_with_shared_prefix_is_refused(
+ tmp_path,
+):
+ """A wrong digest sharing the real one's leading hex characters
+ must still be refused.
+
+ Pins full-digest comparison against a mutant that compares only a
+ prefix (e.g. ``actual_md5[:8] != normalized_md5[:8]``): built to
+ differ from the real digest only in its last character, this value
+ would look like a match to any such prefix-only comparison and be
+ wrongly accepted instead of refused.
+ """
+ actual_md5 = _md5(_BODY)
+ flipped_last = "0" if actual_md5[-1] != "0" else "1"
+ wrong_md5 = actual_md5[:-1] + flipped_last
+ assert wrong_md5 != actual_md5
+ assert wrong_md5[:8] == actual_md5[:8]
+ entry = {"url": _ENTRY, "meta": {"md5": wrong_md5}}
+
+ with pytest.raises(ValueError) as exc_info:
+ _drain(entry, tmp_path, _BODY)
+
+ assert str(exc_info.value) == (
+ f"test partition 0: md5 mismatch — manifest declares "
+ f"{wrong_md5!r} but received {actual_md5!r}; refusing possibly "
+ "corrupted partition"
+ )
+
+
+def test_partition_matching_declared_md5_streams(tmp_path):
+ """Content matching the declaration streams normally.
+
+ Uses the >1 KiB ``_BODY`` fixture: a mutant that hashes only
+ ``resp.content[:1024]`` would compute a different digest than the
+ one declared here (over the full body) and wrongly raise.
+ """
+ entry = {"url": _ENTRY, "meta": {"md5": _md5(_BODY)}}
+
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+
+
+def test_manifest_without_md5_streams_unchanged(tmp_path):
+ """No declaration, no verification — current behaviour preserved."""
+ records = _drain({"url": _ENTRY}, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+
+
+def test_partition_quoted_uppercase_declared_md5_streams(
+ tmp_path, loguru_caplog_full
+):
+ """A quoted, uppercase ETag-style digest is accepted and matched.
+
+ ``meta.md5`` isn't populated on any live OpenAlex manifest today,
+ but AWS's own ETag wire form wraps a non-multipart object's digest
+ in double quotes; accept that shape case-insensitively rather than
+ treating it as malformed the day upstream starts declaring one.
+
+ Asserts the malformed-shape warning is absent, not just that the
+ records come through unchanged — the malformed-skip path streams
+ every record too, so ``== _IDS`` alone can't distinguish "digest
+ verified" from "quote-stripping broke and this was silently
+ skipped instead."
+ """
+ quoted = f'"{_md5(_BODY).upper()}"'
+ entry = {"url": _ENTRY, "meta": {"md5": quoted}}
+
+ with loguru_caplog_full.at_level("WARNING"):
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+ assert "not a 32-character hex digest" not in loguru_caplog_full.text
+
+
+def test_partition_malformed_declared_md5_is_skipped_with_warning(
+ tmp_path, loguru_caplog_full
+):
+ """A multipart ETag (``<hex>-<n>``) isn't a 32-hex digest.
+
+ It must not be compared byte-for-byte against a plain md5 (it
+ would never match, hard-aborting every fetch the day upstream
+ declares this shape) and must not be silently ignored either — log
+ a warning identifying the partition and the malformed shape, then
+ continue downloading as if no digest were declared.
+ """
+ malformed = f"{_md5(_BODY)}-3"
+ entry = {"url": _ENTRY, "meta": {"md5": malformed}}
+
+ with loguru_caplog_full.at_level("WARNING"):
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+ assert (
+ "test partition 0: manifest md5 is not a 32-character hex "
+ "digest (34-character string); skipping the integrity check "
+ "for this partition"
+ ) in loguru_caplog_full.text
+ # The malformed value itself must never be logged, only its shape.
+ assert malformed not in loguru_caplog_full.text
+
+
+def test_partition_non_string_declared_md5_is_skipped_with_warning(
+ tmp_path, loguru_caplog_full
+):
+ """A non-string ``meta.md5`` (e.g. a JSON number) is skipped, not fatal.
+
+ The warning names the value's type, never the value — the same
+ code path has to handle whatever shape a malformed manifest sends.
+ """
+ entry = {"url": _ENTRY, "meta": {"md5": 123456}}
+
+ with loguru_caplog_full.at_level("WARNING"):
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+ assert (
+ "test partition 0: manifest md5 is not a 32-character hex "
+ "digest (int); skipping the integrity check for this partition"
+ ) in loguru_caplog_full.text
+
+
+ "delta",
+ [-100, 100],
+ ids=[
+ "declared-shorter-than-received",
+ "declared-longer-than-received",
+ ],
+)
+def test_partition_content_length_mismatch_is_refused(tmp_path, delta):
+ """A declared content_length that doesn't match the bytes received
+ must abort in both directions, checked (and raised) before the
+ costlier md5 comparison runs.
+
+ A shorter declaration (``delta`` negative, the only direction
+ previously tested here) catches a corrupted transfer padded with
+ extra bytes; a longer declaration (``delta`` positive) catches the
+ opposite and more likely failure — a truncated download that fell
+ short of what the manifest promised.
+ """
+ wrong_length = len(_BODY) + delta
+ entry = {"url": _ENTRY, "meta": {"content_length": wrong_length}}
+
+ with pytest.raises(ValueError) as exc_info:
+ _drain(entry, tmp_path, _BODY)
+
+ assert str(exc_info.value) == (
+ f"test partition 0: content_length mismatch — manifest "
+ f"declares {wrong_length} bytes but received {len(_BODY)} "
+ "bytes; refusing possibly corrupted or truncated partition"
+ )
+
+
+def test_partition_content_length_match_streams(tmp_path):
+ """A correctly declared content_length streams normally."""
+ entry = {"url": _ENTRY, "meta": {"content_length": len(_BODY)}}
+
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+
+
+def test_partition_malformed_content_length_is_skipped_with_warning(
+ tmp_path, loguru_caplog_full
+):
+ """A string content_length isn't silently ignored.
+
+ Even one that spells the correct length right, only as the wrong
+ type, must still warn and continue downloading — mirroring how a
+ malformed ``meta.md5`` is handled — rather than disabling the size
+ check in total silence.
+ """
+ declared = str(len(_BODY))
+ entry = {"url": _ENTRY, "meta": {"content_length": declared}}
+
+ with loguru_caplog_full.at_level("WARNING"):
+ records = _drain(entry, tmp_path, _BODY)
+
+ assert [r["id"] for r in records] == _IDS
+ assert (
+ f"test partition 0: manifest content_length is not an integer "
+ f"({len(declared)}-character string); skipping the size check "
+ "for this partition"
+ ) in loguru_caplog_full.text
+ # The malformed value itself must never be logged, only its shape.
+ assert declared not in loguru_caplog_full.text