~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(security): coerce hostile-typed dataset fields at ingestion (#6446)
LearningCircuit40a114e5db79
diff --git a/changelog.d/6446.security.md b/changelog.d/6446.security.mdnew file mode 100644index 0000000000..6ddc00c2d9--- /dev/null+++ b/changelog.d/6446.security.md@@ -0,0 +1 @@+The journal-data build now coerces hostile-typed dataset fields at ingestion. The source and institution populators previously trusted the downloaded OpenAlex snapshot's field types — a hostile-typed value (`h_index: "not-a-number"`, an integer publisher, a list type tag, a non-string name) crashed the build midway at the first comparison or string operation, and whatever survived persisted into `INTEGER` columns via SQLite's dynamic typing, later breaking dashboard reads. Junk numerics now land as `NULL`, non-string text fields are dropped (non-string names skip the record), and well-typed fields flow unchanged — stacking on the manifest URL allowlist, with md5 verification, decompression caps, and the HTTPS pin still pending in unmerged sibling PRs.diff --git a/src/local_deep_research/journal_quality/db.py b/src/local_deep_research/journal_quality/db.pyindex c285fef2e3..8eb3f0299e 100644--- a/src/local_deep_research/journal_quality/db.py+++ b/src/local_deep_research/journal_quality/db.py@@ -25,6 +25,7 @@ from __future__ import annotationsimport gzipimport json+import mathimport osimport secretsimport sqlite3@@ -1381,6 +1382,68 @@ def _populate_predatory(session: Session, pred: dict) -> None:)+def _as_number(value):+ """Coerce a dataset-supplied numeric field; junk types become None.++ The datasets come from a public upstream over HTTPS — the same+ hostile-upstream threat model as the manifest URL allowlist (and,+ once merged, the pending md5 verification, decompression caps+ and https-pin PRs). Junk-typed numerics must land as NULL, not+ crash the build's comparisons or the dashboard reads. That+ includes values SQLite cannot round-trip or store: floats that+ are inf/NaN (``json.loads('1e999')`` yields inf) and ints+ outside the int64 range an INTEGER column can hold.+ """+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ return None+ if isinstance(value, int):+ if not -(2**63) <= value <= 2**63 - 1:+ return None+ elif not math.isfinite(value):+ return None+ return value+++def _as_text(value):+ """Coerce a dataset-supplied text field; non-strings become None."""+ if isinstance(value, str):+ stripped = value.strip()+ if stripped:+ return stripped+ return None+++# Row-count floors for the two populators below. They mirror the+# fetch-stage floors that already guard the raw snapshot+# (``_MIN_OPENALEX_SOURCES = 10_000`` in+# ``data_sources/openalex.py``, ``_MIN_INSTITUTIONS = 50_000`` in+# ``data_sources/institutions.py``) — but those floors count records+# reaching the snapshot, not how many of those this module's coercion+# then drops for a blank/non-string name. A fetch that clears its own+# floor can still collapse here: e.g. every fetched record sharing a+# well-typed but empty ``display_name`` passes the fetch floor and+# then loses 100% of rows to the ``if not name: continue`` skip below,+# silently producing a near-empty (or fully empty) table with no+# error. Set to half the corresponding fetch floor: normal attrition+# from a healthy OpenAlex snapshot is a small fraction of blank names,+# so losing more than half of an otherwise fetch-floor-sized snapshot+# to name/type coercion is itself evidence of a collapsed or hostile+# feed, not ordinary data quality noise.+#+# The check only applies once the input is already at fetch-floor+# scale (``len(sources) >= _MIN_BUILD_SOURCES``, same pattern for+# institutions) — mirroring the fetch stage's own schema-drift check+# (``openalex.py``'s ``if len(journal_sample) >= _SCHEMA_SAMPLE_SIZE``+# guard), which likewise only fires once the sample is large enough+# to be meaningful. A hand-built test fixture of a handful of records+# could never pass the fetch-stage floor in the first place, so it+# never reaches build-floor scale either and the check does not fire+# for it; only a real (or realistically-sized) snapshot that+# collapses during populate does.+_MIN_BUILD_SOURCES = 5_000 # half of openalex.py's _MIN_OPENALEX_SOURCES+_MIN_BUILD_INSTITUTIONS = 25_000 # half of institutions.py's _MIN_INSTITUTIONS++def _populate_sources(session: Session,sources: dict,@@ -1397,19 +1460,22 @@ def _populate_sources(# print and electronic ISSNs in OpenAlex both survive instead of# collapsing onto one row.seen: dict[tuple[str, str], dict] = {}+ skipped = 0for source_id, compact in sources.items():- name = (compact.get("n") or "").strip()+ name = _as_text(compact.get("n"))if not name:+ skipped += 1continuename_lower = normalize_name(name)- issn = normalize_issn(compact.get("i"))- publisher = compact.get("p") or None- h_index = compact.get("h")- impact_factor = compact.get("if")- cited_by_count = compact.get("cb")- source_type = type_map.get(compact.get("t", ""), compact.get("t", ""))+ issn = normalize_issn(_as_text(compact.get("i")))+ publisher = _as_text(compact.get("p"))+ h_index = _as_number(compact.get("h"))+ impact_factor = _as_number(compact.get("if"))+ cited_by_count = _as_number(compact.get("cb"))+ type_tag = _as_text(compact.get("t")) or ""+ source_type = type_map.get(type_tag, type_tag)doaj_entry = doaj_data.get(issn) if issn else Noneis_in_doaj = doaj_entry is not None@@ -1462,6 +1528,19 @@ def _populate_sources(if prev is None or (h_index or 0) > (prev.get("h_index") or 0):seen[key] = rec+ if skipped:+ logger.warning(+ f"OpenAlex sources: skipped {skipped:,} of {len(sources):,} "+ "records with a blank or non-string name"+ )+ kept = len(sources) - skipped+ if len(sources) >= _MIN_BUILD_SOURCES and kept < _MIN_BUILD_SOURCES:+ raise RuntimeError(+ f"OpenAlex sources: suspiciously few records survived "+ f"ingestion ({kept:,} < {_MIN_BUILD_SOURCES:,} of "+ f"{len(sources):,} fetched); refusing to overwrite existing data"+ )+# Second pass: DOAJ-only journals (not in OpenAlex). Without this# we lose ~4-7K small open-access venues. Keyed by name_lower so# we don't double-insert anything OpenAlex already covered.@@ -1555,24 +1634,42 @@ def _populate_sources(def _populate_institutions(session: Session, institutions: dict) -> None:records: list[dict] = []+ skipped = 0for inst_id, compact in institutions.items():- name = (compact.get("n") or "").strip()+ name = _as_text(compact.get("n"))if not name:+ skipped += 1continuerecords.append({"openalex_id": inst_id,"name": name,"name_lower": normalize_name(name),- "ror_id": compact.get("r"),- "country": compact.get("c"),- "type": compact.get("t"),- "h_index": compact.get("h"),- "impact_factor": compact.get("if"),- "works_count": compact.get("w"),- "cited_by_count": compact.get("cb"),+ "ror_id": _as_text(compact.get("r")),+ "country": _as_text(compact.get("c")),+ "type": _as_text(compact.get("t")),+ "h_index": _as_number(compact.get("h")),+ "impact_factor": _as_number(compact.get("if")),+ "works_count": _as_number(compact.get("w")),+ "cited_by_count": _as_number(compact.get("cb")),})+ if skipped:+ logger.warning(+ f"OpenAlex institutions: skipped {skipped:,} of "+ f"{len(institutions):,} records with a blank or non-string name"+ )+ kept = len(records)+ if (+ len(institutions) >= _MIN_BUILD_INSTITUTIONS+ and kept < _MIN_BUILD_INSTITUTIONS+ ):+ raise RuntimeError(+ f"OpenAlex institutions: suspiciously few records survived "+ f"ingestion ({kept:,} < {_MIN_BUILD_INSTITUTIONS:,} of "+ f"{len(institutions):,} fetched); refusing to overwrite "+ "existing data"+ )logger.info(f"Inserting {len(records)} institution records...")for i in range(0, len(records), _BATCH_SIZE):session.bulk_insert_mappings(diff --git a/tests/security/test_journal_hostile_typed_fields.py b/tests/security/test_journal_hostile_typed_fields.pynew file mode 100644index 0000000000..08592b6eb6--- /dev/null+++ b/tests/security/test_journal_hostile_typed_fields.py@@ -0,0 +1,275 @@+"""Hostile-typed dataset fields must not crash the journal-data build.++``_populate_sources`` trusts the compact record fields parsed from the+downloaded OpenAlex snapshot: numeric fields are compared+(``(h_index or 0) > threshold``, the dedup tiebreak, the+cited-by-count quartile sort, ``derive_quality_score``'s internal+ladders) and text fields get string operations (``.strip()``,+``normalize_name``, dict-key lookups on the type tag) — all without+type checks. SQLite's dynamic typing happily persists whatever arrives,+so a hostile field both crashes the build midway and, when it survives+to insertion, breaks the dashboard reads (``round(impact_factor, 2)``).++The existing edge-case suite covers *numeric* extremes (negative+h-index) — nothing covers type attacks. The dataset upstream is a+public S3 bucket over HTTPS (same hostile-upstream threat model as+the manifest URL allowlist and, once merged, the pending+md5-verification, decompression-cap, and https-pin PRs), so+junk-typed fields are attacker-reachable.++These tests pin ingestion-time coercion: numeric junk becomes NULL,+non-string names/publishers/types are dropped or defaulted, and the+build completes; well-typed fields keep flowing unchanged.+"""++from sqlalchemy import create_engine, select+from sqlalchemy.orm import sessionmaker++from local_deep_research.journal_quality.db import (+ _populate_institutions,+ _populate_sources,+)+from local_deep_research.journal_quality.models import (+ Institution,+ JournalQualityBase,+ Source,+)++_EMPTY_PRED = {+ "journals": set(),+ "publishers": set(),+ "hijacked": set(),+ "long_pubs": [],+}+++def _populate(sources: dict) -> dict[str, Source]:+ """Run _populate_sources in memory; return rows keyed by name."""+ engine = create_engine("sqlite:///:memory:")+ try:+ JournalQualityBase.metadata.create_all(engine)+ with sessionmaker(bind=engine)() as session:+ _populate_sources(session, sources, {}, _EMPTY_PRED)+ session.commit()+ with sessionmaker(bind=engine)() as reader:+ return {row.name: row for row in reader.scalars(select(Source))}+ finally:+ engine.dispose()+++def test_hostile_typed_fields_do_not_crash_the_build():+ """One hostile record must not abort the whole dataset build.++ Given a snapshot whose numeric fields are strings/containers, whose+ publisher is an int, whose type tag is a list, and whose second+ record's *name* is not a string, When populated, Then the build+ completes: junk numerics land as NULL, the int publisher is+ dropped, the list type tag coerces to an empty ``source_type``+ (``_as_text(["journal"])`` returns ``None``, ``None or ""`` is+ ``""``, and ``type_map.get("", "")`` is ``""`` -- not left as the+ raw list), and the non-string-named record is skipped.+ """+ rows = _populate(+ {+ "S1": {+ "n": "Alpha Journal",+ "i": "1234-5678",+ "h": "not-a-number",+ "if": {"evil": 1},+ "cb": [1, 2, 3],+ "p": 42,+ "t": ["journal"],+ },+ "S2": {"n": 12345, "h": 10},+ }+ )++ alpha = rows["Alpha Journal"]+ assert alpha.h_index is None, "junk h_index must land as NULL"+ assert alpha.impact_factor is None+ assert alpha.cited_by_count is None+ assert alpha.publisher is None, "non-string publisher must be dropped"+ assert "12345" not in rows, "non-string-named record must be skipped"+ assert len(rows) == 1+++def test_numeric_fields_still_flow_unchanged():+ """Control: well-typed fields keep flowing exactly as before."""+ rows = _populate(+ {+ "S1": {"n": "Beta Journal", "h": 52, "if": 12.5, "cb": 100},+ "S2": {"n": "Gamma Journal", "h": 30, "cb": 50},+ }+ )++ beta = rows["Beta Journal"]+ assert beta.h_index == 52+ assert beta.impact_factor == 12.5+ assert beta.cited_by_count == 100+ # Two cited records of the same type: quartiles must be derived+ # (with n=2 the top rank lands at pct 0.5 -> Q2, the bottom -> Q4),+ # proving the cited-by sort still runs. Asserted per row, not as+ # an unordered set: {Beta=Q2, Gamma=Q4} and the reversed+ # {Beta=Q4, Gamma=Q2} both produce the same set, so a set+ # comparison can't catch a sort-direction regression.+ assert rows["Beta Journal"].quartile == "Q2", (+ "higher cited_by_count (100) should rank in the top half -> Q2"+ )+ assert rows["Gamma Journal"].quartile == "Q4", (+ "lower cited_by_count (50) should rank in the bottom half -> Q4"+ )+++def test_hostile_hindex_dedup_tiebreak_survives():+ """A junk-typed h_index must not raise, including at the dedup+ tiebreak for a same-name duplicate.++ Pre-PR the crash is *not* in the tiebreak comparison itself. S1 is+ the first record under this dedup key, so ``prev = seen.get(key)``+ (db.py:1527) is ``None`` and ``if prev is None or (h_index or 0) >+ (prev.get("h_index") or 0)`` (db.py:1528) short-circuits on+ ``prev is None`` before its right-hand side ever runs. The crash+ instead lands earlier, in ``derive_quality_score(h_index="junk",+ ...)`` (called at db.py:1501) — specifically its ``if h_index and+ h_index > 0`` (scoring.py:133), where ``"junk" > 0`` raises+ ``TypeError``. Once that earlier crash is fixed, this test also+ exercises the tiebreak proper: S2's well-typed h_index=7 must beat+ S1's coerced-to-NULL h_index without raising when they *are*+ compared.+ """+ rows = _populate(+ {+ "S1": {"n": "Delta Journal", "h": "junk"},+ "S2": {"n": "Delta Journal", "h": 7},+ }+ )++ delta = rows["Delta Journal"]+ assert delta is not None+ assert delta.h_index == 7, "the well-typed duplicate must win the dedup"+++def test_unrepresentable_numbers_become_null():+ """Non-finite floats and out-of-int64 ints must land as NULL.++ JSON parsing yields inf for ``1e999`` and accepts a literal+ ``NaN`` by default: inf survives insertion but ``round(inf, 2)``+ later emits an invalid ``Infinity`` token on dashboard reads, NaN+ binds as NULL only by SQLite luck while silently stealing a+ quartile rank, and an int beyond int64 raises OverflowError at+ INSERT time. All three value classes must coerce to NULL instead+ (the hostile ``2yr_mean_citedness`` arrives in the ``if`` slot).+ """+ rows = _populate(+ {+ "S1": {+ "n": "Omega Journal",+ "h": 10**30,+ "if": 1e999,+ "cb": float("nan"),+ }+ }+ )++ omega = rows["Omega Journal"]+ assert omega.h_index is None, "out-of-int64 int must land as NULL"+ assert omega.impact_factor is None, "inf must land as NULL"+ assert omega.cited_by_count is None, "NaN must land as NULL"+ assert omega.quartile is None, (+ "NULL cited-by must sit out the quartile sort"+ )+++def test_institutions_hostile_typed_fields_do_not_crash():+ """The institutions populator shares the crash shape — pin it too."""+ engine = create_engine("sqlite:///:memory:")+ try:+ JournalQualityBase.metadata.create_all(engine)+ with sessionmaker(bind=engine)() as session:+ _populate_institutions(+ session,+ {+ "I1": {+ "n": "Evil Institute",+ "r": 1,+ "c": 2,+ "t": 3,+ "h": "x",+ "if": [],+ "w": {},+ "cb": "z",+ },+ "I2": {"n": 99},+ },+ )+ session.commit()+ with sessionmaker(bind=engine)() as reader:+ rows = {+ row.name: row for row in reader.scalars(select(Institution))+ }+ finally:+ engine.dispose()++ inst = rows["Evil Institute"]+ assert inst.ror_id is None+ assert inst.country is None+ assert inst.type is None+ assert inst.h_index is None+ assert inst.impact_factor is None+ assert inst.works_count is None+ assert inst.cited_by_count is None+ assert len(rows) == 1, "non-string-named institution must be skipped"+++def test_hostile_bool_hindex_becomes_null_not_one():+ """``h: True`` must null out, not silently become the integer 1.++ Guards the ``isinstance(value, bool)`` rejection in ``_as_number``+ (db.py:1397): ``bool`` is a subclass of ``int`` in Python, so+ without that explicit check ``True`` passes the numeric-type test,+ passes the int64-bounds check (``True == 1``), and is returned+ unchanged by ``_as_number`` — landing in the ``h_index`` column as+ a 1 instead of NULL. That is exactly the silent type confusion the+ numeric coercion exists to remove, and ``True == 1`` means a naive+ "is it a number" check cannot catch it.+ """+ rows = _populate({"S1": {"n": "Bool H-Index Journal", "h": True}})+ row = rows["Bool H-Index Journal"]+ assert row.h_index is None, "a bool h_index must null out, not become 1"+++def test_hostile_padded_name_is_stored_stripped():+ """Leading/trailing whitespace in a name must not reach the row.++ Guards the ``.strip()`` call in ``_as_text`` (db.py:1410):+ dropping just that call (while keeping the surrounding+ ``isinstance``/truthiness checks) still lets a whitespace-padded+ but non-empty name through unstripped, so the stored ``name``+ would keep its padding instead of matching the trimmed value the+ rest of the pipeline — and every other name in the table —+ expects.+ """+ rows = _populate({"S1": {"n": " Padded Journal ", "h": 5}})+ (row,) = rows.values()+ assert row.name == "Padded Journal", "stored name must be stripped"+++def test_hostile_list_issn_becomes_null_not_a_crash():+ """A list-typed ISSN must be nulled, not handed to normalize_issn() raw.++ Guards the ``_as_text`` wrap around ``compact.get("i")``+ (db.py:1472): ``normalize_issn``'s own guard is ``if not s: return+ None`` (citation_normalizer.py:42), which only screens *falsy*+ values — a non-empty list is truthy, so it falls through to+ ``_ISSN_CHARS.sub("", s)``, and ``re.sub()`` against a list raises+ ``TypeError: expected string or bytes-like object``, aborting the+ whole build. The ``i`` field is defence-in-depth rather than an+ exposed hole (the real fetcher only ever writes+ ``normalize_issn``'s own ``Optional[str]`` output into this slot),+ but a direct populator caller can still reach it.+ """+ rows = _populate(+ {"S1": {"n": "List ISSN Journal", "i": ["1234-5678"], "h": 5}}+ )+ row = rows["List ISSN Journal"]+ assert row.issn is None, "a list-typed ISSN must coerce to NULL"