Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(security): coerce hostile-typed dataset fields at ingestion (#6446)

LearningCircuit40a114e5db79
diff --git a/changelog.d/6446.security.md b/changelog.d/6446.security.md
new file mode 100644
index 0000000000..6ddc00c2d9
--- /dev/null
+++ b/changelog.d/6446.security.md
@@ -0,0 +1 @@
+The journal-data build now coerces hostile-typed dataset fields at ingestion. The source and institution populators previously trusted the downloaded OpenAlex snapshot's field types — a hostile-typed value (`h_index: "not-a-number"`, an integer publisher, a list type tag, a non-string name) crashed the build midway at the first comparison or string operation, and whatever survived persisted into `INTEGER` columns via SQLite's dynamic typing, later breaking dashboard reads. Junk numerics now land as `NULL`, non-string text fields are dropped (non-string names skip the record), and well-typed fields flow unchanged — stacking on the manifest URL allowlist, with md5 verification, decompression caps, and the HTTPS pin still pending in unmerged sibling PRs.
diff --git a/src/local_deep_research/journal_quality/db.py b/src/local_deep_research/journal_quality/db.py
index c285fef2e3..8eb3f0299e 100644
--- a/src/local_deep_research/journal_quality/db.py
+++ b/src/local_deep_research/journal_quality/db.py
@@ -25,6 +25,7 @@ from __future__ import annotations
import gzip
import json
+import math
import os
import secrets
import sqlite3
@@ -1381,6 +1382,68 @@ def _populate_predatory(session: Session, pred: dict) -> None:
)
+def _as_number(value):
+ """Coerce a dataset-supplied numeric field; junk types become None.
+
+ The datasets come from a public upstream over HTTPS — the same
+ hostile-upstream threat model as the manifest URL allowlist (and,
+ once merged, the pending md5 verification, decompression caps
+ and https-pin PRs). Junk-typed numerics must land as NULL, not
+ crash the build's comparisons or the dashboard reads. That
+ includes values SQLite cannot round-trip or store: floats that
+ are inf/NaN (``json.loads('1e999')`` yields inf) and ints
+ outside the int64 range an INTEGER column can hold.
+ """
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
+ return None
+ if isinstance(value, int):
+ if not -(2**63) <= value <= 2**63 - 1:
+ return None
+ elif not math.isfinite(value):
+ return None
+ return value
+
+
+def _as_text(value):
+ """Coerce a dataset-supplied text field; non-strings become None."""
+ if isinstance(value, str):
+ stripped = value.strip()
+ if stripped:
+ return stripped
+ return None
+
+
+# Row-count floors for the two populators below. They mirror the
+# fetch-stage floors that already guard the raw snapshot
+# (``_MIN_OPENALEX_SOURCES = 10_000`` in
+# ``data_sources/openalex.py``, ``_MIN_INSTITUTIONS = 50_000`` in
+# ``data_sources/institutions.py``) — but those floors count records
+# reaching the snapshot, not how many of those this module's coercion
+# then drops for a blank/non-string name. A fetch that clears its own
+# floor can still collapse here: e.g. every fetched record sharing a
+# well-typed but empty ``display_name`` passes the fetch floor and
+# then loses 100% of rows to the ``if not name: continue`` skip below,
+# silently producing a near-empty (or fully empty) table with no
+# error. Set to half the corresponding fetch floor: normal attrition
+# from a healthy OpenAlex snapshot is a small fraction of blank names,
+# so losing more than half of an otherwise fetch-floor-sized snapshot
+# to name/type coercion is itself evidence of a collapsed or hostile
+# feed, not ordinary data quality noise.
+#
+# The check only applies once the input is already at fetch-floor
+# scale (``len(sources) >= _MIN_BUILD_SOURCES``, same pattern for
+# institutions) — mirroring the fetch stage's own schema-drift check
+# (``openalex.py``'s ``if len(journal_sample) >= _SCHEMA_SAMPLE_SIZE``
+# guard), which likewise only fires once the sample is large enough
+# to be meaningful. A hand-built test fixture of a handful of records
+# could never pass the fetch-stage floor in the first place, so it
+# never reaches build-floor scale either and the check does not fire
+# for it; only a real (or realistically-sized) snapshot that
+# collapses during populate does.
+_MIN_BUILD_SOURCES = 5_000 # half of openalex.py's _MIN_OPENALEX_SOURCES
+_MIN_BUILD_INSTITUTIONS = 25_000 # half of institutions.py's _MIN_INSTITUTIONS
+
+
def _populate_sources(
session: Session,
sources: dict,
@@ -1397,19 +1460,22 @@ def _populate_sources(
# print and electronic ISSNs in OpenAlex both survive instead of
# collapsing onto one row.
seen: dict[tuple[str, str], dict] = {}
+ skipped = 0
for source_id, compact in sources.items():
- name = (compact.get("n") or "").strip()
+ name = _as_text(compact.get("n"))
if not name:
+ skipped += 1
continue
name_lower = normalize_name(name)
- issn = normalize_issn(compact.get("i"))
- publisher = compact.get("p") or None
- h_index = compact.get("h")
- impact_factor = compact.get("if")
- cited_by_count = compact.get("cb")
- source_type = type_map.get(compact.get("t", ""), compact.get("t", ""))
+ issn = normalize_issn(_as_text(compact.get("i")))
+ publisher = _as_text(compact.get("p"))
+ h_index = _as_number(compact.get("h"))
+ impact_factor = _as_number(compact.get("if"))
+ cited_by_count = _as_number(compact.get("cb"))
+ type_tag = _as_text(compact.get("t")) or ""
+ source_type = type_map.get(type_tag, type_tag)
doaj_entry = doaj_data.get(issn) if issn else None
is_in_doaj = doaj_entry is not None
@@ -1462,6 +1528,19 @@ def _populate_sources(
if prev is None or (h_index or 0) > (prev.get("h_index") or 0):
seen[key] = rec
+ if skipped:
+ logger.warning(
+ f"OpenAlex sources: skipped {skipped:,} of {len(sources):,} "
+ "records with a blank or non-string name"
+ )
+ kept = len(sources) - skipped
+ if len(sources) >= _MIN_BUILD_SOURCES and kept < _MIN_BUILD_SOURCES:
+ raise RuntimeError(
+ f"OpenAlex sources: suspiciously few records survived "
+ f"ingestion ({kept:,} < {_MIN_BUILD_SOURCES:,} of "
+ f"{len(sources):,} fetched); refusing to overwrite existing data"
+ )
+
# Second pass: DOAJ-only journals (not in OpenAlex). Without this
# we lose ~4-7K small open-access venues. Keyed by name_lower so
# we don't double-insert anything OpenAlex already covered.
@@ -1555,24 +1634,42 @@ def _populate_sources(
def _populate_institutions(session: Session, institutions: dict) -> None:
records: list[dict] = []
+ skipped = 0
for inst_id, compact in institutions.items():
- name = (compact.get("n") or "").strip()
+ name = _as_text(compact.get("n"))
if not name:
+ skipped += 1
continue
records.append(
{
"openalex_id": inst_id,
"name": name,
"name_lower": normalize_name(name),
- "ror_id": compact.get("r"),
- "country": compact.get("c"),
- "type": compact.get("t"),
- "h_index": compact.get("h"),
- "impact_factor": compact.get("if"),
- "works_count": compact.get("w"),
- "cited_by_count": compact.get("cb"),
+ "ror_id": _as_text(compact.get("r")),
+ "country": _as_text(compact.get("c")),
+ "type": _as_text(compact.get("t")),
+ "h_index": _as_number(compact.get("h")),
+ "impact_factor": _as_number(compact.get("if")),
+ "works_count": _as_number(compact.get("w")),
+ "cited_by_count": _as_number(compact.get("cb")),
}
)
+ if skipped:
+ logger.warning(
+ f"OpenAlex institutions: skipped {skipped:,} of "
+ f"{len(institutions):,} records with a blank or non-string name"
+ )
+ kept = len(records)
+ if (
+ len(institutions) >= _MIN_BUILD_INSTITUTIONS
+ and kept < _MIN_BUILD_INSTITUTIONS
+ ):
+ raise RuntimeError(
+ f"OpenAlex institutions: suspiciously few records survived "
+ f"ingestion ({kept:,} < {_MIN_BUILD_INSTITUTIONS:,} of "
+ f"{len(institutions):,} fetched); refusing to overwrite "
+ "existing data"
+ )
logger.info(f"Inserting {len(records)} institution records...")
for i in range(0, len(records), _BATCH_SIZE):
session.bulk_insert_mappings(
diff --git a/tests/security/test_journal_hostile_typed_fields.py b/tests/security/test_journal_hostile_typed_fields.py
new file mode 100644
index 0000000000..08592b6eb6
--- /dev/null
+++ b/tests/security/test_journal_hostile_typed_fields.py
@@ -0,0 +1,275 @@
+"""Hostile-typed dataset fields must not crash the journal-data build.
+
+``_populate_sources`` trusts the compact record fields parsed from the
+downloaded OpenAlex snapshot: numeric fields are compared
+(``(h_index or 0) > threshold``, the dedup tiebreak, the
+cited-by-count quartile sort, ``derive_quality_score``'s internal
+ladders) and text fields get string operations (``.strip()``,
+``normalize_name``, dict-key lookups on the type tag) — all without
+type checks. SQLite's dynamic typing happily persists whatever arrives,
+so a hostile field both crashes the build midway and, when it survives
+to insertion, breaks the dashboard reads (``round(impact_factor, 2)``).
+
+The existing edge-case suite covers *numeric* extremes (negative
+h-index) — nothing covers type attacks. The dataset upstream is a
+public S3 bucket over HTTPS (same hostile-upstream threat model as
+the manifest URL allowlist and, once merged, the pending
+md5-verification, decompression-cap, and https-pin PRs), so
+junk-typed fields are attacker-reachable.
+
+These tests pin ingestion-time coercion: numeric junk becomes NULL,
+non-string names/publishers/types are dropped or defaulted, and the
+build completes; well-typed fields keep flowing unchanged.
+"""
+
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import sessionmaker
+
+from local_deep_research.journal_quality.db import (
+ _populate_institutions,
+ _populate_sources,
+)
+from local_deep_research.journal_quality.models import (
+ Institution,
+ JournalQualityBase,
+ Source,
+)
+
+_EMPTY_PRED = {
+ "journals": set(),
+ "publishers": set(),
+ "hijacked": set(),
+ "long_pubs": [],
+}
+
+
+def _populate(sources: dict) -> dict[str, Source]:
+ """Run _populate_sources in memory; return rows keyed by name."""
+ engine = create_engine("sqlite:///:memory:")
+ try:
+ JournalQualityBase.metadata.create_all(engine)
+ with sessionmaker(bind=engine)() as session:
+ _populate_sources(session, sources, {}, _EMPTY_PRED)
+ session.commit()
+ with sessionmaker(bind=engine)() as reader:
+ return {row.name: row for row in reader.scalars(select(Source))}
+ finally:
+ engine.dispose()
+
+
+def test_hostile_typed_fields_do_not_crash_the_build():
+ """One hostile record must not abort the whole dataset build.
+
+ Given a snapshot whose numeric fields are strings/containers, whose
+ publisher is an int, whose type tag is a list, and whose second
+ record's *name* is not a string, When populated, Then the build
+ completes: junk numerics land as NULL, the int publisher is
+ dropped, the list type tag coerces to an empty ``source_type``
+ (``_as_text(["journal"])`` returns ``None``, ``None or ""`` is
+ ``""``, and ``type_map.get("", "")`` is ``""`` -- not left as the
+ raw list), and the non-string-named record is skipped.
+ """
+ rows = _populate(
+ {
+ "S1": {
+ "n": "Alpha Journal",
+ "i": "1234-5678",
+ "h": "not-a-number",
+ "if": {"evil": 1},
+ "cb": [1, 2, 3],
+ "p": 42,
+ "t": ["journal"],
+ },
+ "S2": {"n": 12345, "h": 10},
+ }
+ )
+
+ alpha = rows["Alpha Journal"]
+ assert alpha.h_index is None, "junk h_index must land as NULL"
+ assert alpha.impact_factor is None
+ assert alpha.cited_by_count is None
+ assert alpha.publisher is None, "non-string publisher must be dropped"
+ assert "12345" not in rows, "non-string-named record must be skipped"
+ assert len(rows) == 1
+
+
+def test_numeric_fields_still_flow_unchanged():
+ """Control: well-typed fields keep flowing exactly as before."""
+ rows = _populate(
+ {
+ "S1": {"n": "Beta Journal", "h": 52, "if": 12.5, "cb": 100},
+ "S2": {"n": "Gamma Journal", "h": 30, "cb": 50},
+ }
+ )
+
+ beta = rows["Beta Journal"]
+ assert beta.h_index == 52
+ assert beta.impact_factor == 12.5
+ assert beta.cited_by_count == 100
+ # Two cited records of the same type: quartiles must be derived
+ # (with n=2 the top rank lands at pct 0.5 -> Q2, the bottom -> Q4),
+ # proving the cited-by sort still runs. Asserted per row, not as
+ # an unordered set: {Beta=Q2, Gamma=Q4} and the reversed
+ # {Beta=Q4, Gamma=Q2} both produce the same set, so a set
+ # comparison can't catch a sort-direction regression.
+ assert rows["Beta Journal"].quartile == "Q2", (
+ "higher cited_by_count (100) should rank in the top half -> Q2"
+ )
+ assert rows["Gamma Journal"].quartile == "Q4", (
+ "lower cited_by_count (50) should rank in the bottom half -> Q4"
+ )
+
+
+def test_hostile_hindex_dedup_tiebreak_survives():
+ """A junk-typed h_index must not raise, including at the dedup
+ tiebreak for a same-name duplicate.
+
+ Pre-PR the crash is *not* in the tiebreak comparison itself. S1 is
+ the first record under this dedup key, so ``prev = seen.get(key)``
+ (db.py:1527) is ``None`` and ``if prev is None or (h_index or 0) >
+ (prev.get("h_index") or 0)`` (db.py:1528) short-circuits on
+ ``prev is None`` before its right-hand side ever runs. The crash
+ instead lands earlier, in ``derive_quality_score(h_index="junk",
+ ...)`` (called at db.py:1501) — specifically its ``if h_index and
+ h_index > 0`` (scoring.py:133), where ``"junk" > 0`` raises
+ ``TypeError``. Once that earlier crash is fixed, this test also
+ exercises the tiebreak proper: S2's well-typed h_index=7 must beat
+ S1's coerced-to-NULL h_index without raising when they *are*
+ compared.
+ """
+ rows = _populate(
+ {
+ "S1": {"n": "Delta Journal", "h": "junk"},
+ "S2": {"n": "Delta Journal", "h": 7},
+ }
+ )
+
+ delta = rows["Delta Journal"]
+ assert delta is not None
+ assert delta.h_index == 7, "the well-typed duplicate must win the dedup"
+
+
+def test_unrepresentable_numbers_become_null():
+ """Non-finite floats and out-of-int64 ints must land as NULL.
+
+ JSON parsing yields inf for ``1e999`` and accepts a literal
+ ``NaN`` by default: inf survives insertion but ``round(inf, 2)``
+ later emits an invalid ``Infinity`` token on dashboard reads, NaN
+ binds as NULL only by SQLite luck while silently stealing a
+ quartile rank, and an int beyond int64 raises OverflowError at
+ INSERT time. All three value classes must coerce to NULL instead
+ (the hostile ``2yr_mean_citedness`` arrives in the ``if`` slot).
+ """
+ rows = _populate(
+ {
+ "S1": {
+ "n": "Omega Journal",
+ "h": 10**30,
+ "if": 1e999,
+ "cb": float("nan"),
+ }
+ }
+ )
+
+ omega = rows["Omega Journal"]
+ assert omega.h_index is None, "out-of-int64 int must land as NULL"
+ assert omega.impact_factor is None, "inf must land as NULL"
+ assert omega.cited_by_count is None, "NaN must land as NULL"
+ assert omega.quartile is None, (
+ "NULL cited-by must sit out the quartile sort"
+ )
+
+
+def test_institutions_hostile_typed_fields_do_not_crash():
+ """The institutions populator shares the crash shape — pin it too."""
+ engine = create_engine("sqlite:///:memory:")
+ try:
+ JournalQualityBase.metadata.create_all(engine)
+ with sessionmaker(bind=engine)() as session:
+ _populate_institutions(
+ session,
+ {
+ "I1": {
+ "n": "Evil Institute",
+ "r": 1,
+ "c": 2,
+ "t": 3,
+ "h": "x",
+ "if": [],
+ "w": {},
+ "cb": "z",
+ },
+ "I2": {"n": 99},
+ },
+ )
+ session.commit()
+ with sessionmaker(bind=engine)() as reader:
+ rows = {
+ row.name: row for row in reader.scalars(select(Institution))
+ }
+ finally:
+ engine.dispose()
+
+ inst = rows["Evil Institute"]
+ assert inst.ror_id is None
+ assert inst.country is None
+ assert inst.type is None
+ assert inst.h_index is None
+ assert inst.impact_factor is None
+ assert inst.works_count is None
+ assert inst.cited_by_count is None
+ assert len(rows) == 1, "non-string-named institution must be skipped"
+
+
+def test_hostile_bool_hindex_becomes_null_not_one():
+ """``h: True`` must null out, not silently become the integer 1.
+
+ Guards the ``isinstance(value, bool)`` rejection in ``_as_number``
+ (db.py:1397): ``bool`` is a subclass of ``int`` in Python, so
+ without that explicit check ``True`` passes the numeric-type test,
+ passes the int64-bounds check (``True == 1``), and is returned
+ unchanged by ``_as_number`` — landing in the ``h_index`` column as
+ a 1 instead of NULL. That is exactly the silent type confusion the
+ numeric coercion exists to remove, and ``True == 1`` means a naive
+ "is it a number" check cannot catch it.
+ """
+ rows = _populate({"S1": {"n": "Bool H-Index Journal", "h": True}})
+ row = rows["Bool H-Index Journal"]
+ assert row.h_index is None, "a bool h_index must null out, not become 1"
+
+
+def test_hostile_padded_name_is_stored_stripped():
+ """Leading/trailing whitespace in a name must not reach the row.
+
+ Guards the ``.strip()`` call in ``_as_text`` (db.py:1410):
+ dropping just that call (while keeping the surrounding
+ ``isinstance``/truthiness checks) still lets a whitespace-padded
+ but non-empty name through unstripped, so the stored ``name``
+ would keep its padding instead of matching the trimmed value the
+ rest of the pipeline — and every other name in the table —
+ expects.
+ """
+ rows = _populate({"S1": {"n": " Padded Journal ", "h": 5}})
+ (row,) = rows.values()
+ assert row.name == "Padded Journal", "stored name must be stripped"
+
+
+def test_hostile_list_issn_becomes_null_not_a_crash():
+ """A list-typed ISSN must be nulled, not handed to normalize_issn() raw.
+
+ Guards the ``_as_text`` wrap around ``compact.get("i")``
+ (db.py:1472): ``normalize_issn``'s own guard is ``if not s: return
+ None`` (citation_normalizer.py:42), which only screens *falsy*
+ values — a non-empty list is truthy, so it falls through to
+ ``_ISSN_CHARS.sub("", s)``, and ``re.sub()`` against a list raises
+ ``TypeError: expected string or bytes-like object``, aborting the
+ whole build. The ``i`` field is defence-in-depth rather than an
+ exposed hole (the real fetcher only ever writes
+ ``normalize_issn``'s own ``Optional[str]`` output into this slot),
+ but a direct populator caller can still reach it.
+ """
+ rows = _populate(
+ {"S1": {"n": "List ISSN Journal", "i": ["1234-5678"], "h": 5}}
+ )
+ row = rows["List ISSN Journal"]
+ assert row.issn is None, "a list-typed ISSN must coerce to NULL"