~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for citation_normalizer — engine-specific dict → CSL-JSON metadata."""
import pytest
from local_deep_research.utilities.citation_normalizer import (
detect_engine,
normalize_citation,
normalize_issn,
_parse_authors_list,
_parse_name,
_extract_doi,
_with_scheme_for_arxiv_host,
_extract_arxiv_id,
)
class TestNormalizeIssn:
def test_dashed_form(self):
assert normalize_issn("1522-9645") == "15229645"
def test_undashed_form(self):
assert normalize_issn("15229645") == "15229645"
def test_round_trip_equality(self):
assert normalize_issn("1522-9645") == normalize_issn("15229645")
def test_uppercase_x_check_digit(self):
assert normalize_issn("1234-567x") == "1234567X"
def test_already_uppercase_x(self):
assert normalize_issn("1234-567X") == "1234567X"
def test_whitespace_stripped(self):
assert normalize_issn(" 1522-9645 ") == "15229645"
def test_empty_returns_none(self):
assert normalize_issn("") is None
def test_none_returns_none(self):
assert normalize_issn(None) is None
def test_too_short_returns_none(self):
assert normalize_issn("1234") is None
def test_too_long_returns_none(self):
assert normalize_issn("123456789") is None
def test_non_digit_returns_none(self):
assert normalize_issn("abcd-efgh") is None
class TestDetectEngine:
def test_arxiv_url(self):
assert (
detect_engine({"link": "https://arxiv.org/abs/2401.12345"})
== "arxiv"
)
def test_openalex_url(self):
assert (
detect_engine({"link": "https://openalex.org/W12345"}) == "openalex"
)
def test_pubmed_url(self):
assert (
detect_engine({"link": "https://pubmed.ncbi.nlm.nih.gov/12345"})
== "pubmed"
)
def test_explicit_source(self):
assert detect_engine({"source": "arXiv"}) == "arxiv"
def test_web_returns_none(self):
assert detect_engine({"link": "https://example.com/blog"}) is None
def test_empty_returns_none(self):
assert detect_engine({}) is None
class TestParseAuthors:
def test_string_list(self):
result = _parse_authors_list(["John Smith", "Jane Doe"])
assert len(result) == 2
assert result[0]["given"] == "John"
assert result[0]["family"] == "Smith"
def test_comma_string(self):
result = _parse_authors_list("John Smith, Jane Doe")
assert len(result) == 2
def test_dict_with_name(self):
result = _parse_authors_list([{"name": "John Smith"}])
assert result[0]["family"] == "Smith"
def test_already_csl(self):
csl = [{"family": "Smith", "given": "John"}]
result = _parse_authors_list(csl)
assert result == csl
def test_empty(self):
assert _parse_authors_list(None) is None
assert _parse_authors_list([]) is None
assert _parse_authors_list("") is None
def test_normalize_citation_prefers_authors_csl(self):
"""When both authors_csl (structured) and authors (display string)
are present, the structured list wins so 'Last, First' name pairs
survive the round-trip — the bare comma-joined string would split
into 4 singletons instead of 2 author pairs.
"""
from local_deep_research.utilities.citation_normalizer import (
normalize_citation,
)
source = {
"source_engine": "nasa_ads",
"title": "Test",
"authors": "Smith, John, Doe, Jane", # ambiguous joined form
"authors_csl": [
{"family": "Smith", "given": "John"},
{"family": "Doe", "given": "Jane"},
],
"year": 2023,
}
result = normalize_citation(source)
assert result is not None
assert len(result["authors"]) == 2
assert result["authors"][0] == {"family": "Smith", "given": "John"}
assert result["authors"][1] == {"family": "Doe", "given": "Jane"}
class TestParseName:
def test_first_last(self):
assert _parse_name("John Smith") == {
"given": "John",
"family": "Smith",
}
def test_last_comma_first(self):
assert _parse_name("Smith, John") == {
"family": "Smith",
"given": "John",
}
def test_single_name(self):
assert _parse_name("Aristotle") == {"literal": "Aristotle"}
class TestExtractDoi:
def test_bare_doi(self):
assert _extract_doi({"doi": "10.1234/test"}) == "10.1234/test"
def test_url_doi(self):
assert (
_extract_doi({"doi": "https://doi.org/10.1234/test"})
== "10.1234/test"
)
def test_doi_list(self):
assert (
_extract_doi({"doi": ["10.1234/test", "10.5678/other"]})
== "10.1234/test"
)
def test_none(self):
assert _extract_doi({}) is None
def test_external_ids_doi(self):
# Semantic Scholar exposes DOIs through `external_ids`. After
# consolidating `_extract_doi` into citation_normalizer, this is
# the canonical home for DOI extraction.
assert (
_extract_doi({"external_ids": {"DOI": "10.1234/test"}})
== "10.1234/test"
)
def test_externalIds_camelCase_doi(self):
# Some Semantic Scholar payloads use camelCase `externalIds`.
assert (
_extract_doi({"externalIds": {"DOI": "10.1234/test"}})
== "10.1234/test"
)
def test_lowercase_doi_key_in_external_ids(self):
# Tolerant of lowercase variant.
assert (
_extract_doi({"external_ids": {"doi": "10.1234/test"}})
== "10.1234/test"
)
def test_link_with_doi_org(self):
# ArXiv-style result that embeds the DOI in the link URL.
assert (
_extract_doi({"link": "https://doi.org/10.1234/test"})
== "10.1234/test"
)
def test_link_with_dx_doi_org(self):
# The legacy `dx.doi.org` redirect form.
assert (
_extract_doi({"link": "https://dx.doi.org/10.1234/test"})
== "10.1234/test"
)
def test_link_with_http_doi_org(self):
# http (not https) variant.
assert (
_extract_doi({"link": "http://doi.org/10.1234/test"})
== "10.1234/test"
)
def test_doi_field_takes_priority_over_external_ids(self):
# When both are present, the explicit `doi` field wins.
assert (
_extract_doi(
{
"doi": "10.1234/explicit",
"external_ids": {"DOI": "10.5678/from-external"},
}
)
== "10.1234/explicit"
)
def test_link_without_doi_returns_none(self):
# A link that doesn't contain a doi.org URL must NOT be matched
# (CodeQL py/incomplete-url-substring-sanitization guard).
assert (
_extract_doi(
{"link": "https://attacker.example/doi.org/10.1234/test"}
)
is None
)
class TestExtractArxivId:
def test_from_url(self):
assert (
_extract_arxiv_id({"link": "https://arxiv.org/abs/2401.12345"})
== "2401.12345"
)
def test_explicit(self):
assert _extract_arxiv_id({"arxiv_id": "2401.12345"}) == "2401.12345"
def test_explicit_takes_precedence_over_url(self):
# Given an explicit ID that differs from the URL fallback
source = {
"arxiv_id": "2401.99999v2",
"link": "https://arxiv.org/abs/2401.12345v1",
}
# When the citation ID is extracted
result = _extract_arxiv_id(source)
# Then the explicit value wins unchanged
assert result == "2401.99999v2"
def test_old_format_hyphen_archive(self):
assert (
_extract_arxiv_id(
{"link": "https://arxiv.org/abs/cond-mat/0501001"}
)
== "cond-mat/0501001"
)
def test_old_format_dotted_subject_class(self):
assert (
_extract_arxiv_id({"link": "https://arxiv.org/abs/math.AG/0601001"})
== "math.AG/0601001"
)
def test_old_format_hep_th(self):
assert (
_extract_arxiv_id({"link": "http://arxiv.org/abs/hep-th/9802150"})
== "hep-th/9802150"
)
def test_new_format_5digit_sequence(self):
assert (
_extract_arxiv_id({"link": "https://arxiv.org/abs/1501.00001"})
== "1501.00001"
)
def test_new_format_with_version(self):
assert (
_extract_arxiv_id({"link": "https://arxiv.org/abs/2501.12345v2"})
== "2501.12345v2"
)
def test_from_versioned_pdf_url(self):
# Given an arXiv PDF URL with query and fragment adornments
source = {
"link": "https://arxiv.org/pdf/math.AG/0601001v4.pdf?download=1#page=3"
}
# When the citation ID is extracted
result = _extract_arxiv_id(source)
# Then the legacy case and explicit version are preserved
assert result == "math.AG/0601001v4"
def test_url_fallback_rejects_unrelated_host(self):
# Given an unrelated host with an arXiv-looking path
source = {"link": "https://example.com/arxiv.org/abs/2501.12345v2"}
# When the citation ID is extracted
result = _extract_arxiv_id(source)
# Then no arXiv ID is inferred from the untrusted path text
assert result is None
def test_schemeless_arxiv_url(self):
assert (
_extract_arxiv_id({"link": "arxiv.org/abs/2401.12345"})
== "2401.12345"
)
def test_schemeless_ar5iv_url(self):
assert (
_extract_arxiv_id({"link": "ar5iv.org/2401.12345"}) == "2401.12345"
)
def test_mixed_case_schemeless_arxiv_url(self):
assert (
_extract_arxiv_id({"link": "ArXiV.OrG/abs/2401.12345"})
== "2401.12345"
)
@pytest.mark.parametrize(
"url",
(
"https://arxiv.org/abs/2401.12345",
"http://arxiv.org/abs/2401.12345",
),
)
def test_fully_qualified_arxiv_url_regression(self, url):
assert _extract_arxiv_id({"link": url}) == "2401.12345"
@pytest.mark.parametrize(
"url",
(
"example.com/abs/2401.12345",
"arxiv.org.evil.com/abs/2401.12345",
),
)
def test_schemeless_non_arxiv_host_is_rejected(self, url):
assert _extract_arxiv_id({"link": url}) is None
@pytest.mark.parametrize(
("url", "expected"),
(
("www.arxiv.org/abs/2401.12345", "2401.12345"),
("export.arxiv.org/abs/2401.12345", "2401.12345"),
("ar5iv.labs.arxiv.org/html/2401.12345", "2401.12345"),
("www.ar5iv.org/abs/2401.12345", "2401.12345"),
("arxiv.org./abs/2401.12345", "2401.12345"),
),
)
def test_schemeless_arxiv_subdomain_url(self, url, expected):
assert _extract_arxiv_id({"link": url}) == expected
class TestWithSchemeForArxivHost:
"""The helper itself: only arXiv-family hosts gain a scheme.
The end-to-end rejection cases above hold even without the helper's
host check, because the strict parser rejects the prefixed lookalike
too; these cases pin the check directly.
"""
@pytest.mark.parametrize(
"url",
(
"arxiv.org/abs/2401.12345",
"ArXiV.OrG/abs/2401.12345",
"ar5iv.org/2401.12345",
"www.arxiv.org/abs/2401.12345",
"export.arxiv.org/abs/2401.12345",
"www.ar5iv.org/abs/2401.12345",
# A trailing-dot (fully qualified) host is the same host.
"arxiv.org./abs/2401.12345",
),
)
def test_arxiv_family_host_gains_a_scheme(self, url):
assert _with_scheme_for_arxiv_host(url) == f"https://{url}"
@pytest.mark.parametrize(
"url",
(
"example.com/abs/2401.12345",
"arxiv.org.evil.com/abs/2401.12345",
"evilarxiv.org/abs/2401.12345",
"[email protected]/abs/2401.12345",
"evil.com\\.arxiv.org/abs/2401.12345",
"[email protected]/abs/2401.12345", # email-domains: allow
"https://arxiv.org/abs/2401.12345",
"/abs/2401.12345",
# Empty and hyphen-edged labels are not plain DNS names.
".arxiv.org/abs/2401.12345",
"a..arxiv.org/abs/2401.12345",
"-.arxiv.org/abs/2401.12345",
),
)
def test_other_input_is_returned_unchanged(self, url):
assert _with_scheme_for_arxiv_host(url) == url
@pytest.mark.parametrize(
"url",
(
# urlparse raises ValueError on each of these: an unbalanced
# bracket in the netloc, and a netloc NFKC folds into "/".
"//[arxiv.org/abs/2101.00001",
"//arxiv.org]/abs/2101.00001",
"//ar\uff0fxiv.org/abs/2101.00001",
),
)
def test_input_urlparse_rejects_is_returned_unchanged(self, url):
assert _with_scheme_for_arxiv_host(url) == url
assert _extract_arxiv_id({"link": url}) is None
result = normalize_citation(
{"title": "T", "link": url, "source": "arXiv"}
)
assert result is not None
assert result.get("arxiv_id") is None
class TestNormalizeCitation:
def test_arxiv_result(self):
source = {
"title": "Attention Is All You Need",
"link": "https://arxiv.org/abs/1706.03762",
"authors": ["Ashish Vaswani", "Noam Shazeer"],
"published": "2017-06-12",
"journal_ref": None,
"source": "arXiv",
"doi": "10.5555/3295222.3295349",
}
result = normalize_citation(source)
assert result is not None
assert result["source_engine"] == "arxiv"
assert result["arxiv_id"] == "1706.03762"
assert result["doi"] == "10.5555/3295222.3295349"
assert result["year"] == 2017
assert len(result["authors"]) == 2
assert result["item_type"] == "article" # No journal_ref = preprint
assert "csl_json" in result
def test_arxiv_published_result(self):
source = {
"title": "Some Paper",
"link": "https://arxiv.org/abs/2401.12345",
"journal_ref": "Nature 612, 34-41 (2023)",
"source": "arXiv",
}
result = normalize_citation(source)
assert result["item_type"] == "article-journal"
assert result["container_title"] == "Nature 612, 34-41 (2023)"
def test_openalex_result(self):
source = {
"title": "A Study",
"link": "https://doi.org/10.1234/test",
"authors": "John Smith, Jane Doe et al.",
"year": 2023,
"journal": "Nature",
"journal_ref": "Nature",
"openalex_source_id": "S12345",
"source_type": "journal",
"doi": "https://doi.org/10.1234/test",
"source": "openalex",
}
result = normalize_citation(source)
assert result is not None
assert result["source_engine"] == "openalex"
assert result["doi"] == "10.1234/test"
assert result["container_title"] == "Nature"
assert result["year"] == 2023
def test_web_result_returns_none(self):
source = {
"title": "Blog Post",
"link": "https://example.com/blog",
}
assert normalize_citation(source) is None
def test_unknown_placeholder_is_filtered(self):
"""OpenAlex / NASA ADS used to emit ``journal="unknown"`` when no
venue was indexed. The normalizer must strip that literal so it
doesn't become a container_title — there's an actual OpenAlex
source named "unknown" (Q1, h_index=5) that would otherwise get
matched by the reputation filter's name-based lookup.
"""
source = {
"title": "Paper",
"link": "https://doi.org/10.1/x",
"journal": "unknown",
"journal_ref": None,
"source": "openalex",
"doi": "10.1/x",
}
result = normalize_citation(source)
assert result is not None
assert result.get("container_title") is None
def test_unknown_placeholder_case_insensitive(self):
"""Same filter catches "Unknown", "UNKNOWN", " unknown " variants."""
for placeholder in ("Unknown", "UNKNOWN", " unknown "):
source = {
"title": "Paper",
"link": "https://doi.org/10.1/x",
"journal": placeholder,
"source": "openalex",
"doi": "10.1/x",
}
result = normalize_citation(source)
assert result is not None
assert result.get("container_title") is None, (
f"placeholder {placeholder!r} leaked through"
)
def test_csl_json_built(self):
source = {
"title": "Test Paper",
"link": "https://arxiv.org/abs/2401.12345",
"authors": ["John Smith"],
"published": "2024-01-15",
"source": "arXiv",
}
result = normalize_citation(source)
csl = result["csl_json"]
assert csl["title"] == "Test Paper"
assert csl["type"] == "article"
assert csl["author"][0]["family"] == "Smith"
assert csl["issued"]["date-parts"] == [[2024, 1, 15]]
assert csl["URL"] == "https://arxiv.org/abs/2401.12345"