An open source, non-profit web search engine
"""
Tests for blacklist providers and the abstraction system.
"""
import re
from unittest.mock import MagicMock, patch
import pytest
import requests
from mwmbl.indexer.blacklist import get_default_blacklist_provider
from mwmbl.indexer.blacklist_providers import (
AdultContentBlacklistProvider,
BuiltInRulesBlacklistProvider,
CombinedBlacklistProvider,
HaGeZiBlacklistProvider,
StaticBlacklistProvider,
domain_and_parents,
domains_to_unblock,
)
def test_static_blacklist_provider():
"""Test StaticBlacklistProvider with is_domain_blacklisted method."""
test_domains = {"spam.com", "malware.example"}
provider = StaticBlacklistProvider(test_domains)
# Test domains that should be blacklisted
assert provider.is_domain_blacklisted("spam.com")
assert provider.is_domain_blacklisted("malware.example")
# Test domain that should not be blacklisted
assert not provider.is_domain_blacklisted("github.com")
def test_hagezi_blacklist_provider_success():
"""Test HaGeZiBlacklistProvider with successful HTTP response."""
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = """# HaGeZi DNS Blocklist
# Comments should be ignored
spam.com
malware.example
# Another comment
badsite.net
"""
mock_get.return_value = mock_response
provider = HaGeZiBlacklistProvider("light")
# Test domains that should be blacklisted
assert provider.is_domain_blacklisted("spam.com")
assert provider.is_domain_blacklisted("malware.example")
assert provider.is_domain_blacklisted("badsite.net")
# Test domain that should not be blacklisted
assert not provider.is_domain_blacklisted("github.com")
def test_adult_content_blacklist_provider_success():
"""Test AdultContentBlacklistProvider parses hosts-format lines correctly."""
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = """# Title: Porn Block List
# Homepage: https://github.com/blocklistproject/Lists
0.0.0.0 fineartteens.com
0.0.0.0 milforia.com
"""
mock_get.return_value = mock_response
provider = AdultContentBlacklistProvider()
assert provider.is_domain_blacklisted("fineartteens.com")
assert provider.is_domain_blacklisted("milforia.com")
assert not provider.is_domain_blacklisted("github.com")
def test_hagezi_blacklist_provider_invalid_type():
"""Test HaGeZiBlacklistProvider rejects invalid list types."""
with pytest.raises(ValueError):
HaGeZiBlacklistProvider("invalid_type")
def test_combined_blacklist_provider():
"""Test CombinedBlacklistProvider with is_domain_blacklisted method."""
provider1 = StaticBlacklistProvider({"spam.com", "malware.example"})
provider2 = StaticBlacklistProvider({"badsite.net", "phishing.site"})
combined = CombinedBlacklistProvider([provider1, provider2])
# Test domains from both providers
assert combined.is_domain_blacklisted("spam.com")
assert combined.is_domain_blacklisted("badsite.net")
assert combined.is_domain_blacklisted("phishing.site")
# Test domain that should not be blacklisted
assert not combined.is_domain_blacklisted("github.com")
def test_combined_blacklist_provider_handles_failures():
"""Test CombinedBlacklistProvider continues even if one provider fails."""
good_provider = StaticBlacklistProvider({"spam.com"})
bad_provider = MagicMock()
bad_provider.is_domain_blacklisted.side_effect = Exception("Provider failure")
combined = CombinedBlacklistProvider([good_provider, bad_provider])
# Should still work with the good provider despite the bad one failing
assert combined.is_domain_blacklisted("spam.com")
assert not combined.is_domain_blacklisted("github.com")
def test_integration_with_blacklist_module():
"""Test integration with the main blacklist module."""
# Test the default provider factory
default_provider = get_default_blacklist_provider()
assert default_provider is not None
# Test that it can check domains
# Test with a domain that should be blacklisted by built-in rules
assert default_provider.is_domain_blacklisted("59648.etnomurcia.com")
# Test with a domain that should not be blacklisted
assert not default_provider.is_domain_blacklisted("github.com")
def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():
"""These lists are tens of megabytes. mwmbl.utils.request_cache is a *filesystem* cache
rooted at settings.REQUEST_CACHE_PATH, on the same volume as the index - caching them
there fills that volume, and once it is full every other user of request_cache breaks,
including get_wiki_results() on the live search path (which then retries a live fetch
4x per uncached query until the worker dies). Regression test: fetch must not go
through request_cache."""
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = "spam.com\n"
mock_get.return_value = mock_response
# If the module still imported request_cache, patching it would succeed; assert the
# name is gone from the module so this can't silently regress.
import mwmbl.indexer.blacklist_providers as bp
assert not hasattr(bp, "request_cache"), "blacklist_providers must not use the shared filesystem request_cache"
provider = HaGeZiBlacklistProvider("light")
assert provider.is_domain_blacklisted("spam.com") is True
assert mock_get.called
def test_fetch_failure_propagates():
"""A failed fetch must not look like a list that blacklists nothing.
build_snapshot() publishes the union of these lists to every search worker, so a
provider that swallowed the error and returned an empty set would replace a good
snapshot with one missing its entire contribution."""
provider = HaGeZiBlacklistProvider("light")
with patch("mwmbl.indexer.blacklist_providers.requests.get", side_effect=requests.RequestException("boom")):
with pytest.raises(requests.RequestException):
provider.is_domain_blacklisted("spam.com")
def test_the_list_is_fetched_once_per_instance():
"""The cost of a remote list is owned by whoever holds the provider.
There is no cache behind the module - that made get_default_blacklist_provider() look
cheap enough to call from a request handler. Reuse comes from holding the instance."""
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = "spam.com\n"
mock_get.return_value = mock_response
provider = HaGeZiBlacklistProvider("light")
assert provider.is_domain_blacklisted("spam.com") is True
assert provider.is_domain_blacklisted("other.com") is False
assert mock_get.call_count == 1
# A second instance is a second download - nothing is shared between them.
assert HaGeZiBlacklistProvider("light").is_domain_blacklisted("spam.com") is True
assert mock_get.call_count == 2
def test_hosts_lines_with_trailing_comments_keep_the_domain():
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = "0.0.0.0 spam.com # added 2026-01-01\n# a comment line\nplain.com\n"
mock_get.return_value = mock_response
provider = AdultContentBlacklistProvider()
assert provider._get_blacklisted_domains() == {"spam.com", "plain.com"}
def test_domain_and_parents():
assert domain_and_parents("nyou.booru.org") == ["nyou.booru.org", "booru.org"]
assert domain_and_parents("booru.org") == ["booru.org"]
# Never down to a bare TLD, which would take out everything under it.
assert "org" not in domain_and_parents("a.b.c.org")
assert domain_and_parents("localhost") == ["localhost"]
def test_a_remote_list_entry_covers_subdomains():
"""The lists are apex rules, so www.spam.com must be caught by an entry for spam.com."""
with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
mock_response = MagicMock()
mock_response.text = "spam.com\n"
mock_get.return_value = mock_response
provider = HaGeZiBlacklistProvider("light")
assert provider.is_domain_blacklisted("spam.com") is True
assert provider.is_domain_blacklisted("www.spam.com") is True
assert provider.is_domain_blacklisted("cdn.images.spam.com") is True
assert provider.is_domain_blacklisted("notspam.com") is False
assert provider.is_domain_blacklisted("spam.com.example.org") is False
def test_excluded_domains_cover_subdomains():
"""EXCLUDED_DOMAINS entries are written apex-first; the www. form must still match."""
with patch.object(BuiltInRulesBlacklistProvider, "__init__", lambda self: None):
provider = BuiltInRulesBlacklistProvider()
provider.excluded_domains = {"fineartteens.com"}
provider.blacklist_regex = re.compile(r"^$")
provider.trusted_domains = set()
assert provider.is_domain_blacklisted("fineartteens.com") is True
assert provider.is_domain_blacklisted("www.fineartteens.com") is True
assert provider.is_domain_blacklisted("otherfineartteens.com") is False
def test_domains_to_unblock_adds_the_apex_for_www_submissions():
"""Submissions store whatever netloc was submitted, but the lists hold the apex."""
assert domains_to_unblock({"www.contactmusic.com"}) == {"www.contactmusic.com", "contactmusic.com"}
assert domains_to_unblock({"pudding.cool"}) == {"pudding.cool"}
# Only the www. label is stripped: clearing en-academic.com off the strength of an
# approved subdomain would unblock every other subdomain too.
assert domains_to_unblock({"blog.example.com"}) == {"blog.example.com"}
def test_curated_domains_override_the_providers():
provider = CombinedBlacklistProvider(
[StaticBlacklistProvider({"pudding.cool", "reallybad.example"})],
get_exempt_domains=lambda: {"pudding.cool"},
)
assert provider.is_domain_blacklisted("pudding.cool") is False
assert provider.is_domain_blacklisted("reallybad.example") is True
def test_curated_domains_override_the_built_in_rules():
"""A moderator's approval beats the adult-content regex, which is what makes the
approval flow usable for domains the lists get wrong."""
provider = CombinedBlacklistProvider(
[BuiltInRulesBlacklistProvider()],
get_exempt_domains=lambda: {"adultswim.example"},
)
assert provider.is_domain_blacklisted("adultswim.example") is False
assert provider.is_domain_blacklisted("otheradult.example") is True
def test_curated_domains_match_either_www_form():
provider = CombinedBlacklistProvider(
[StaticBlacklistProvider({"contactmusic.com", "www.queer.example"})],
get_exempt_domains=lambda: {"www.contactmusic.com", "queer.example"},
)
assert provider.is_domain_blacklisted("contactmusic.com") is False
assert provider.is_domain_blacklisted("www.queer.example") is False
def test_exempt_domains_are_resolved_once():
"""Resolving costs a database query or an HTTP fetch, so it happens once per provider,
like the remote lists themselves."""
calls = []
def get_exempt_domains():
calls.append(1)
return {"pudding.cool"}
provider = CombinedBlacklistProvider(
[StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains
)
for _ in range(5):
provider.is_domain_blacklisted("pudding.cool")
assert len(calls) == 1
def test_default_provider_takes_the_curated_domains_it_is_given():
provider = get_default_blacklist_provider(lambda: {"badsite.example"})
assert provider.get_exempt_domains() == {"badsite.example"}