Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

11.8 KB
Raw Blame
"""
Tests for blacklist providers and the abstraction system.
"""

import re
from unittest.mock import MagicMock, patch

import pytest
import requests

from mwmbl.indexer.blacklist import get_default_blacklist_provider
from mwmbl.indexer.blacklist_providers import (
    AdultContentBlacklistProvider,
    BuiltInRulesBlacklistProvider,
    CombinedBlacklistProvider,
    HaGeZiBlacklistProvider,
    StaticBlacklistProvider,
    domain_and_parents,
    domains_to_unblock,
)


def test_static_blacklist_provider():
    """Test StaticBlacklistProvider with is_domain_blacklisted method."""
    test_domains = {"spam.com", "malware.example"}
    provider = StaticBlacklistProvider(test_domains)

    # Test domains that should be blacklisted
    assert provider.is_domain_blacklisted("spam.com")
    assert provider.is_domain_blacklisted("malware.example")

    # Test domain that should not be blacklisted
    assert not provider.is_domain_blacklisted("github.com")


def test_hagezi_blacklist_provider_success():
    """Test HaGeZiBlacklistProvider with successful HTTP response."""
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = """# HaGeZi DNS Blocklist
# Comments should be ignored
spam.com
malware.example

# Another comment
badsite.net
"""
        mock_get.return_value = mock_response

        provider = HaGeZiBlacklistProvider("light")

        # Test domains that should be blacklisted
        assert provider.is_domain_blacklisted("spam.com")
        assert provider.is_domain_blacklisted("malware.example")
        assert provider.is_domain_blacklisted("badsite.net")

        # Test domain that should not be blacklisted
        assert not provider.is_domain_blacklisted("github.com")


def test_adult_content_blacklist_provider_success():
    """Test AdultContentBlacklistProvider parses hosts-format lines correctly."""
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = """# Title: Porn Block List
# Homepage: https://github.com/blocklistproject/Lists

0.0.0.0 fineartteens.com
0.0.0.0 milforia.com
"""
        mock_get.return_value = mock_response

        provider = AdultContentBlacklistProvider()

        assert provider.is_domain_blacklisted("fineartteens.com")
        assert provider.is_domain_blacklisted("milforia.com")
        assert not provider.is_domain_blacklisted("github.com")


def test_hagezi_blacklist_provider_invalid_type():
    """Test HaGeZiBlacklistProvider rejects invalid list types."""
    with pytest.raises(ValueError):
        HaGeZiBlacklistProvider("invalid_type")


def test_combined_blacklist_provider():
    """Test CombinedBlacklistProvider with is_domain_blacklisted method."""
    provider1 = StaticBlacklistProvider({"spam.com", "malware.example"})
    provider2 = StaticBlacklistProvider({"badsite.net", "phishing.site"})

    combined = CombinedBlacklistProvider([provider1, provider2])

    # Test domains from both providers
    assert combined.is_domain_blacklisted("spam.com")
    assert combined.is_domain_blacklisted("badsite.net")
    assert combined.is_domain_blacklisted("phishing.site")

    # Test domain that should not be blacklisted
    assert not combined.is_domain_blacklisted("github.com")


def test_combined_blacklist_provider_handles_failures():
    """Test CombinedBlacklistProvider continues even if one provider fails."""
    good_provider = StaticBlacklistProvider({"spam.com"})
    bad_provider = MagicMock()
    bad_provider.is_domain_blacklisted.side_effect = Exception("Provider failure")

    combined = CombinedBlacklistProvider([good_provider, bad_provider])

    # Should still work with the good provider despite the bad one failing
    assert combined.is_domain_blacklisted("spam.com")
    assert not combined.is_domain_blacklisted("github.com")


def test_integration_with_blacklist_module():
    """Test integration with the main blacklist module."""
    # Test the default provider factory
    default_provider = get_default_blacklist_provider()
    assert default_provider is not None

    # Test that it can check domains
    # Test with a domain that should be blacklisted by built-in rules
    assert default_provider.is_domain_blacklisted("59648.etnomurcia.com")

    # Test with a domain that should not be blacklisted
    assert not default_provider.is_domain_blacklisted("github.com")


def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():
    """These lists are tens of megabytes. mwmbl.utils.request_cache is a *filesystem* cache
    rooted at settings.REQUEST_CACHE_PATH, on the same volume as the index - caching them
    there fills that volume, and once it is full every other user of request_cache breaks,
    including get_wiki_results() on the live search path (which then retries a live fetch
    4x per uncached query until the worker dies). Regression test: fetch must not go
    through request_cache."""
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = "spam.com\n"
        mock_get.return_value = mock_response

        # If the module still imported request_cache, patching it would succeed; assert the
        # name is gone from the module so this can't silently regress.
        import mwmbl.indexer.blacklist_providers as bp

        assert not hasattr(bp, "request_cache"), "blacklist_providers must not use the shared filesystem request_cache"

        provider = HaGeZiBlacklistProvider("light")
        assert provider.is_domain_blacklisted("spam.com") is True
        assert mock_get.called


def test_fetch_failure_propagates():
    """A failed fetch must not look like a list that blacklists nothing.

    build_snapshot() publishes the union of these lists to every search worker, so a
    provider that swallowed the error and returned an empty set would replace a good
    snapshot with one missing its entire contribution."""
    provider = HaGeZiBlacklistProvider("light")
    with patch("mwmbl.indexer.blacklist_providers.requests.get", side_effect=requests.RequestException("boom")):
        with pytest.raises(requests.RequestException):
            provider.is_domain_blacklisted("spam.com")


def test_the_list_is_fetched_once_per_instance():
    """The cost of a remote list is owned by whoever holds the provider.

    There is no cache behind the module - that made get_default_blacklist_provider() look
    cheap enough to call from a request handler. Reuse comes from holding the instance."""
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = "spam.com\n"
        mock_get.return_value = mock_response

        provider = HaGeZiBlacklistProvider("light")
        assert provider.is_domain_blacklisted("spam.com") is True
        assert provider.is_domain_blacklisted("other.com") is False
        assert mock_get.call_count == 1

        # A second instance is a second download - nothing is shared between them.
        assert HaGeZiBlacklistProvider("light").is_domain_blacklisted("spam.com") is True
        assert mock_get.call_count == 2


def test_hosts_lines_with_trailing_comments_keep_the_domain():
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = "0.0.0.0 spam.com # added 2026-01-01\n# a comment line\nplain.com\n"
        mock_get.return_value = mock_response

        provider = AdultContentBlacklistProvider()
        assert provider._get_blacklisted_domains() == {"spam.com", "plain.com"}


def test_domain_and_parents():
    assert domain_and_parents("nyou.booru.org") == ["nyou.booru.org", "booru.org"]
    assert domain_and_parents("booru.org") == ["booru.org"]
    # Never down to a bare TLD, which would take out everything under it.
    assert "org" not in domain_and_parents("a.b.c.org")
    assert domain_and_parents("localhost") == ["localhost"]


def test_a_remote_list_entry_covers_subdomains():
    """The lists are apex rules, so www.spam.com must be caught by an entry for spam.com."""
    with patch("mwmbl.indexer.blacklist_providers.requests.get") as mock_get:
        mock_response = MagicMock()
        mock_response.text = "spam.com\n"
        mock_get.return_value = mock_response

        provider = HaGeZiBlacklistProvider("light")
        assert provider.is_domain_blacklisted("spam.com") is True
        assert provider.is_domain_blacklisted("www.spam.com") is True
        assert provider.is_domain_blacklisted("cdn.images.spam.com") is True
        assert provider.is_domain_blacklisted("notspam.com") is False
        assert provider.is_domain_blacklisted("spam.com.example.org") is False


def test_excluded_domains_cover_subdomains():
    """EXCLUDED_DOMAINS entries are written apex-first; the www. form must still match."""
    with patch.object(BuiltInRulesBlacklistProvider, "__init__", lambda self: None):
        provider = BuiltInRulesBlacklistProvider()
        provider.excluded_domains = {"fineartteens.com"}
        provider.blacklist_regex = re.compile(r"^$")
        provider.trusted_domains = set()

        assert provider.is_domain_blacklisted("fineartteens.com") is True
        assert provider.is_domain_blacklisted("www.fineartteens.com") is True
        assert provider.is_domain_blacklisted("otherfineartteens.com") is False


def test_domains_to_unblock_adds_the_apex_for_www_submissions():
    """Submissions store whatever netloc was submitted, but the lists hold the apex."""
    assert domains_to_unblock({"www.contactmusic.com"}) == {"www.contactmusic.com", "contactmusic.com"}
    assert domains_to_unblock({"pudding.cool"}) == {"pudding.cool"}
    # Only the www. label is stripped: clearing en-academic.com off the strength of an
    # approved subdomain would unblock every other subdomain too.
    assert domains_to_unblock({"blog.example.com"}) == {"blog.example.com"}


def test_curated_domains_override_the_providers():
    provider = CombinedBlacklistProvider(
        [StaticBlacklistProvider({"pudding.cool", "reallybad.example"})],
        get_exempt_domains=lambda: {"pudding.cool"},
    )

    assert provider.is_domain_blacklisted("pudding.cool") is False
    assert provider.is_domain_blacklisted("reallybad.example") is True


def test_curated_domains_override_the_built_in_rules():
    """A moderator's approval beats the adult-content regex, which is what makes the
    approval flow usable for domains the lists get wrong."""
    provider = CombinedBlacklistProvider(
        [BuiltInRulesBlacklistProvider()],
        get_exempt_domains=lambda: {"adultswim.example"},
    )

    assert provider.is_domain_blacklisted("adultswim.example") is False
    assert provider.is_domain_blacklisted("otheradult.example") is True


def test_curated_domains_match_either_www_form():
    provider = CombinedBlacklistProvider(
        [StaticBlacklistProvider({"contactmusic.com", "www.queer.example"})],
        get_exempt_domains=lambda: {"www.contactmusic.com", "queer.example"},
    )

    assert provider.is_domain_blacklisted("contactmusic.com") is False
    assert provider.is_domain_blacklisted("www.queer.example") is False


def test_exempt_domains_are_resolved_once():
    """Resolving costs a database query or an HTTP fetch, so it happens once per provider,
    like the remote lists themselves."""
    calls = []

    def get_exempt_domains():
        calls.append(1)
        return {"pudding.cool"}

    provider = CombinedBlacklistProvider(
        [StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains
    )

    for _ in range(5):
        provider.is_domain_blacklisted("pudding.cool")

    assert len(calls) == 1


def test_default_provider_takes_the_curated_domains_it_is_given():
    provider = get_default_blacklist_provider(lambda: {"badsite.example"})

    assert provider.get_exempt_domains() == {"badsite.example"}