An open source, non-profit web search engine
Let curated domains override the remote blocklists
Daoud Clarke83cd13a6cb52
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.pyindex 5303883..f84a86e 100644--- a/mwmbl/admin_views.py+++ b/mwmbl/admin_views.py@@ -22,6 +22,7 @@ from django.shortcuts import renderfrom redis import RedisErrorfrom mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY+from mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer import blacklist_snapshot, purge_queuefrom mwmbl.indexer.blacklist_snapshot import (HASH_DTYPE, SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY, get_snapshot_blacklist,@@ -100,6 +101,23 @@ def _removed_counts(days: int) -> list[dict]:return [{"date": date, "count": int(count) if count else 0} for date, count in zip(dates, counts)]+def _curated_status() -> dict:+ """Approved domains, and whether any of them are still being filtered out.++ Curated domains are subtracted from the remote lists when the snapshot is built, so+ "still blacklisted" is normally zero. Anything listed here is either waiting for the+ next rebuild, or blocked by a local rule curation does not override - EXCLUDED_DOMAINS+ or DOMAIN_BLACKLIST_REGEX - which is otherwise invisible.+ """+ curated_domains = get_curated_domains()+ still_blacklisted = get_snapshot_blacklist().filter_blacklisted(curated_domains)+ return {+ "count": len(curated_domains),+ "still_blacklisted": sorted(still_blacklisted),+ "approval_delay_seconds": settings.BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS,+ }++def _task_status() -> list[dict]:"""The two background tasks that maintain the state above.@@ -137,6 +155,7 @@ def blacklist_status_view(request):# handler covers the lot.try:context["snapshot"] = _snapshot_status()+ context["curated"] = _curated_status()context["queue"] = _queue_status()context["removed_counts"] = _removed_counts(REMOVED_COUNT_DAYS)except RedisError as e:diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex 524d2db..c51665f 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -42,6 +42,7 @@ class MwmblConfig(AppConfig):create_index()if settings.HAS_DATABASE:create_index_db()+ import mwmbl.signals # noqa: F401 - connects the post_save receiversself._schedule_background_tasks()# Cache key guarding the scheduling critical section below. Held just long@@ -109,7 +110,11 @@ class MwmblConfig(AppConfig):# on the task means the first run happens immediately rather than one full# refresh interval after deploy, which matters because until it lands the# search workers have only the built-in rules to go on.- if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():+ #+ # Only the *repeating* row counts here: approving a domain submission schedules+ # a one-off rebuild under the same task name (mwmbl.signals), and one of those+ # sitting in the queue at deploy time must not suppress the periodic task.+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, repeat__gt=0).exists():refresh_blacklist_snapshot(repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 3407052..836d8ff 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -255,7 +255,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch),)def get_curated_domains_endpoint(request) -> CuratedDomainsResponse:- from mwmbl.search_setup import get_curated_domains+ from mwmbl.curated_domains import get_curated_domainsreturn CuratedDomainsResponse(domains=[CuratedDomain(name=d) for d in sorted(get_curated_domains())])diff --git a/mwmbl/curated_domains.py b/mwmbl/curated_domains.pynew file mode 100644index 0000000..a28593b--- /dev/null+++ b/mwmbl/curated_domains.py@@ -0,0 +1,32 @@+"""The set of domains a human moderator has approved.++Approved DomainSubmissions are the override for the remote blocklists. Those lists are+maintained for DNS ad-blocking and carry false positives that are wrong for a search index+- pudding.cool, contactmusic.com and character.ai are all in The Block List Project's+porn.txt - so an approval here removes the domain from the blacklist rather than merely+outranking it.++This lives outside search_setup because search_setup imports blacklist_snapshot, and the+blacklist code needs to import this. The curated set is only ever read where a blacklist is+*constructed* (build_snapshot, CombinedBlacklistProvider), never per query.+"""+from django.conf import settings+from django.core.cache import cache++from mwmbl.models import DomainSubmission++CURATED_DOMAINS_CACHE_KEY = "curated-domains"+CURATED_DOMAINS_CACHE_TIMEOUT = 300+++def get_curated_domains() -> set[str]:+ # The standalone crawler runs django.setup() against settings_crawler, which has no+ # database; it fetches the same names over HTTP instead, see crawl.py.+ if not settings.HAS_DATABASE:+ return set()++ curated_domains = cache.get(CURATED_DOMAINS_CACHE_KEY)+ if curated_domains is None:+ curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))+ cache.set(CURATED_DOMAINS_CACHE_KEY, curated_domains, CURATED_DOMAINS_CACHE_TIMEOUT)+ return curated_domainsdiff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.pyindex 08f4652..507c2bb 100644--- a/mwmbl/indexer/blacklist.py+++ b/mwmbl/indexer/blacklist.py@@ -5,6 +5,9 @@ This module provides utility functions for creating default blacklist providers.The main logic has been moved to blacklist_providers.py for better modularity."""+from typing import Callable, Optional, Set++from mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer.blacklist_providers import (BuiltInRulesBlacklistProvider,HaGeZiBlacklistProvider,@@ -14,10 +17,19 @@ from mwmbl.indexer.blacklist_providers import ()-def get_default_blacklist_provider() -> BlacklistProvider:- """Get the default blacklist provider configuration."""- return CombinedBlacklistProvider([- BuiltInRulesBlacklistProvider(),- HaGeZiBlacklistProvider('tif_medium'),- AdultContentBlacklistProvider(),- ])+def get_default_blacklist_provider(+ get_exempt_domains: Optional[Callable[[], Set[str]]] = None) -> BlacklistProvider:+ """Get the default blacklist provider configuration.++ Domains a moderator has approved override the lists. Callers without a database - the+ standalone crawler - pass their own source of approved names; the default reads them+ from DomainSubmission and returns nothing when there is no database.+ """+ return CombinedBlacklistProvider(+ [+ BuiltInRulesBlacklistProvider(),+ HaGeZiBlacklistProvider('tif_medium'),+ AdultContentBlacklistProvider(),+ ],+ get_exempt_domains=get_exempt_domains if get_exempt_domains is not None else get_curated_domains,+ )diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.pyindex c4affff..1141948 100644--- a/mwmbl/indexer/blacklist_providers.py+++ b/mwmbl/indexer/blacklist_providers.py@@ -6,7 +6,7 @@ making the system more flexible and testable."""from abc import ABC, abstractmethod-from typing import Optional, Set+from typing import Callable, Optional, Setimport requests@@ -32,6 +32,19 @@ def domain_and_parents(domain: str) -> list[str]:return ['.'.join(parts[i:]) for i in range(len(parts) - 1)]+def domains_to_unblock(curated_domains: Set[str]) -> Set[str]:+ """The blocklist entries these curated domains should clear.++ DomainSubmissionForm.clean_name stores whatever netloc was submitted, so an approval can+ arrive as www.contactmusic.com while the blocklist holds the apex contactmusic.com.+ Stripping the www. covers that; nothing deeper is stripped, because clearing an apex+ entry on the strength of an approved subdomain would unblock every one of its siblings.+ """+ unblocked = set(curated_domains)+ unblocked.update(domain[len("www."):] for domain in curated_domains if domain.startswith("www."))+ return unblocked++class BlacklistProvider(ABC):"""Abstract base class for blacklist providers."""@@ -215,13 +228,33 @@ class AdultContentBlacklistProvider(RemoteListBlacklistProvider):class CombinedBlacklistProvider(BlacklistProvider):- """Provider that combines multiple blacklist providers."""-- def __init__(self, providers: list[BlacklistProvider]):+ """Provider that combines multiple blacklist providers.++ `get_exempt_domains` supplies the domains a moderator has approved, which override every+ sub-provider. It is a callable rather than a set because the caller decides where the+ approved names come from - the database in the server, an HTTP fetch in the standalone+ crawler - and the result is resolved once and held, like the remote lists themselves.+ """++ def __init__(self, providers: list[BlacklistProvider],+ get_exempt_domains: Optional[Callable[[], Set[str]]] = None):self.providers = providers-+ self.get_exempt_domains = get_exempt_domains+ self._exempt_domains: Optional[Set[str]] = None++ def _is_exempt(self, domain: str) -> bool:+ if self.get_exempt_domains is None:+ return False+ if self._exempt_domains is None:+ self._exempt_domains = domains_to_unblock(self.get_exempt_domains())+ apex = domain[len("www."):] if domain.startswith("www.") else domain+ return domain in self._exempt_domains or apex in self._exempt_domains+def is_domain_blacklisted(self, domain: str) -> bool:"""Check if domain is blacklisted by any provider."""+ if self._is_exempt(domain):+ return False+for provider in self.providers:try:if provider.is_domain_blacklisted(domain):diff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.pyindex fb709a4..6cf56d6 100644--- a/mwmbl/indexer/blacklist_snapshot.py+++ b/mwmbl/indexer/blacklist_snapshot.py@@ -30,6 +30,15 @@ EXCLUDED_DOMAINS, the numeric-subdomain heuristic and the hn_top_domains trust eis cheap and stays a local in-process check, so editing mwmbl/settings.py takes effect ondeploy instead of waiting for the next snapshot refresh. Evaluating them as`built_in_rules OR snapshot` reproduces CombinedBlacklistProvider's semantics exactly.++Curated domains - the ones a moderator has approved, see mwmbl.curated_domains - are+subtracted here, at build time, rather than being checked per query. The remote lists are+maintained for DNS ad-blocking and their false positives are wrong for a search index, so+an approval has to remove the domain from the blacklist outright. Dropping the hash while+the snapshot is built means the query path stays a single binary search with no database+access, and every worker picks the exemption up with the next snapshot. The cost is that an+approval only takes effect on the next rebuild, which is why approving a submission+schedules one (mwmbl.signals)."""import hashlibimport threading@@ -42,6 +51,7 @@ import numpy as npimport redisfrom django.conf import settings+from mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer.blacklist import get_default_blacklist_providerfrom mwmbl.indexer.blacklist_providers import (BlacklistProvider,@@ -49,6 +59,7 @@ from mwmbl.indexer.blacklist_providers import (CombinedBlacklistProvider,RemoteListBlacklistProvider,domain_and_parents,+ domains_to_unblock,)logger = getLogger(__name__)@@ -107,12 +118,19 @@ def collect_remote_domains(provider: BlacklistProvider) -> set[str]:def build_snapshot(provider: BlacklistProvider) -> bytes:- """Download/parse the remote lists and return the sorted hash array as bytes."""- domains = collect_remote_domains(provider)+ """Download/parse the remote lists and return the sorted hash array as bytes.++ Curated domains are removed from the listed set first - see the module docstring.+ """+ listed_domains = collect_remote_domains(provider)+ unblocked = domains_to_unblock(get_curated_domains())+ domains = listed_domains - unblocked+ num_unblocked = len(listed_domains) - len(domains)all_hashes = hash_domains(domains)hashes = np.unique(all_hashes) # np.unique sorts, which is what we need- logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB",- len(domains), len(hashes), hashes.nbytes / 1e6)+ logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB; "+ "%d entries removed by curated domains",+ len(domains), len(hashes), hashes.nbytes / 1e6, num_unblocked)return hashes.astype(HASH_DTYPE).tobytes()@@ -268,7 +286,10 @@ class SnapshotBlacklist:blacklisted = {d for d in domains if self._built_in_rules.is_domain_blacklisted(d)}array = self._array # read once: a background refresh may swap it mid-call- if array is None:+ if array is None or len(array) == 0:+ # An empty array is not the same as no array, and it cannot go through the+ # search below: clamping an out-of-range position to index 0 needs there to be+ # an index 0. Nothing is a member of an empty snapshot anyway.return blacklistedremaining = [d for d in domains if d not in blacklisted]diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex bdd2125..aa06815 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -41,7 +41,9 @@ def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue, num_batches: intdef record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: RedisURLQueue):start = datetime.utcnow()- blacklist_provider = get_default_blacklist_provider()+ # Take the approved domains from the queue rather than the database: the standalone+ # crawler is the live caller of this, and it fetches them over HTTP.+ blacklist_provider = get_default_blacklist_provider(new_item_queue.get_curated_domains_function)blacklist_retrieval_time = datetime.utcnow() - startlogger.info(f"Recording URLs in database for {len(batches)} batches, blacklist provider ready "f"in {blacklist_retrieval_time.total_seconds()} seconds")diff --git a/mwmbl/main.py b/mwmbl/main.pyindex 6329f98..bb329f1 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -48,7 +48,7 @@ def run():from mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.count_urls import count_urls_continuouslyfrom mwmbl.indexer.update_urls import update_urls_continuously- from mwmbl.search_setup import get_curated_domains+ from mwmbl.curated_domains import get_curated_domainsif settings.STATIC_ROOT:call_command("collectstatic", "--clear", "--noinput")diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex c816875..db8b11d 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -55,7 +55,10 @@ class RedisURLQueue:def __init__(self, redis: Redis, get_curated_domains_function: Callable[[], set[str]], blacklist_provider=None) -> None:self.redis = redisself.get_curated_domains_function = get_curated_domains_function- self.blacklist_provider = blacklist_provider or get_default_blacklist_provider()+ # Curated domains override the blacklist, and the queue already knows where to get+ # them - over HTTP in the standalone crawler, which has no database.+ self.blacklist_provider = blacklist_provider or get_default_blacklist_provider(+ get_curated_domains_function)def queue_urls(self, found_urls: list[FoundURL]):curated_domains = self.get_curated_domains_function()diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 64b0c2c..96bc2a0 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -2,12 +2,11 @@ import osfrom pathlib import Pathfrom django.conf import settings-from django.core.cache import cachefrom redis import Redis+from mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist-from mwmbl.models import DomainSubmissionfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document@@ -16,18 +15,6 @@ from mwmbl.tinysearchengine.ltr_rank import LTRRankerfrom mwmbl.tinysearchengine.mmr_rank import MMRRankerfrom mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker-CURATED_DOMAINS_CACHE_KEY = "curated-domains"-CURATED_DOMAINS_CACHE_TIMEOUT = 300---def get_curated_domains() -> set[str]:- curated_domains = cache.get(CURATED_DOMAINS_CACHE_KEY)- if curated_domains is None:- curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))- cache.set(CURATED_DOMAINS_CACHE_KEY, curated_domains, CURATED_DOMAINS_CACHE_TIMEOUT)- return curated_domains--redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)# Pull the blacklist snapshot in at worker startup rather than letting the first querydiff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex c6c9644..faa3566 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -332,3 +332,7 @@ BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis forBLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote listsBLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drainedBLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge run+# How far ahead approving a domain submission schedules a snapshot rebuild. Approvers work+# through submissions in batches, so the delay collapses a batch into one rebuild rather+# than one per approval - see mwmbl.signals.+BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600diff --git a/mwmbl/signals.py b/mwmbl/signals.pynew file mode 100644index 0000000..7980e7a--- /dev/null+++ b/mwmbl/signals.py@@ -0,0 +1,53 @@+"""Signal receivers. Connected from MwmblConfig.ready().++Only one so far: approving a domain submission has to reach the blacklist snapshot, because+approved domains are subtracted from the remote lists when the snapshot is built rather than+checked per query (see mwmbl.indexer.blacklist_snapshot). Without this an approval would sit+inert for up to BLACKLIST_SNAPSHOT_REFRESH_SECONDS - six hours - and the moderator would+reasonably conclude it had not worked.+"""+from datetime import timedelta+from logging import getLogger++from background_task.models import Task+from django.conf import settings+from django.db.models.signals import post_save+from django.dispatch import receiver+from django.utils import timezone++from mwmbl.models import DomainSubmission++logger = getLogger(__name__)+++BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"+++@receiver(post_save, sender=DomainSubmission)+def rebuild_blacklist_snapshot_on_approval(sender, instance: DomainSubmission, **kwargs):+ """Schedule a snapshot rebuild when a submission is approved.++ Debounced rather than immediate. Moderators work through the queue in batches, and each+ rebuild downloads and parses tens of megabytes, so the run is scheduled+ BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS ahead and only if nothing is already due by+ then. The check matches the repeating task's row too, so an approval shortly before the+ periodic rebuild adds nothing at all.++ post_save carries no previous value, so this fires on every save of an approved+ submission rather than only on the transition. The debounce absorbs that. Two approvals+ saved at once could both schedule a run, which is harmless: publish_snapshot versions the+ blob by content hash, so an identical snapshot does not make the workers re-read 11 MB.+ """+ if instance.status != "APPROVED":+ return++ from mwmbl.background import refresh_blacklist_snapshot++ delay = settings.BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS+ cutoff = timezone.now() + timedelta(seconds=delay)+ if Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, run_at__lte=cutoff).exists():+ return++ refresh_blacklist_snapshot(schedule=delay)+ logger.info("Scheduled a blacklist snapshot rebuild in %ds after approving %s",+ delay, instance.name)diff --git a/mwmbl/templates/admin/blacklist_status.html b/mwmbl/templates/admin/blacklist_status.htmlindex b2565cf..8a10368 100644--- a/mwmbl/templates/admin/blacklist_status.html+++ b/mwmbl/templates/admin/blacklist_status.html@@ -105,6 +105,39 @@</p></div>+ <div class="module">+ <h2>Curated domains</h2>+ <table>+ <tr>+ <td class="key">Approved domains</td>+ <td>{{ curated.count }}</td>+ </tr>+ <tr>+ <td class="key">Still blacklisted</td>+ <td>+ {% if curated.still_blacklisted %}+ <span class="bad">{{ curated.still_blacklisted|length }}</span>+ —+ {% for domain in curated.still_blacklisted %}+ <code>{{ domain }}</code>{% if not forloop.last %}, {% endif %}+ {% endfor %}+ {% else %}+ <span class="ok">none</span>+ {% endif %}+ </td>+ </tr>+ </table>+ <p class="note">+ Approved domain submissions are subtracted from the remote blocklists when the+ snapshot is built, so approving a domain is how a false positive gets unblocked.+ An approval schedules a rebuild {{ curated.approval_delay_seconds }}s later, which+ collapses a batch of approvals into one rebuild. Domains listed above as still+ blacklisted are either waiting for that rebuild, or blocked by a local rule that+ curation does not override — EXCLUDED_DOMAINS or DOMAIN_BLACKLIST_REGEX in+ mwmbl/settings.py.+ </p>+ </div>+<div class="module"><h2>Purge queue</h2><table>diff --git a/test/test_admin_blacklist_status.py b/test/test_admin_blacklist_status.pyindex d079e61..79a0583 100644--- a/test/test_admin_blacklist_status.py+++ b/test/test_admin_blacklist_status.py@@ -7,7 +7,9 @@ from redis import ConnectionError as RedisConnectionErrorfrom mwmbl import admin_viewsfrom mwmbl.indexer import blacklist_snapshot, purge_queue-from mwmbl.indexer.blacklist_snapshot import HASH_DTYPE, SnapshotBlacklist, publish_snapshot+from mwmbl.indexer.blacklist_snapshot import (+ HASH_DTYPE, SnapshotBlacklist, hash_domains, publish_snapshot,+)from mwmbl.indexer.purge_queue import drain_purge_queue, enqueue_for_purge, peek_purge_queue, queue_sizefrom mwmbl.tinysearchengine.indexer import Document@@ -222,3 +224,30 @@ def test_admin_index_links_to_the_status_page(staff_client):response = staff_client.get("/admin/")assert STATUS_URL in response.content.decode()+++# ---------------------------------------------------------------------------+# Curated domains+# ---------------------------------------------------------------------------++def test_curated_status_reports_nothing_still_blacklisted(wired_redis, monkeypatch):+ monkeypatch.setattr(admin_views, "get_curated_domains", lambda: {"pudding.cool"})+ publish_snapshot(np.array(hash_domains(["badsite.test"]), dtype=HASH_DTYPE).tobytes(), wired_redis)+ admin_views.get_snapshot_blacklist().load_now()++ status = admin_views._curated_status()++ assert status["count"] == 1+ assert status["still_blacklisted"] == []+++def test_curated_status_flags_a_domain_the_snapshot_still_blocks(wired_redis, monkeypatch):+ """Either the rebuild has not run yet, or a local rule curation cannot override is+ catching it. Both are invisible without this."""+ monkeypatch.setattr(admin_views, "get_curated_domains", lambda: {"pudding.cool"})+ publish_snapshot(np.array(hash_domains(["pudding.cool"]), dtype=HASH_DTYPE).tobytes(), wired_redis)+ admin_views.get_snapshot_blacklist().load_now()++ status = admin_views._curated_status()++ assert status["still_blacklisted"] == ["pudding.cool"]diff --git a/test/test_apps_background_scheduling.py b/test/test_apps_background_scheduling.pyindex 453a616..55e9d95 100644--- a/test/test_apps_background_scheduling.py+++ b/test/test_apps_background_scheduling.py@@ -83,3 +83,17 @@ def test_blacklist_tasks_repeat_at_the_configured_intervals():assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDSassert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDS+++def test_a_one_off_snapshot_rebuild_does_not_suppress_the_periodic_one():+ """Approving a domain submission schedules a one-off rebuild under the same task name+ (mwmbl.signals). One of those sitting in the queue at deploy time must not stop the+ six-hourly task being registered - that would silently leave the snapshot to whatever+ approvals happened to trigger."""+ from mwmbl.background import refresh_blacklist_snapshot+ refresh_blacklist_snapshot(schedule=600)++ MwmblConfig._schedule_background_tasks()++ assert Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, repeat__gt=0).count() == 1diff --git a/test/test_blacklist_approval_signal.py b/test/test_blacklist_approval_signal.pynew file mode 100644index 0000000..ca87d91--- /dev/null+++ b/test/test_blacklist_approval_signal.py@@ -0,0 +1,81 @@+"""Tests for the snapshot rebuild scheduled when a domain submission is approved.++Approved domains are subtracted from the remote blocklists when the snapshot is built, so+without this an approval would sit inert until the next periodic rebuild - six hours by+default. The scheduling is debounced because moderators work through the queue in batches.+"""+from datetime import timedelta++import pytest+from background_task.models import Task+from django.test import override_settings+from django.utils import timezone++from mwmbl.models import DomainSubmission, MwmblUser+from mwmbl.signals import BLACKLIST_SNAPSHOT_TASK++APPROVAL_DELAY = 600+++def user(db):+ return MwmblUser.objects.create(username="curator")+++def snapshot_tasks():+ return Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK)+++@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)+def test_approving_a_submission_schedules_a_rebuild(user):+ before = timezone.now()+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)++ task = snapshot_tasks().get()+ assert task.run_at >= before + timedelta(seconds=APPROVAL_DELAY - 5)+ assert task.repeat == 0+++@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)+def test_a_batch_of_approvals_schedules_one_rebuild(user):+ for i in range(10):+ DomainSubmission.objects.create(name=f"site{i}.example", status="APPROVED", submitted_by=user)++ assert snapshot_tasks().count() == 1+++@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)+def test_pending_and_rejected_submissions_schedule_nothing(user):+ DomainSubmission.objects.create(name="pending.example", status="PENDING", submitted_by=user)+ DomainSubmission.objects.create(name="rejected.example", status="REJECTED", submitted_by=user)++ assert snapshot_tasks().count() == 0+++@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)+def test_a_rebuild_already_due_inside_the_window_suppresses_a_new_one(user):+ """The periodic task's own row counts, so an approval shortly before the six-hourly+ rebuild adds nothing at all."""+ from mwmbl.background import refresh_blacklist_snapshot+ refresh_blacklist_snapshot(repeat=6 * 60 * 60, repeat_until=None)+ assert snapshot_tasks().count() == 1++ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)++ assert snapshot_tasks().count() == 1+++@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)+def test_a_rebuild_due_after_the_window_does_not_suppress_a_new_one(user):+ from mwmbl.background import refresh_blacklist_snapshot+ refresh_blacklist_snapshot(schedule=APPROVAL_DELAY * 10)+ assert snapshot_tasks().count() == 1++ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)++ assert snapshot_tasks().count() == 2diff --git a/test/test_blacklist_providers.py b/test/test_blacklist_providers.pyindex a17efb7..e44ec25 100644--- a/test/test_blacklist_providers.py+++ b/test/test_blacklist_providers.py@@ -15,6 +15,7 @@ from mwmbl.indexer.blacklist_providers import (AdultContentBlacklistProvider,CombinedBlacklistProvider,domain_and_parents,+ domains_to_unblock,)from mwmbl.indexer.blacklist import get_default_blacklist_provider@@ -225,3 +226,68 @@ def test_excluded_domains_cover_subdomains():assert provider.is_domain_blacklisted('fineartteens.com') is Trueassert provider.is_domain_blacklisted('www.fineartteens.com') is Trueassert provider.is_domain_blacklisted('otherfineartteens.com') is False+++def test_domains_to_unblock_adds_the_apex_for_www_submissions():+ """Submissions store whatever netloc was submitted, but the lists hold the apex."""+ assert domains_to_unblock({"www.contactmusic.com"}) == {"www.contactmusic.com", "contactmusic.com"}+ assert domains_to_unblock({"pudding.cool"}) == {"pudding.cool"}+ # Only the www. label is stripped: clearing en-academic.com off the strength of an+ # approved subdomain would unblock every other subdomain too.+ assert domains_to_unblock({"blog.example.com"}) == {"blog.example.com"}+++def test_curated_domains_override_the_providers():+ provider = CombinedBlacklistProvider(+ [StaticBlacklistProvider({"pudding.cool", "reallybad.example"})],+ get_exempt_domains=lambda: {"pudding.cool"},+ )++ assert provider.is_domain_blacklisted('pudding.cool') is False+ assert provider.is_domain_blacklisted('reallybad.example') is True+++def test_curated_domains_override_the_built_in_rules():+ """A moderator's approval beats the adult-content regex, which is what makes the+ approval flow usable for domains the lists get wrong."""+ provider = CombinedBlacklistProvider(+ [BuiltInRulesBlacklistProvider()],+ get_exempt_domains=lambda: {"adultswim.example"},+ )++ assert provider.is_domain_blacklisted('adultswim.example') is False+ assert provider.is_domain_blacklisted('otheradult.example') is True+++def test_curated_domains_match_either_www_form():+ provider = CombinedBlacklistProvider(+ [StaticBlacklistProvider({"contactmusic.com", "www.queer.example"})],+ get_exempt_domains=lambda: {"www.contactmusic.com", "queer.example"},+ )++ assert provider.is_domain_blacklisted('contactmusic.com') is False+ assert provider.is_domain_blacklisted('www.queer.example') is False+++def test_exempt_domains_are_resolved_once():+ """Resolving costs a database query or an HTTP fetch, so it happens once per provider,+ like the remote lists themselves."""+ calls = []++ def get_exempt_domains():+ calls.append(1)+ return {"pudding.cool"}++ provider = CombinedBlacklistProvider(+ [StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains)++ for _ in range(5):+ provider.is_domain_blacklisted('pudding.cool')++ assert len(calls) == 1+++def test_default_provider_takes_the_curated_domains_it_is_given():+ provider = get_default_blacklist_provider(lambda: {"badsite.example"})++ assert provider.get_exempt_domains() == {"badsite.example"}diff --git a/test/test_blacklist_snapshot.py b/test/test_blacklist_snapshot.pyindex 167e754..41a6552 100644--- a/test/test_blacklist_snapshot.py+++ b/test/test_blacklist_snapshot.py@@ -224,3 +224,48 @@ def test_a_truncated_blob_is_ignored_rather_than_raising(provider, redis_client)assert blacklist.load_now() is False# The good array it already had is kept rather than being replaced by a bad one.assert blacklist.is_domain_blacklisted("badsite.test") is True+++def test_build_snapshot_omits_curated_domains(monkeypatch):+ """An approved domain is removed from the lists when the snapshot is built, so no+ worker ever has to consult the curated set on the query path."""+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"evil.test"})+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"badsite.test", "evil.test"})])++ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)++ assert blacklist.filter_blacklisted(["badsite.test", "evil.test"]) == {"badsite.test"}+++def test_build_snapshot_clears_the_apex_for_a_www_approval(monkeypatch):+ """DomainSubmissionForm stores the submitted netloc, so an approval for+ www.contactmusic.com has to clear the apex entry the blocklist actually holds."""+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"www.contactmusic.test"})+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"contactmusic.test"})])++ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)++ assert blacklist.filter_blacklisted(["contactmusic.test", "www.contactmusic.test"]) == set()+++def test_build_snapshot_keeps_a_listed_parent_of_a_curated_subdomain(monkeypatch):+ """A flat hash set cannot express "this subdomain only", so curating a subdomain does+ not clear its listed parent - approving the apex is what unblocks it."""+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"blog.evil.test"})+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"evil.test"})])++ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)++ assert blacklist.filter_blacklisted(["blog.evil.test"]) == {"blog.evil.test"}+++def test_filter_blacklisted_handles_an_empty_snapshot():+ """A snapshot with every entry cleared is empty, not absent, and the vectorised search+ below needs an index 0 to clamp to."""+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))+ blacklist._array = np.array([], dtype=blacklist_snapshot.HASH_DTYPE)++ assert blacklist.filter_blacklisted(["anything.test"]) == set()diff --git a/test/test_curated_domains.py b/test/test_curated_domains.pynew file mode 100644index 0000000..0c43e6a--- /dev/null+++ b/test/test_curated_domains.py@@ -0,0 +1,53 @@+"""Tests for the approved-domain lookup that overrides the blacklists."""+from unittest.mock import patch++import pytest+from django.core.cache import cache+from django.test import override_settings++from mwmbl.curated_domains import CURATED_DOMAINS_CACHE_KEY, get_curated_domains+from mwmbl.models import DomainSubmission, MwmblUser++[email protected](autouse=True)+def clear_curated_cache():+ cache.delete(CURATED_DOMAINS_CACHE_KEY)+ yield+ cache.delete(CURATED_DOMAINS_CACHE_KEY)+++def test_no_database_returns_empty_without_querying():+ """The standalone crawler runs under settings_crawler, which has no database.++ It must not be possible for a blacklist construction there to reach for DomainSubmission+ - the query would raise, and the crawler fetches the same names over HTTP instead.+ """+ with patch("mwmbl.curated_domains.DomainSubmission.objects") as objects:+ with override_settings(HAS_DATABASE=False):+ assert get_curated_domains() == set()+ objects.filter.assert_not_called()+++@override_settings(HAS_DATABASE=True)+def test_returns_approved_submissions_only():+ user = MwmblUser.objects.create(username="curator")+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)+ DomainSubmission.objects.create(name="pending.example", status="PENDING", submitted_by=user)+ DomainSubmission.objects.create(name="rejected.example", status="REJECTED", submitted_by=user)++ assert get_curated_domains() == {"pudding.cool"}+++@override_settings(HAS_DATABASE=True)+def test_result_is_cached():+ user = MwmblUser.objects.create(username="curator")+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)+ assert get_curated_domains() == {"pudding.cool"}++ DomainSubmission.objects.create(name="later.example", status="APPROVED", submitted_by=user)+ assert get_curated_domains() == {"pudding.cool"}++ cache.delete(CURATED_DOMAINS_CACHE_KEY)+ assert get_curated_domains() == {"pudding.cool", "later.example"}diff --git a/test/test_update_urls.py b/test/test_update_urls.pyindex 47894c6..ce58630 100644--- a/test/test_update_urls.py+++ b/test/test_update_urls.py@@ -2,7 +2,7 @@ from collections import defaultdictfrom mwmbl.crawler.batch import Linkfrom mwmbl.indexer.update_urls import process_link-from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider+from mwmbl.indexer.blacklist_providers import CombinedBlacklistProvider, StaticBlacklistProviderdef test_process_link_normal():@@ -49,3 +49,29 @@ def test_process_link_excludes_porn():assert url_timestamps == {}assert url_users == {}assert domain_links == {}+++def test_process_link_keeps_a_curated_domain():+ """The blocklists carry false positives that are wrong for a search index, so a domain+ a moderator has approved must survive link discovery. This is the crawler's path: it+ takes the approved names from the URL queue, which fetches them over HTTP."""+ url_timestamps = {}+ url_users = {}+ domain_links = defaultdict(set)+ blacklist_provider = CombinedBlacklistProvider(+ [StaticBlacklistProvider({"pudding.cool"})],+ get_exempt_domains=lambda: {"pudding.cool"},+ )++ process_link(+ user_id_hash="abc123",+ crawled_page_domain="somewhere.com",+ link=Link(url="https://pudding.cool/2025/12/motifs", link_type="content"),+ timestamp=1234,+ url_timestamps=url_timestamps,+ url_users=url_users,+ blacklist_provider=blacklist_provider,+ domain_links=domain_links,+ )++ assert domain_links == {"somewhere.com": {"pudding.cool"}}diff --git a/test/test_url_database.py b/test/test_url_database.pyindex 9b99680..79da5af 100644--- a/test/test_url_database.py+++ b/test/test_url_database.py@@ -95,3 +95,12 @@ def test_crawled_url_is_not_requeued(clean_bloom_filters):assert url_queue.get_domain_count("crawled.example") == 0assert url_queue.get_domain_count("discovered.example") == 1+++def test_url_queue_gives_its_curated_domains_to_the_default_blacklist():+ """The standalone crawler has no database and fetches approved domains over HTTP, so+ the queue's own source of curated domains is what has to reach the blacklist."""+ redis = fakeredis.FakeStrictRedis(decode_responses=True)+ url_queue = RedisURLQueue(redis, lambda: {"pudding.cool"})++ assert url_queue.blacklist_provider.get_exempt_domains() == {"pudding.cool"}