Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Let curated domains override the remote blocklists

Daoud Clarke83cd13a6cb52
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.py
index 5303883..f84a86e 100644
--- a/mwmbl/admin_views.py
+++ b/mwmbl/admin_views.py
@@ -22,6 +22,7 @@ from django.shortcuts import render
from redis import RedisError
from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY
+from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer import blacklist_snapshot, purge_queue
from mwmbl.indexer.blacklist_snapshot import (
HASH_DTYPE, SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY, get_snapshot_blacklist,
@@ -100,6 +101,23 @@ def _removed_counts(days: int) -> list[dict]:
return [{"date": date, "count": int(count) if count else 0} for date, count in zip(dates, counts)]
+def _curated_status() -> dict:
+ """Approved domains, and whether any of them are still being filtered out.
+
+ Curated domains are subtracted from the remote lists when the snapshot is built, so
+ "still blacklisted" is normally zero. Anything listed here is either waiting for the
+ next rebuild, or blocked by a local rule curation does not override - EXCLUDED_DOMAINS
+ or DOMAIN_BLACKLIST_REGEX - which is otherwise invisible.
+ """
+ curated_domains = get_curated_domains()
+ still_blacklisted = get_snapshot_blacklist().filter_blacklisted(curated_domains)
+ return {
+ "count": len(curated_domains),
+ "still_blacklisted": sorted(still_blacklisted),
+ "approval_delay_seconds": settings.BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS,
+ }
+
+
def _task_status() -> list[dict]:
"""The two background tasks that maintain the state above.
@@ -137,6 +155,7 @@ def blacklist_status_view(request):
# handler covers the lot.
try:
context["snapshot"] = _snapshot_status()
+ context["curated"] = _curated_status()
context["queue"] = _queue_status()
context["removed_counts"] = _removed_counts(REMOVED_COUNT_DAYS)
except RedisError as e:
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index 524d2db..c51665f 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -42,6 +42,7 @@ class MwmblConfig(AppConfig):
create_index()
if settings.HAS_DATABASE:
create_index_db()
+ import mwmbl.signals # noqa: F401 - connects the post_save receivers
self._schedule_background_tasks()
# Cache key guarding the scheduling critical section below. Held just long
@@ -109,7 +110,11 @@ class MwmblConfig(AppConfig):
# on the task means the first run happens immediately rather than one full
# refresh interval after deploy, which matters because until it lands the
# search workers have only the built-in rules to go on.
- if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():
+ #
+ # Only the *repeating* row counts here: approving a domain submission schedules
+ # a one-off rebuild under the same task name (mwmbl.signals), and one of those
+ # sitting in the queue at deploy time must not suppress the periodic task.
+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, repeat__gt=0).exists():
refresh_blacklist_snapshot(
repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 3407052..836d8ff 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -255,7 +255,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
),
)
def get_curated_domains_endpoint(request) -> CuratedDomainsResponse:
- from mwmbl.search_setup import get_curated_domains
+ from mwmbl.curated_domains import get_curated_domains
return CuratedDomainsResponse(
domains=[CuratedDomain(name=d) for d in sorted(get_curated_domains())]
)
diff --git a/mwmbl/curated_domains.py b/mwmbl/curated_domains.py
new file mode 100644
index 0000000..a28593b
--- /dev/null
+++ b/mwmbl/curated_domains.py
@@ -0,0 +1,32 @@
+"""The set of domains a human moderator has approved.
+
+Approved DomainSubmissions are the override for the remote blocklists. Those lists are
+maintained for DNS ad-blocking and carry false positives that are wrong for a search index
+- pudding.cool, contactmusic.com and character.ai are all in The Block List Project's
+porn.txt - so an approval here removes the domain from the blacklist rather than merely
+outranking it.
+
+This lives outside search_setup because search_setup imports blacklist_snapshot, and the
+blacklist code needs to import this. The curated set is only ever read where a blacklist is
+*constructed* (build_snapshot, CombinedBlacklistProvider), never per query.
+"""
+from django.conf import settings
+from django.core.cache import cache
+
+from mwmbl.models import DomainSubmission
+
+CURATED_DOMAINS_CACHE_KEY = "curated-domains"
+CURATED_DOMAINS_CACHE_TIMEOUT = 300
+
+
+def get_curated_domains() -> set[str]:
+ # The standalone crawler runs django.setup() against settings_crawler, which has no
+ # database; it fetches the same names over HTTP instead, see crawl.py.
+ if not settings.HAS_DATABASE:
+ return set()
+
+ curated_domains = cache.get(CURATED_DOMAINS_CACHE_KEY)
+ if curated_domains is None:
+ curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))
+ cache.set(CURATED_DOMAINS_CACHE_KEY, curated_domains, CURATED_DOMAINS_CACHE_TIMEOUT)
+ return curated_domains
diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.py
index 08f4652..507c2bb 100644
--- a/mwmbl/indexer/blacklist.py
+++ b/mwmbl/indexer/blacklist.py
@@ -5,6 +5,9 @@ This module provides utility functions for creating default blacklist providers.
The main logic has been moved to blacklist_providers.py for better modularity.
"""
+from typing import Callable, Optional, Set
+
+from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer.blacklist_providers import (
BuiltInRulesBlacklistProvider,
HaGeZiBlacklistProvider,
@@ -14,10 +17,19 @@ from mwmbl.indexer.blacklist_providers import (
)
-def get_default_blacklist_provider() -> BlacklistProvider:
- """Get the default blacklist provider configuration."""
- return CombinedBlacklistProvider([
- BuiltInRulesBlacklistProvider(),
- HaGeZiBlacklistProvider('tif_medium'),
- AdultContentBlacklistProvider(),
- ])
+def get_default_blacklist_provider(
+ get_exempt_domains: Optional[Callable[[], Set[str]]] = None) -> BlacklistProvider:
+ """Get the default blacklist provider configuration.
+
+ Domains a moderator has approved override the lists. Callers without a database - the
+ standalone crawler - pass their own source of approved names; the default reads them
+ from DomainSubmission and returns nothing when there is no database.
+ """
+ return CombinedBlacklistProvider(
+ [
+ BuiltInRulesBlacklistProvider(),
+ HaGeZiBlacklistProvider('tif_medium'),
+ AdultContentBlacklistProvider(),
+ ],
+ get_exempt_domains=get_exempt_domains if get_exempt_domains is not None else get_curated_domains,
+ )
diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.py
index c4affff..1141948 100644
--- a/mwmbl/indexer/blacklist_providers.py
+++ b/mwmbl/indexer/blacklist_providers.py
@@ -6,7 +6,7 @@ making the system more flexible and testable.
"""
from abc import ABC, abstractmethod
-from typing import Optional, Set
+from typing import Callable, Optional, Set
import requests
@@ -32,6 +32,19 @@ def domain_and_parents(domain: str) -> list[str]:
return ['.'.join(parts[i:]) for i in range(len(parts) - 1)]
+def domains_to_unblock(curated_domains: Set[str]) -> Set[str]:
+ """The blocklist entries these curated domains should clear.
+
+ DomainSubmissionForm.clean_name stores whatever netloc was submitted, so an approval can
+ arrive as www.contactmusic.com while the blocklist holds the apex contactmusic.com.
+ Stripping the www. covers that; nothing deeper is stripped, because clearing an apex
+ entry on the strength of an approved subdomain would unblock every one of its siblings.
+ """
+ unblocked = set(curated_domains)
+ unblocked.update(domain[len("www."):] for domain in curated_domains if domain.startswith("www."))
+ return unblocked
+
+
class BlacklistProvider(ABC):
"""Abstract base class for blacklist providers."""
@@ -215,13 +228,33 @@ class AdultContentBlacklistProvider(RemoteListBlacklistProvider):
class CombinedBlacklistProvider(BlacklistProvider):
- """Provider that combines multiple blacklist providers."""
-
- def __init__(self, providers: list[BlacklistProvider]):
+ """Provider that combines multiple blacklist providers.
+
+ `get_exempt_domains` supplies the domains a moderator has approved, which override every
+ sub-provider. It is a callable rather than a set because the caller decides where the
+ approved names come from - the database in the server, an HTTP fetch in the standalone
+ crawler - and the result is resolved once and held, like the remote lists themselves.
+ """
+
+ def __init__(self, providers: list[BlacklistProvider],
+ get_exempt_domains: Optional[Callable[[], Set[str]]] = None):
self.providers = providers
-
+ self.get_exempt_domains = get_exempt_domains
+ self._exempt_domains: Optional[Set[str]] = None
+
+ def _is_exempt(self, domain: str) -> bool:
+ if self.get_exempt_domains is None:
+ return False
+ if self._exempt_domains is None:
+ self._exempt_domains = domains_to_unblock(self.get_exempt_domains())
+ apex = domain[len("www."):] if domain.startswith("www.") else domain
+ return domain in self._exempt_domains or apex in self._exempt_domains
+
def is_domain_blacklisted(self, domain: str) -> bool:
"""Check if domain is blacklisted by any provider."""
+ if self._is_exempt(domain):
+ return False
+
for provider in self.providers:
try:
if provider.is_domain_blacklisted(domain):
diff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.py
index fb709a4..6cf56d6 100644
--- a/mwmbl/indexer/blacklist_snapshot.py
+++ b/mwmbl/indexer/blacklist_snapshot.py
@@ -30,6 +30,15 @@ EXCLUDED_DOMAINS, the numeric-subdomain heuristic and the hn_top_domains trust e
is cheap and stays a local in-process check, so editing mwmbl/settings.py takes effect on
deploy instead of waiting for the next snapshot refresh. Evaluating them as
`built_in_rules OR snapshot` reproduces CombinedBlacklistProvider's semantics exactly.
+
+Curated domains - the ones a moderator has approved, see mwmbl.curated_domains - are
+subtracted here, at build time, rather than being checked per query. The remote lists are
+maintained for DNS ad-blocking and their false positives are wrong for a search index, so
+an approval has to remove the domain from the blacklist outright. Dropping the hash while
+the snapshot is built means the query path stays a single binary search with no database
+access, and every worker picks the exemption up with the next snapshot. The cost is that an
+approval only takes effect on the next rebuild, which is why approving a submission
+schedules one (mwmbl.signals).
"""
import hashlib
import threading
@@ -42,6 +51,7 @@ import numpy as np
import redis
from django.conf import settings
+from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer.blacklist import get_default_blacklist_provider
from mwmbl.indexer.blacklist_providers import (
BlacklistProvider,
@@ -49,6 +59,7 @@ from mwmbl.indexer.blacklist_providers import (
CombinedBlacklistProvider,
RemoteListBlacklistProvider,
domain_and_parents,
+ domains_to_unblock,
)
logger = getLogger(__name__)
@@ -107,12 +118,19 @@ def collect_remote_domains(provider: BlacklistProvider) -> set[str]:
def build_snapshot(provider: BlacklistProvider) -> bytes:
- """Download/parse the remote lists and return the sorted hash array as bytes."""
- domains = collect_remote_domains(provider)
+ """Download/parse the remote lists and return the sorted hash array as bytes.
+
+ Curated domains are removed from the listed set first - see the module docstring.
+ """
+ listed_domains = collect_remote_domains(provider)
+ unblocked = domains_to_unblock(get_curated_domains())
+ domains = listed_domains - unblocked
+ num_unblocked = len(listed_domains) - len(domains)
all_hashes = hash_domains(domains)
hashes = np.unique(all_hashes) # np.unique sorts, which is what we need
- logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB",
- len(domains), len(hashes), hashes.nbytes / 1e6)
+ logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB; "
+ "%d entries removed by curated domains",
+ len(domains), len(hashes), hashes.nbytes / 1e6, num_unblocked)
return hashes.astype(HASH_DTYPE).tobytes()
@@ -268,7 +286,10 @@ class SnapshotBlacklist:
blacklisted = {d for d in domains if self._built_in_rules.is_domain_blacklisted(d)}
array = self._array # read once: a background refresh may swap it mid-call
- if array is None:
+ if array is None or len(array) == 0:
+ # An empty array is not the same as no array, and it cannot go through the
+ # search below: clamping an out-of-range position to index 0 needs there to be
+ # an index 0. Nothing is a member of an empty snapshot anyway.
return blacklisted
remaining = [d for d in domains if d not in blacklisted]
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index bdd2125..aa06815 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -41,7 +41,9 @@ def run(batch_cache: BatchCache, new_item_queue: RedisURLQueue, num_batches: int
def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: RedisURLQueue):
start = datetime.utcnow()
- blacklist_provider = get_default_blacklist_provider()
+ # Take the approved domains from the queue rather than the database: the standalone
+ # crawler is the live caller of this, and it fetches them over HTTP.
+ blacklist_provider = get_default_blacklist_provider(new_item_queue.get_curated_domains_function)
blacklist_retrieval_time = datetime.utcnow() - start
logger.info(f"Recording URLs in database for {len(batches)} batches, blacklist provider ready "
f"in {blacklist_retrieval_time.total_seconds()} seconds")
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 6329f98..bb329f1 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -48,7 +48,7 @@ def run():
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.count_urls import count_urls_continuously
from mwmbl.indexer.update_urls import update_urls_continuously
- from mwmbl.search_setup import get_curated_domains
+ from mwmbl.curated_domains import get_curated_domains
if settings.STATIC_ROOT:
call_command("collectstatic", "--clear", "--noinput")
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.py
index c816875..db8b11d 100644
--- a/mwmbl/redis_url_queue.py
+++ b/mwmbl/redis_url_queue.py
@@ -55,7 +55,10 @@ class RedisURLQueue:
def __init__(self, redis: Redis, get_curated_domains_function: Callable[[], set[str]], blacklist_provider=None) -> None:
self.redis = redis
self.get_curated_domains_function = get_curated_domains_function
- self.blacklist_provider = blacklist_provider or get_default_blacklist_provider()
+ # Curated domains override the blacklist, and the queue already knows where to get
+ # them - over HTTP in the standalone crawler, which has no database.
+ self.blacklist_provider = blacklist_provider or get_default_blacklist_provider(
+ get_curated_domains_function)
def queue_urls(self, found_urls: list[FoundURL]):
curated_domains = self.get_curated_domains_function()
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 64b0c2c..96bc2a0 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -2,12 +2,11 @@ import os
from pathlib import Path
from django.conf import settings
-from django.core.cache import cache
from redis import Redis
+from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
-from mwmbl.models import DomainSubmission
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
@@ -16,18 +15,6 @@ from mwmbl.tinysearchengine.ltr_rank import LTRRanker
from mwmbl.tinysearchengine.mmr_rank import MMRRanker
from mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker
-CURATED_DOMAINS_CACHE_KEY = "curated-domains"
-CURATED_DOMAINS_CACHE_TIMEOUT = 300
-
-
-def get_curated_domains() -> set[str]:
- curated_domains = cache.get(CURATED_DOMAINS_CACHE_KEY)
- if curated_domains is None:
- curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))
- cache.set(CURATED_DOMAINS_CACHE_KEY, curated_domains, CURATED_DOMAINS_CACHE_TIMEOUT)
- return curated_domains
-
-
redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)
# Pull the blacklist snapshot in at worker startup rather than letting the first query
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index c6c9644..faa3566 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -332,3 +332,7 @@ BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis for
BLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote lists
BLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drained
BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge run
+# How far ahead approving a domain submission schedules a snapshot rebuild. Approvers work
+# through submissions in batches, so the delay collapses a batch into one rebuild rather
+# than one per approval - see mwmbl.signals.
+BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600
diff --git a/mwmbl/signals.py b/mwmbl/signals.py
new file mode 100644
index 0000000..7980e7a
--- /dev/null
+++ b/mwmbl/signals.py
@@ -0,0 +1,53 @@
+"""Signal receivers. Connected from MwmblConfig.ready().
+
+Only one so far: approving a domain submission has to reach the blacklist snapshot, because
+approved domains are subtracted from the remote lists when the snapshot is built rather than
+checked per query (see mwmbl.indexer.blacklist_snapshot). Without this an approval would sit
+inert for up to BLACKLIST_SNAPSHOT_REFRESH_SECONDS - six hours - and the moderator would
+reasonably conclude it had not worked.
+"""
+from datetime import timedelta
+from logging import getLogger
+
+from background_task.models import Task
+from django.conf import settings
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+from django.utils import timezone
+
+from mwmbl.models import DomainSubmission
+
+logger = getLogger(__name__)
+
+
+BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"
+
+
+@receiver(post_save, sender=DomainSubmission)
+def rebuild_blacklist_snapshot_on_approval(sender, instance: DomainSubmission, **kwargs):
+ """Schedule a snapshot rebuild when a submission is approved.
+
+ Debounced rather than immediate. Moderators work through the queue in batches, and each
+ rebuild downloads and parses tens of megabytes, so the run is scheduled
+ BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS ahead and only if nothing is already due by
+ then. The check matches the repeating task's row too, so an approval shortly before the
+ periodic rebuild adds nothing at all.
+
+ post_save carries no previous value, so this fires on every save of an approved
+ submission rather than only on the transition. The debounce absorbs that. Two approvals
+ saved at once could both schedule a run, which is harmless: publish_snapshot versions the
+ blob by content hash, so an identical snapshot does not make the workers re-read 11 MB.
+ """
+ if instance.status != "APPROVED":
+ return
+
+ from mwmbl.background import refresh_blacklist_snapshot
+
+ delay = settings.BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS
+ cutoff = timezone.now() + timedelta(seconds=delay)
+ if Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, run_at__lte=cutoff).exists():
+ return
+
+ refresh_blacklist_snapshot(schedule=delay)
+ logger.info("Scheduled a blacklist snapshot rebuild in %ds after approving %s",
+ delay, instance.name)
diff --git a/mwmbl/templates/admin/blacklist_status.html b/mwmbl/templates/admin/blacklist_status.html
index b2565cf..8a10368 100644
--- a/mwmbl/templates/admin/blacklist_status.html
+++ b/mwmbl/templates/admin/blacklist_status.html
@@ -105,6 +105,39 @@
</p>
</div>
+ <div class="module">
+ <h2>Curated domains</h2>
+ <table>
+ <tr>
+ <td class="key">Approved domains</td>
+ <td>{{ curated.count }}</td>
+ </tr>
+ <tr>
+ <td class="key">Still blacklisted</td>
+ <td>
+ {% if curated.still_blacklisted %}
+ <span class="bad">{{ curated.still_blacklisted|length }}</span>
+ &mdash;
+ {% for domain in curated.still_blacklisted %}
+ <code>{{ domain }}</code>{% if not forloop.last %}, {% endif %}
+ {% endfor %}
+ {% else %}
+ <span class="ok">none</span>
+ {% endif %}
+ </td>
+ </tr>
+ </table>
+ <p class="note">
+ Approved domain submissions are subtracted from the remote blocklists when the
+ snapshot is built, so approving a domain is how a false positive gets unblocked.
+ An approval schedules a rebuild {{ curated.approval_delay_seconds }}s later, which
+ collapses a batch of approvals into one rebuild. Domains listed above as still
+ blacklisted are either waiting for that rebuild, or blocked by a local rule that
+ curation does not override &mdash; EXCLUDED_DOMAINS or DOMAIN_BLACKLIST_REGEX in
+ mwmbl/settings.py.
+ </p>
+ </div>
+
<div class="module">
<h2>Purge queue</h2>
<table>
diff --git a/test/test_admin_blacklist_status.py b/test/test_admin_blacklist_status.py
index d079e61..79a0583 100644
--- a/test/test_admin_blacklist_status.py
+++ b/test/test_admin_blacklist_status.py
@@ -7,7 +7,9 @@ from redis import ConnectionError as RedisConnectionError
from mwmbl import admin_views
from mwmbl.indexer import blacklist_snapshot, purge_queue
-from mwmbl.indexer.blacklist_snapshot import HASH_DTYPE, SnapshotBlacklist, publish_snapshot
+from mwmbl.indexer.blacklist_snapshot import (
+ HASH_DTYPE, SnapshotBlacklist, hash_domains, publish_snapshot,
+)
from mwmbl.indexer.purge_queue import drain_purge_queue, enqueue_for_purge, peek_purge_queue, queue_size
from mwmbl.tinysearchengine.indexer import Document
@@ -222,3 +224,30 @@ def test_admin_index_links_to_the_status_page(staff_client):
response = staff_client.get("/admin/")
assert STATUS_URL in response.content.decode()
+
+
+# ---------------------------------------------------------------------------
+# Curated domains
+# ---------------------------------------------------------------------------
+
+def test_curated_status_reports_nothing_still_blacklisted(wired_redis, monkeypatch):
+ monkeypatch.setattr(admin_views, "get_curated_domains", lambda: {"pudding.cool"})
+ publish_snapshot(np.array(hash_domains(["badsite.test"]), dtype=HASH_DTYPE).tobytes(), wired_redis)
+ admin_views.get_snapshot_blacklist().load_now()
+
+ status = admin_views._curated_status()
+
+ assert status["count"] == 1
+ assert status["still_blacklisted"] == []
+
+
+def test_curated_status_flags_a_domain_the_snapshot_still_blocks(wired_redis, monkeypatch):
+ """Either the rebuild has not run yet, or a local rule curation cannot override is
+ catching it. Both are invisible without this."""
+ monkeypatch.setattr(admin_views, "get_curated_domains", lambda: {"pudding.cool"})
+ publish_snapshot(np.array(hash_domains(["pudding.cool"]), dtype=HASH_DTYPE).tobytes(), wired_redis)
+ admin_views.get_snapshot_blacklist().load_now()
+
+ status = admin_views._curated_status()
+
+ assert status["still_blacklisted"] == ["pudding.cool"]
diff --git a/test/test_apps_background_scheduling.py b/test/test_apps_background_scheduling.py
index 453a616..55e9d95 100644
--- a/test/test_apps_background_scheduling.py
+++ b/test/test_apps_background_scheduling.py
@@ -83,3 +83,17 @@ def test_blacklist_tasks_repeat_at_the_configured_intervals():
assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS
assert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDS
+
+
+def test_a_one_off_snapshot_rebuild_does_not_suppress_the_periodic_one():
+ """Approving a domain submission schedules a one-off rebuild under the same task name
+ (mwmbl.signals). One of those sitting in the queue at deploy time must not stop the
+ six-hourly task being registered - that would silently leave the snapshot to whatever
+ approvals happened to trigger."""
+ from mwmbl.background import refresh_blacklist_snapshot
+ refresh_blacklist_snapshot(schedule=600)
+
+ MwmblConfig._schedule_background_tasks()
+
+ assert Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK, repeat__gt=0).count() == 1
diff --git a/test/test_blacklist_approval_signal.py b/test/test_blacklist_approval_signal.py
new file mode 100644
index 0000000..ca87d91
--- /dev/null
+++ b/test/test_blacklist_approval_signal.py
@@ -0,0 +1,81 @@
+"""Tests for the snapshot rebuild scheduled when a domain submission is approved.
+
+Approved domains are subtracted from the remote blocklists when the snapshot is built, so
+without this an approval would sit inert until the next periodic rebuild - six hours by
+default. The scheduling is debounced because moderators work through the queue in batches.
+"""
+from datetime import timedelta
+
+import pytest
+from background_task.models import Task
+from django.test import override_settings
+from django.utils import timezone
+
+from mwmbl.models import DomainSubmission, MwmblUser
+from mwmbl.signals import BLACKLIST_SNAPSHOT_TASK
+
+APPROVAL_DELAY = 600
+
+
+def user(db):
+ return MwmblUser.objects.create(username="curator")
+
+
+def snapshot_tasks():
+ return Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK)
+
+
+@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)
+def test_approving_a_submission_schedules_a_rebuild(user):
+ before = timezone.now()
+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)
+
+ task = snapshot_tasks().get()
+ assert task.run_at >= before + timedelta(seconds=APPROVAL_DELAY - 5)
+ assert task.repeat == 0
+
+
+@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)
+def test_a_batch_of_approvals_schedules_one_rebuild(user):
+ for i in range(10):
+ DomainSubmission.objects.create(name=f"site{i}.example", status="APPROVED", submitted_by=user)
+
+ assert snapshot_tasks().count() == 1
+
+
+@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)
+def test_pending_and_rejected_submissions_schedule_nothing(user):
+ DomainSubmission.objects.create(name="pending.example", status="PENDING", submitted_by=user)
+ DomainSubmission.objects.create(name="rejected.example", status="REJECTED", submitted_by=user)
+
+ assert snapshot_tasks().count() == 0
+
+
+@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)
+def test_a_rebuild_already_due_inside_the_window_suppresses_a_new_one(user):
+ """The periodic task's own row counts, so an approval shortly before the six-hourly
+ rebuild adds nothing at all."""
+ from mwmbl.background import refresh_blacklist_snapshot
+ refresh_blacklist_snapshot(repeat=6 * 60 * 60, repeat_until=None)
+ assert snapshot_tasks().count() == 1
+
+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)
+
+ assert snapshot_tasks().count() == 1
+
+
+@override_settings(BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS=APPROVAL_DELAY)
+def test_a_rebuild_due_after_the_window_does_not_suppress_a_new_one(user):
+ from mwmbl.background import refresh_blacklist_snapshot
+ refresh_blacklist_snapshot(schedule=APPROVAL_DELAY * 10)
+ assert snapshot_tasks().count() == 1
+
+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)
+
+ assert snapshot_tasks().count() == 2
diff --git a/test/test_blacklist_providers.py b/test/test_blacklist_providers.py
index a17efb7..e44ec25 100644
--- a/test/test_blacklist_providers.py
+++ b/test/test_blacklist_providers.py
@@ -15,6 +15,7 @@ from mwmbl.indexer.blacklist_providers import (
AdultContentBlacklistProvider,
CombinedBlacklistProvider,
domain_and_parents,
+ domains_to_unblock,
)
from mwmbl.indexer.blacklist import get_default_blacklist_provider
@@ -225,3 +226,68 @@ def test_excluded_domains_cover_subdomains():
assert provider.is_domain_blacklisted('fineartteens.com') is True
assert provider.is_domain_blacklisted('www.fineartteens.com') is True
assert provider.is_domain_blacklisted('otherfineartteens.com') is False
+
+
+def test_domains_to_unblock_adds_the_apex_for_www_submissions():
+ """Submissions store whatever netloc was submitted, but the lists hold the apex."""
+ assert domains_to_unblock({"www.contactmusic.com"}) == {"www.contactmusic.com", "contactmusic.com"}
+ assert domains_to_unblock({"pudding.cool"}) == {"pudding.cool"}
+ # Only the www. label is stripped: clearing en-academic.com off the strength of an
+ # approved subdomain would unblock every other subdomain too.
+ assert domains_to_unblock({"blog.example.com"}) == {"blog.example.com"}
+
+
+def test_curated_domains_override_the_providers():
+ provider = CombinedBlacklistProvider(
+ [StaticBlacklistProvider({"pudding.cool", "reallybad.example"})],
+ get_exempt_domains=lambda: {"pudding.cool"},
+ )
+
+ assert provider.is_domain_blacklisted('pudding.cool') is False
+ assert provider.is_domain_blacklisted('reallybad.example') is True
+
+
+def test_curated_domains_override_the_built_in_rules():
+ """A moderator's approval beats the adult-content regex, which is what makes the
+ approval flow usable for domains the lists get wrong."""
+ provider = CombinedBlacklistProvider(
+ [BuiltInRulesBlacklistProvider()],
+ get_exempt_domains=lambda: {"adultswim.example"},
+ )
+
+ assert provider.is_domain_blacklisted('adultswim.example') is False
+ assert provider.is_domain_blacklisted('otheradult.example') is True
+
+
+def test_curated_domains_match_either_www_form():
+ provider = CombinedBlacklistProvider(
+ [StaticBlacklistProvider({"contactmusic.com", "www.queer.example"})],
+ get_exempt_domains=lambda: {"www.contactmusic.com", "queer.example"},
+ )
+
+ assert provider.is_domain_blacklisted('contactmusic.com') is False
+ assert provider.is_domain_blacklisted('www.queer.example') is False
+
+
+def test_exempt_domains_are_resolved_once():
+ """Resolving costs a database query or an HTTP fetch, so it happens once per provider,
+ like the remote lists themselves."""
+ calls = []
+
+ def get_exempt_domains():
+ calls.append(1)
+ return {"pudding.cool"}
+
+ provider = CombinedBlacklistProvider(
+ [StaticBlacklistProvider({"pudding.cool"})], get_exempt_domains=get_exempt_domains)
+
+ for _ in range(5):
+ provider.is_domain_blacklisted('pudding.cool')
+
+ assert len(calls) == 1
+
+
+def test_default_provider_takes_the_curated_domains_it_is_given():
+ provider = get_default_blacklist_provider(lambda: {"badsite.example"})
+
+ assert provider.get_exempt_domains() == {"badsite.example"}
diff --git a/test/test_blacklist_snapshot.py b/test/test_blacklist_snapshot.py
index 167e754..41a6552 100644
--- a/test/test_blacklist_snapshot.py
+++ b/test/test_blacklist_snapshot.py
@@ -224,3 +224,48 @@ def test_a_truncated_blob_is_ignored_rather_than_raising(provider, redis_client)
assert blacklist.load_now() is False
# The good array it already had is kept rather than being replaced by a bad one.
assert blacklist.is_domain_blacklisted("badsite.test") is True
+
+
+def test_build_snapshot_omits_curated_domains(monkeypatch):
+ """An approved domain is removed from the lists when the snapshot is built, so no
+ worker ever has to consult the curated set on the query path."""
+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"evil.test"})
+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"badsite.test", "evil.test"})])
+
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))
+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)
+
+ assert blacklist.filter_blacklisted(["badsite.test", "evil.test"]) == {"badsite.test"}
+
+
+def test_build_snapshot_clears_the_apex_for_a_www_approval(monkeypatch):
+ """DomainSubmissionForm stores the submitted netloc, so an approval for
+ www.contactmusic.com has to clear the apex entry the blocklist actually holds."""
+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"www.contactmusic.test"})
+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"contactmusic.test"})])
+
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))
+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)
+
+ assert blacklist.filter_blacklisted(["contactmusic.test", "www.contactmusic.test"]) == set()
+
+
+def test_build_snapshot_keeps_a_listed_parent_of_a_curated_subdomain(monkeypatch):
+ """A flat hash set cannot express "this subdomain only", so curating a subdomain does
+ not clear its listed parent - approving the apex is what unblocks it."""
+ monkeypatch.setattr(blacklist_snapshot, "get_curated_domains", lambda: {"blog.evil.test"})
+ provider = CombinedBlacklistProvider([FakeRemoteProvider({"evil.test"})])
+
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))
+ blacklist._array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)
+
+ assert blacklist.filter_blacklisted(["blog.evil.test"]) == {"blog.evil.test"}
+
+
+def test_filter_blacklisted_handles_an_empty_snapshot():
+ """A snapshot with every entry cleared is empty, not absent, and the vectorised search
+ below needs an index 0 to clamp to."""
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()))
+ blacklist._array = np.array([], dtype=blacklist_snapshot.HASH_DTYPE)
+
+ assert blacklist.filter_blacklisted(["anything.test"]) == set()
diff --git a/test/test_curated_domains.py b/test/test_curated_domains.py
new file mode 100644
index 0000000..0c43e6a
--- /dev/null
+++ b/test/test_curated_domains.py
@@ -0,0 +1,53 @@
+"""Tests for the approved-domain lookup that overrides the blacklists."""
+from unittest.mock import patch
+
+import pytest
+from django.core.cache import cache
+from django.test import override_settings
+
+from mwmbl.curated_domains import CURATED_DOMAINS_CACHE_KEY, get_curated_domains
+from mwmbl.models import DomainSubmission, MwmblUser
+
+
[email protected](autouse=True)
+def clear_curated_cache():
+ cache.delete(CURATED_DOMAINS_CACHE_KEY)
+ yield
+ cache.delete(CURATED_DOMAINS_CACHE_KEY)
+
+
+def test_no_database_returns_empty_without_querying():
+ """The standalone crawler runs under settings_crawler, which has no database.
+
+ It must not be possible for a blacklist construction there to reach for DomainSubmission
+ - the query would raise, and the crawler fetches the same names over HTTP instead.
+ """
+ with patch("mwmbl.curated_domains.DomainSubmission.objects") as objects:
+ with override_settings(HAS_DATABASE=False):
+ assert get_curated_domains() == set()
+ objects.filter.assert_not_called()
+
+
+@override_settings(HAS_DATABASE=True)
+def test_returns_approved_submissions_only():
+ user = MwmblUser.objects.create(username="curator")
+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)
+ DomainSubmission.objects.create(name="pending.example", status="PENDING", submitted_by=user)
+ DomainSubmission.objects.create(name="rejected.example", status="REJECTED", submitted_by=user)
+
+ assert get_curated_domains() == {"pudding.cool"}
+
+
+@override_settings(HAS_DATABASE=True)
+def test_result_is_cached():
+ user = MwmblUser.objects.create(username="curator")
+ DomainSubmission.objects.create(name="pudding.cool", status="APPROVED", submitted_by=user)
+ assert get_curated_domains() == {"pudding.cool"}
+
+ DomainSubmission.objects.create(name="later.example", status="APPROVED", submitted_by=user)
+ assert get_curated_domains() == {"pudding.cool"}
+
+ cache.delete(CURATED_DOMAINS_CACHE_KEY)
+ assert get_curated_domains() == {"pudding.cool", "later.example"}
diff --git a/test/test_update_urls.py b/test/test_update_urls.py
index 47894c6..ce58630 100644
--- a/test/test_update_urls.py
+++ b/test/test_update_urls.py
@@ -2,7 +2,7 @@ from collections import defaultdict
from mwmbl.crawler.batch import Link
from mwmbl.indexer.update_urls import process_link
-from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
+from mwmbl.indexer.blacklist_providers import CombinedBlacklistProvider, StaticBlacklistProvider
def test_process_link_normal():
@@ -49,3 +49,29 @@ def test_process_link_excludes_porn():
assert url_timestamps == {}
assert url_users == {}
assert domain_links == {}
+
+
+def test_process_link_keeps_a_curated_domain():
+ """The blocklists carry false positives that are wrong for a search index, so a domain
+ a moderator has approved must survive link discovery. This is the crawler's path: it
+ takes the approved names from the URL queue, which fetches them over HTTP."""
+ url_timestamps = {}
+ url_users = {}
+ domain_links = defaultdict(set)
+ blacklist_provider = CombinedBlacklistProvider(
+ [StaticBlacklistProvider({"pudding.cool"})],
+ get_exempt_domains=lambda: {"pudding.cool"},
+ )
+
+ process_link(
+ user_id_hash="abc123",
+ crawled_page_domain="somewhere.com",
+ link=Link(url="https://pudding.cool/2025/12/motifs", link_type="content"),
+ timestamp=1234,
+ url_timestamps=url_timestamps,
+ url_users=url_users,
+ blacklist_provider=blacklist_provider,
+ domain_links=domain_links,
+ )
+
+ assert domain_links == {"somewhere.com": {"pudding.cool"}}
diff --git a/test/test_url_database.py b/test/test_url_database.py
index 9b99680..79da5af 100644
--- a/test/test_url_database.py
+++ b/test/test_url_database.py
@@ -95,3 +95,12 @@ def test_crawled_url_is_not_requeued(clean_bloom_filters):
assert url_queue.get_domain_count("crawled.example") == 0
assert url_queue.get_domain_count("discovered.example") == 1
+
+
+def test_url_queue_gives_its_curated_domains_to_the_default_blacklist():
+ """The standalone crawler has no database and fetches approved domains over HTTP, so
+ the queue's own source of curated domains is what has to reach the blacklist."""
+ redis = fakeredis.FakeStrictRedis(decode_responses=True)
+ url_queue = RedisURLQueue(redis, lambda: {"pudding.cool"})
+
+ assert url_queue.blacklist_provider.get_exempt_domains() == {"pudding.cool"}