An open source, non-profit web search engine
Filter blacklisted domains at retrieval, then purge them in the background
Daoud Clarkece627e385f1d
diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex e5f1b07..60a17be 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -67,14 +67,31 @@ class MwmblConfig(AppConfig):try:from background_task.models import Task- from mwmbl.background import sync_search_counts+ from mwmbl.background import (+ purge_blacklisted_from_queue, refresh_blacklist_snapshot, sync_search_counts,+ )SYNC_TASK = "mwmbl.background.sync_search_counts"+ BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"+ BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"# Sync search counts once per hour (3600 seconds)if not Task.objects.filter(task_name=SYNC_TASK).exists():sync_search_counts(repeat=3600, repeat_until=None)+ # Rebuild the blacklist snapshot the search path filters against. schedule=0+ # on the task means the first run happens immediately rather than one full+ # refresh interval after deploy, which matters because until it lands the+ # search workers have only the built-in rules to go on.+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():+ refresh_blacklist_snapshot(+ repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)++ # Drain the queue of blacklisted documents that retrieval has filtered out+ if not Task.objects.filter(task_name=BLACKLIST_PURGE_TASK).exists():+ purge_blacklisted_from_queue(+ repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)+except Exception:# Don't prevent startup if background task scheduling failslog.exception("Failed to schedule background tasks")diff --git a/mwmbl/background.py b/mwmbl/background.pyindex 50cca81..062bb0d 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -1,8 +1,10 @@"""Script that updates data in a background process.-Also contains Django Background Tasks for periodic quota maintenance:+Also contains Django Background Tasks for periodic maintenance:- sync_search_counts: syncs Redis monthly counters → DB once per hour+ - refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against+ - purge_blacklisted_from_queue: removes retrieval-filtered documents from the index"""import loggingimport sys@@ -131,3 +133,48 @@ def sync_search_counts():logger.exception("Error syncing search count for key %s", key)+# ---------------------------------------------------------------------------+# Blacklisted-domain maintenance (Django Background Tasks)+# ---------------------------------------------------------------------------++@background(schedule=0)+def refresh_blacklist_snapshot():+ """+ Rebuild the blacklist snapshot that the search workers filter against.++ This is the only place the multi-megabyte remote blocklists are downloaded and+ parsed. Search workers only ever read the ~11 MB hash array this publishes to Redis,+ so no query ever waits on a blocklist fetch. See mwmbl.indexer.blacklist_snapshot.+ """+ from mwmbl.indexer.blacklist_snapshot import refresh_snapshot++ refresh_snapshot()+++@background(schedule=0)+def purge_blacklisted_from_queue():+ """+ Remove the documents that retrieval filtered out from the index itself.++ Retrieval already guarantees these are never shown; this closes the loop so the same+ documents are not re-filtered on every future query. The queue is best-effort, and+ that is fine: anything lost is re-queued the next time a query surfaces it.+ """+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist+ from mwmbl.indexer.purge_blacklisted import purge_documents+ from mwmbl.indexer.purge_queue import drain_purge_queue, queue_size+ from mwmbl.tinysearchengine.indexer import Document, TinyIndex++ documents = drain_purge_queue(settings.BLACKLIST_PURGE_BATCH_SIZE)+ if not documents:+ return++ blacklist = get_snapshot_blacklist()+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME+ with TinyIndex(Document, str(index_path), 'w') as index:+ removed_by_domain = purge_documents(index, documents, blacklist.is_domain_blacklisted)++ logger.info("Purged %d documents from the index across %d domains; %d still queued",+ sum(removed_by_domain.values()), len(removed_by_domain), queue_size())++diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.pyindex 5aa9c36..08f4652 100644--- a/mwmbl/indexer/blacklist.py+++ b/mwmbl/indexer/blacklist.py@@ -6,8 +6,9 @@ The main logic has been moved to blacklist_providers.py for better modularity."""from mwmbl.indexer.blacklist_providers import (- BuiltInRulesBlacklistProvider,- HaGeZiBlacklistProvider,+ BuiltInRulesBlacklistProvider,+ HaGeZiBlacklistProvider,+ AdultContentBlacklistProvider,CombinedBlacklistProvider,BlacklistProvider)@@ -17,5 +18,6 @@ def get_default_blacklist_provider() -> BlacklistProvider:"""Get the default blacklist provider configuration."""return CombinedBlacklistProvider([BuiltInRulesBlacklistProvider(),- HaGeZiBlacklistProvider('light')+ HaGeZiBlacklistProvider('tif_medium'),+ AdultContentBlacklistProvider(),])diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.pyindex 37075ee..aea61e6 100644--- a/mwmbl/indexer/blacklist_providers.py+++ b/mwmbl/indexer/blacklist_providers.py@@ -5,11 +5,16 @@ This module provides different ways to check if domains should be blacklisted,making the system more flexible and testable."""+import timefrom abc import ABC, abstractmethod-from datetime import timedeltafrom typing import Setimport requests-from mwmbl.utils import request_cache++# Parsed remote lists (~950k+ lines) are expensive to re-parse, and get_default_blacklist_provider()+# is now called on every index_documents() call (indexing needs to check the blacklist too, not just+# crawling), so the parsed result is cached here at module level - shared across every+# RemoteListBlacklistProvider instance, however many get constructed - keyed by URL.+_parsed_domains_cache: dict[str, tuple[float, Set[str]]] = {}class BlacklistProvider(ABC):@@ -57,73 +62,127 @@ class BuiltInRulesBlacklistProvider(BlacklistProvider):# Trusted domains are never blacklistedif domain in self.trusted_domains:return False-+# Spam detection heuristics for SEO spam domainsdomain_parts = domain.split('.')-- # Domains like: brofqpxj.uelinc.com, gzsmjc.fba01.com, 59648.etnomurcia.com- if (len(domain_parts) == 3 and- domain_parts[2] == "com" and- len(domain_parts[0]) in {6, 8}):- return True-- # Domains with all numeric first parts- if len(domain_parts) > 0 and set(domain_parts[0]) <= set("1234567890"):++ # Domains with all-numeric generated-looking subdomains, e.g. 59648.etnomurcia.com.+ # Restricted to len > 2 so this doesn't catch a numeric *apex* domain like 350.org,+ # where the numeric part is the actual site name rather than a generated subdomain.+ if len(domain_parts) > 2 and set(domain_parts[0]) <= set("1234567890"):return Truereturn False-class HaGeZiBlacklistProvider(BlacklistProvider):- """Provider that fetches HaGeZi blocklist."""-- # HaGeZi provides several lists, this is their main threat intelligence feeds- HAGEZI_URLS = {- 'light': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/light.txt',- 'normal': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/normal.txt',- 'pro': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/pro.txt',- 'ultimate': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/ultimate.txt',- }-- def __init__(self, list_type: str = 'light', cache_expire_days: int = 1):- if list_type not in self.HAGEZI_URLS:- raise ValueError(f"Invalid list_type. Must be one of: {list(self.HAGEZI_URLS.keys())}")-- self.url = self.HAGEZI_URLS[list_type]+class RemoteListBlacklistProvider(BlacklistProvider):+ """Provider that fetches a remote newline-delimited domain list, with caching.++ Handles both plain domain-per-line lists and hosts-file format+ (``0.0.0.0 domain.com``), which is what most public blocklists use.+ """++ def __init__(self, url: str, cache_expire_days: int = 1):+ self.url = urlself.cache_expire_days = cache_expire_days- self._cached_domains = None-+def _get_blacklisted_domains(self) -> Set[str]:- """Fetch HaGeZi blacklist with caching."""- if self._cached_domains is not None:- return self._cached_domains-- with request_cache(expire_after=timedelta(days=self.cache_expire_days)) as session:- try:- response = session.get(self.url)- response.raise_for_status()-- # Parse HaGeZi format - one domain per line, skip comments and empty lines- domains = set()- for line in response.text.split('\n'):- line = line.strip()- if line and not line.startswith('#'):- domains.add(line)-- self._cached_domains = domains+ cached = _parsed_domains_cache.get(self.url)+ if cached is not None:+ fetched_at, domains = cached+ if time.monotonic() - fetched_at < self.cache_expire_days * 86400:return domains- except requests.RequestException as e:- # Log the error but don't fail - return empty set as fallback- print(f"Warning: Failed to fetch HaGeZi blacklist from {self.url}: {e}")- self._cached_domains = set()- return set()-++ # Deliberately NOT using mwmbl.utils.request_cache here. That is a *filesystem*+ # cache rooted at settings.REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache" - the+ # same volume that holds the multi-hundred-GB index. These blocklists are tens of+ # megabytes each, so caching them there fills that volume, and once it is full+ # *every* user of request_cache breaks - including get_wiki_results(), which runs+ # on the normal search path and then retries a live fetch 4 times per uncached+ # query (WIKI_RETRY, 5s timeout each) until the worker is killed. A big periodic+ # download has no business in a shared small-response cache on the index volume;+ # the module-level parsed cache above already gives us reuse without touching disk.+ try:+ response = requests.get(self.url, timeout=60)+ response.raise_for_status()++ domains = set()+ for line in response.text.split('\n'):+ line = line.strip()+ if not line or line.startswith('#'):+ continue+ # Hosts format: "0.0.0.0 domain.com" or "127.0.0.1 domain.com"+ parts = line.split()+ domains.add(parts[1] if len(parts) == 2 and parts[0] in ('0.0.0.0', '127.0.0.1') else parts[0])++ _parsed_domains_cache[self.url] = (time.monotonic(), domains)+ return domains+ except requests.RequestException as e:+ # Log the error but don't fail. Fall back to the last known-good parsed set+ # rather than caching an empty one - a transient fetch error shouldn't silently+ # disable this provider's coverage until the next successful fetch.+ print(f"Warning: Failed to fetch blacklist from {self.url}: {e}")+ return cached[1] if cached is not None else set()+def is_domain_blacklisted(self, domain: str) -> bool:- """Check if domain is in the HaGeZi blacklist."""domains = self._get_blacklisted_domains()return domain in domains+class HaGeZiBlacklistProvider(RemoteListBlacklistProvider):+ """Provider that fetches a HaGeZi blocklist.++ NB: upstream restructured their repo from domains/{tier}.txt to+ wildcard/{tier}-onlydomains.txt at some point - the old domains/*.txt URLs now+ 404 silently (request_cache treats that as "list temporarily empty", not an+ error), so whichever tier was previously configured here had been contributing+ nothing.++ The 'light'/'normal'/'pro'/'ultimate' tiers are HaGeZi's general "Multi" lists:+ ads, affiliate, tracking, telemetry, and only *also* phishing/malware/scam mixed+ in. They're built for personal ad-blocking, not "should this be excluded from a+ search index" - they flag plenty of legitimate businesses purely for running+ analytics/trackers (e.g. baremetrics.com, covidtracking.com), which is a false+ positive for our purposes even though it's correct behaviour for a DNS ad-blocker.++ The 'tif'/'tif_medium'/'tif_mini' tiers are HaGeZi's dedicated Threat+ Intelligence Feed - malware, phishing, scam and C2 domains specifically, no+ general ad/tracker noise. That's the actually-relevant category here, and+ tif_medium (390k entries) tested clean against every legitimate domain in our+ Search Console sample. get_default_blacklist_provider() uses 'tif_medium'.+ """++ HAGEZI_URLS = {+ 'light': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt',+ 'normal': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/multi-onlydomains.txt',+ 'pro': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/pro-onlydomains.txt',+ 'ultimate': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt',+ 'tif': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif-onlydomains.txt',+ 'tif_medium': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.medium-onlydomains.txt',+ 'tif_mini': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.mini-onlydomains.txt',+ }++ def __init__(self, list_type: str = 'tif_medium', cache_expire_days: int = 1):+ if list_type not in self.HAGEZI_URLS:+ raise ValueError(f"Invalid list_type. Must be one of: {list(self.HAGEZI_URLS.keys())}")++ super().__init__(self.HAGEZI_URLS[list_type], cache_expire_days)+++class AdultContentBlacklistProvider(RemoteListBlacklistProvider):+ """Provider that fetches The Block List Project's maintained adult-content domain list.++ See https://github.com/blocklistproject/Lists - hosts-format, ~950k domains,+ updated regularly. Covers the general adult-content gap that the built-in+ regex/domain rules (aimed at spam and a handful of known-bad domains) don't.+ """++ URL = 'https://raw.githubusercontent.com/blocklistproject/Lists/master/porn.txt'++ def __init__(self, cache_expire_days: int = 7):+ super().__init__(self.URL, cache_expire_days)++class CombinedBlacklistProvider(BlacklistProvider):"""Provider that combines multiple blacklist providers."""diff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.pynew file mode 100644index 0000000..9ef7309--- /dev/null+++ b/mwmbl/indexer/blacklist_snapshot.py@@ -0,0 +1,269 @@+"""A blacklist representation the search path can afford to consult on every query.++The blacklist providers in blacklist_providers.py are built for the indexing path: the+remote-list ones download tens of megabytes and parse them into a Python set of ~1.3M+domain strings. That is fine for a batch job but wrong for search in two ways:++ * Memory. The combined HaGeZi tif_medium + Block List Project adult list is 1,338,063+ unique domains, which costs ~156 MB as a set[str] - and gunicorn runs+ (cpu_count * 2 + 1) workers, so on an 8-core box that would be ~2.6 GB of duplicated+ domain strings.+ * Latency. RemoteListBlacklistProvider fetches lazily, so the first query in each+ worker - and the first query after the module cache expires - would block on a+ multi-megabyte HTTP download with a 60 second timeout. That is exactly the failure+ mode that commit c28be4e was fixing.++So the query path never fetches, parses or holds the domain strings. A background task+(mwmbl.background.refresh_blacklist_snapshot) does that work once, hashes every domain to+a 64-bit integer, and publishes the sorted array to Redis as one ~11 MB blob. Workers+load the blob into a numpy array and answer membership with a vectorised binary search:+10.7 MB resident, and ~94 us to classify the ~200 distinct domains of a typical query -+noise next to the mmap page reads and the Wikipedia call the same request already makes.++Hashing rather than storing the domains is what buys the 15x memory saving. It costs+exactness in principle: two different domains could collide and a clean domain be treated+as blacklisted. With 64-bit hashes over 1.3M entries the chance any given lookup collides+is ~7e-14, i.e. it will not happen.++Only the *remote* lists go in the snapshot. BuiltInRulesBlacklistProvider (the regex,+EXCLUDED_DOMAINS, the numeric-subdomain heuristic and the hn_top_domains trust exemption)+is cheap and stays a local in-process check, so editing mwmbl/settings.py takes effect on+deploy instead of waiting for the next snapshot refresh. Evaluating them as+`built_in_rules OR snapshot` reproduces CombinedBlacklistProvider's semantics exactly.+"""+import hashlib+import threading+import time+from logging import getLogger+from typing import Iterable, Optional++import mmh3+import numpy as np+import redis+from django.conf import settings++from mwmbl.indexer.blacklist_providers import (+ BlacklistProvider,+ BuiltInRulesBlacklistProvider,+ CombinedBlacklistProvider,+ RemoteListBlacklistProvider,+)++logger = getLogger(__name__)+++SNAPSHOT_KEY = "blacklist:domain-hashes:v1"+SNAPSHOT_VERSION_KEY = "blacklist:domain-hashes:v1:version"++# Hashes are stored little-endian so the blob is portable between machines.+HASH_DTYPE = np.dtype(np.uint64).newbyteorder('<')+++_redis: Optional[redis.Redis] = None+++def get_redis() -> redis.Redis:+ """Binary-safe Redis connection - the snapshot blob is not text."""+ global _redis+ if _redis is None:+ _redis = redis.from_url(settings.REDIS_URL)+ return _redis+++def hash_domain(domain: str) -> int:+ return mmh3.hash64(domain, signed=False)[0]+++def hash_domains(domains: Iterable[str]) -> np.ndarray:+ return np.fromiter((hash_domain(d) for d in domains), dtype=HASH_DTYPE)+++# ---------------------------------------------------------------------------+# Write side - runs in the background task only+# ---------------------------------------------------------------------------++def collect_remote_domains(provider: BlacklistProvider) -> set[str]:+ """Every domain from the remote lists reachable from this provider.++ Walks CombinedBlacklistProvider recursively so callers can just hand over+ get_default_blacklist_provider() without knowing how it is composed. Providers that+ are not remote lists (the built-in rules) have no enumerable domain set and are+ skipped - they are evaluated locally instead, see the module docstring.+ """+ if isinstance(provider, CombinedBlacklistProvider):+ domains = set()+ for sub_provider in provider.providers:+ domains |= collect_remote_domains(sub_provider)+ return domains+ if isinstance(provider, RemoteListBlacklistProvider):+ return provider._get_blacklisted_domains()+ return set()+++def build_snapshot(provider: BlacklistProvider) -> bytes:+ """Download/parse the remote lists and return the sorted hash array as bytes."""+ domains = collect_remote_domains(provider)+ hashes = np.unique(hash_domains(domains)) # np.unique sorts, which is what we need+ logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB",+ len(domains), len(hashes), hashes.nbytes / 1e6)+ return hashes.astype(HASH_DTYPE).tobytes()+++def publish_snapshot(blob: bytes, redis_client: Optional[redis.Redis] = None) -> str:+ """Publish the blob and return its version.++ The version is a content hash, so a refresh that produces an identical snapshot does+ not make every worker re-download 11 MB. Blob first, then version: a worker that sees+ a new version is then guaranteed to find the matching blob.+ """+ redis_client = redis_client if redis_client is not None else get_redis()+ version = hashlib.sha256(blob).hexdigest()+ redis_client.set(SNAPSHOT_KEY, blob)+ redis_client.set(SNAPSHOT_VERSION_KEY, version)+ logger.info("Published blacklist snapshot version %s (%.1f MB)", version[:12], len(blob) / 1e6)+ return version+++def refresh_snapshot(provider: Optional[BlacklistProvider] = None,+ redis_client: Optional[redis.Redis] = None) -> str:+ if provider is None:+ from mwmbl.indexer.blacklist import get_default_blacklist_provider+ provider = get_default_blacklist_provider()+ return publish_snapshot(build_snapshot(provider), redis_client)+++# ---------------------------------------------------------------------------+# Read side - runs in every search worker+# ---------------------------------------------------------------------------++class SnapshotBlacklist:+ """Membership queries against the published snapshot, plus the local built-in rules.++ One instance per process. Nothing here ever blocks a request on the network: the+ snapshot is refreshed by a daemon thread, and the array is swapped in by plain+ attribute assignment (atomic in CPython), so an in-flight query simply finishes+ against the previous array.+ """++ def __init__(self, built_in_rules: Optional[BlacklistProvider] = None,+ redis_client: Optional[redis.Redis] = None):+ self._built_in_rules = built_in_rules if built_in_rules is not None else BuiltInRulesBlacklistProvider()+ self._redis = redis_client+ self._array: Optional[np.ndarray] = None+ self._version: Optional[str] = None+ self._last_checked: float = 0.0+ self._loading = False+ self._lock = threading.Lock()++ @property+ def loaded(self) -> bool:+ return self._array is not None++ def _get_redis(self) -> redis.Redis:+ return self._redis if self._redis is not None else get_redis()++ def load_now(self) -> bool:+ """Synchronously fetch the snapshot if its version has changed. Returns True if+ the in-memory array was replaced."""+ try:+ client = self._get_redis()+ version = client.get(SNAPSHOT_VERSION_KEY)+ except Exception:+ logger.warning("Could not read blacklist snapshot version from Redis", exc_info=True)+ return False++ if version is None:+ # Production Redis runs allkeys-lru, so the snapshot can be evicted. Keep+ # whatever we already have rather than falling back to an empty array, which+ # would silently disable filtering; the refresh task will republish it.+ logger.warning("Blacklist snapshot missing from Redis (evicted?); keeping the loaded copy")+ return False++ version = version.decode() if isinstance(version, bytes) else version+ if version == self._version:+ return False++ try:+ blob = client.get(SNAPSHOT_KEY)+ except Exception:+ logger.warning("Could not read blacklist snapshot from Redis", exc_info=True)+ return False++ if not blob:+ logger.warning("Blacklist snapshot version %s present but blob is missing", version[:12])+ return False++ array = np.frombuffer(blob, dtype=HASH_DTYPE)+ self._array = array+ self._version = version+ logger.info("Loaded blacklist snapshot version %s: %d domains", version[:12], len(array))+ return True++ def _maybe_refresh(self):+ """Kick off a refresh in the background if it has been long enough."""+ interval = getattr(settings, "BLACKLIST_SNAPSHOT_CHECK_SECONDS", 300)+ if time.monotonic() - self._last_checked < interval:+ return++ with self._lock:+ if time.monotonic() - self._last_checked < interval or self._loading:+ return+ self._last_checked = time.monotonic()+ self._loading = True++ if self._array is None:+ # Logged here rather than per query: this is once per check interval, which+ # is often enough to be noticed and rare enough not to drown the logs.+ logger.error("No blacklist snapshot has loaded; filtering on built-in rules only")++ def load():+ try:+ self.load_now()+ finally:+ self._loading = False++ threading.Thread(target=load, name="blacklist-snapshot-refresh", daemon=True).start()++ def filter_blacklisted(self, domains: Iterable[str]) -> set[str]:+ """Return the subset of these domains that are blacklisted."""+ domains = list(dict.fromkeys(domains)) # de-duplicate, keeping order for zip below+ if not domains:+ return set()++ self._maybe_refresh()++ blacklisted = {d for d in domains if self._built_in_rules.is_domain_blacklisted(d)}++ array = self._array # read once: a background refresh may swap it mid-call+ if array is None:+ return blacklisted++ remaining = [d for d in domains if d not in blacklisted]+ if not remaining:+ return blacklisted++ hashes = hash_domains(remaining)+ positions = np.searchsorted(array, hashes)+ # searchsorted can return len(array) for a hash above every entry; clamp so the+ # lookup below stays in bounds. Index 0 is always a safe stand-in because the+ # equality check is what decides membership.+ positions[positions >= len(array)] = 0+ found = array[positions] == hashes++ blacklisted.update(domain for domain, hit in zip(remaining, found) if hit)+ return blacklisted++ def is_domain_blacklisted(self, domain: str) -> bool:+ return domain in self.filter_blacklisted([domain])+++_snapshot_blacklist: Optional[SnapshotBlacklist] = None+++def get_snapshot_blacklist() -> SnapshotBlacklist:+ """The per-process singleton, shared by search_setup and the background tasks."""+ global _snapshot_blacklist+ if _snapshot_blacklist is None:+ _snapshot_blacklist = SnapshotBlacklist()+ _snapshot_blacklist.load_now()+ return _snapshot_blacklistdiff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.pyindex c952fa9..290f45e 100644--- a/mwmbl/indexer/index_batches.py+++ b/mwmbl/indexer/index_batches.py@@ -13,12 +13,13 @@ from mwmbl.crawler.batch import HashedBatch, Itemfrom mwmbl.crawler.urls import URLStatusfrom mwmbl.indexer import process_batchfrom mwmbl.indexer.batch_cache import BatchCache+from mwmbl.indexer.blacklist import get_default_blacklist_providerfrom mwmbl.indexer.index import tokenize_document, prepare_url_for_tokenizingfrom mwmbl.indexer.indexdb import BatchStatusfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex, DocumentState, CURATED_STATESfrom mwmbl.tinysearchengine.rank import score_result, DOCUMENT_FREQUENCIES, N_DOCUMENTS, HeuristicRankerfrom mwmbl.tokenizer import tokenize, get_bigrams-from mwmbl.utils import add_term_infos+from mwmbl.utils import add_term_infos, get_domainlogger = getLogger(__name__)@@ -69,12 +70,37 @@ def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Countdef index_documents(documents, index_path):+ """The common choke point every indexing path (offline batch processing, the+ trusted-crawler POST /results endpoint, the standalone crawl tool) goes through, so+ this is where the blacklist is enforced. Crawling/link-discovery also check the+ blacklist (RedisURLQueue, update_urls.process_link) but only to stop *new* crawling -+ a submitted batch or a direct /results submission can contain a blacklisted domain's+ pages regardless of whether that domain was ever handed out to be crawled (e.g. a+ browser-extension user organically visiting the site), so indexing needs its own check+ rather than relying on those upstream gates."""+ documents = filter_blacklisted_documents(documents)page_documents = preprocess_documents(documents, index_path)new_page_doc_counts = index_pages(index_path, page_documents)end_time = datetime.utcnow()return end_time, new_page_doc_counts+def filter_blacklisted_documents(documents: list[Document]) -> list[Document]:+ blacklist_provider = get_default_blacklist_provider()+ kept = []+ for document in documents:+ try:+ domain = get_domain(document.url)+ except ValueError:+ kept.append(document)+ continue+ if blacklist_provider.is_domain_blacklisted(domain):+ logger.info(f"Skipping indexing for blacklisted domain {domain}: {document.url}")+ continue+ kept.append(document)+ return kept++def index_pages(index_path: str, page_documents: dict[int, list[Document]], mark_synced: bool = False) -> Counter:term_new_doc_counts = Counter()with TinyIndex(Document, index_path, 'w') as indexer:diff --git a/mwmbl/indexer/purge_blacklisted.py b/mwmbl/indexer/purge_blacklisted.pynew file mode 100644index 0000000..a076418--- /dev/null+++ b/mwmbl/indexer/purge_blacklisted.py@@ -0,0 +1,80 @@+"""Removal of specific already-indexed documents from the index.++TinyIndex has no per-document delete: a page is a compressed blob of documents, so+removal means reading the page, dropping the document from the list, and writing the+whole page back. The only hard part is knowing *which* pages to rewrite.++A document is filed under every token that tokenize_document() derives from it - each+token hashes to one page - so a document typically lives on a dozen or more pages, and+removing it from just the page you happened to find it on leaves it retrievable by every+other term. tokenize_document() is a pure function of the url/title/extract/score stored+in the index, so recomputing it recovers that full token set exactly.+"""+from collections import defaultdict+from logging import getLogger+from typing import Callable, Iterable, Optional++from mwmbl.indexer.index import tokenize_document+from mwmbl.tinysearchengine.indexer import Document, TinyIndex+from mwmbl.utils import get_domain++logger = getLogger(__name__)+++def pages_for_document(index: TinyIndex, document: Document) -> set[int]:+ """Every page this document is filed under."""+ tokenized = tokenize_document(document.url, document.title, document.extract, document.score)+ return {index.get_key_page_index(token) for token in tokenized.tokens}+++def purge_documents(index: TinyIndex, documents: Iterable[Document],+ is_blacklisted: Optional[Callable[[str], bool]] = None) -> dict[str, int]:+ """Remove exactly these documents, by URL, from every page they are filed under.++ Returns a count of removed documents by domain.++ Removal is by URL membership, never by re-applying the blacklist to the whole page.+ Common tokens - "com", "you", "the" - hash to pages shared with thousands of+ unrelated documents, so re-testing the page would sweep away blacklisted documents+ that were never in the set we were asked to remove and that nobody has looked at.+ Those will come round again on their own once a query surfaces them.++ `is_blacklisted` re-checks each document before removal, so a domain that came off+ the blacklist between being queued and being purged is left alone. Pass None to skip+ the re-check and remove everything given.+ """+ target_urls = set()+ pages_to_urls: dict[int, set[str]] = defaultdict(set)+ for document in documents:+ if is_blacklisted is not None:+ try:+ domain = get_domain(document.url)+ except ValueError:+ continue+ if not is_blacklisted(domain):+ logger.info("Skipping purge of %s: no longer blacklisted", document.url)+ continue++ target_urls.add(document.url)+ for page_index in pages_for_document(index, document):+ pages_to_urls[page_index].add(document.url)++ removed_by_domain: dict[str, int] = defaultdict(int)+ for page_index, urls in pages_to_urls.items():+ page = index.get_page(page_index)+ kept = [d for d in page if d.url not in urls]+ if len(kept) == len(page):+ continue++ for document in page:+ if document.url in urls:+ try:+ removed_by_domain[get_domain(document.url)] += 1+ except ValueError:+ removed_by_domain[document.url] += 1+ index.store_in_page(page_index, kept)++ if removed_by_domain:+ logger.info("Purged %d documents across %d pages from %d URLs",+ sum(removed_by_domain.values()), len(pages_to_urls), len(target_urls))+ return dict(removed_by_domain)diff --git a/mwmbl/indexer/purge_queue.py b/mwmbl/indexer/purge_queue.pynew file mode 100644index 0000000..d6c390e--- /dev/null+++ b/mwmbl/indexer/purge_queue.py@@ -0,0 +1,128 @@+"""The hand-off between the search path and the background index purge.++When retrieval filters a blacklisted document out of a result set, that document is still+sitting in the index and will be re-filtered on every future query that touches its page.+The search worker cannot remove it there and then - that is a read-modify-write of an+index page - so it drops it on this queue and a background task+(mwmbl.background.purge_blacklisted_from_queue) does the removal.++Two properties matter:++ * Enqueueing must never be able to affect a search response. Every Redis call here is+ wrapped, and a failure is logged and swallowed. Losing an entry is harmless: the+ document is still filtered out of the results, and the next query that surfaces it+ enqueues it again. The whole loop is self-healing, which is also why it is safe that+ production Redis runs allkeys-lru and can evict this key outright.+ * It must not grow without bound. A Redis SET dedupes identical payloads for free -+ the same popular blacklisted document surfacing on a thousand queries is one entry -+ and MAX_QUEUE_SIZE caps the pathological case where the purge task is not running.++The payload carries exactly the four fields tokenize_document() needs to work out which+index pages a document is filed under. They are read straight out of the index, so the+serialised JSON is byte-identical between retrievals and SET dedupe works on it.+"""+import json+import threading+import time+from collections import OrderedDict+from logging import getLogger+from typing import Iterable, Optional++import redis+from django.conf import settings++from mwmbl.tinysearchengine.indexer import Document++logger = getLogger(__name__)+++PURGE_QUEUE_KEY = "blacklist:purge-queue"+MAX_QUEUE_SIZE = 100_000++# A blacklisted document stays in the index until the next purge run, so every query that+# surfaces it in the meantime tries to queue it again - two Redis round trips to re-add+# something already in the SET. This remembers what this process has queued recently and+# skips those, which matters most for exactly the documents that matter most: the ones+# turning up on popular queries. The TTL means a document whose purge failed or was lost+# is retried rather than suppressed forever.+_RECENTLY_ENQUEUED_TTL = 30 * 60+_MAX_RECENTLY_ENQUEUED = 10_000+_recently_enqueued: "OrderedDict[str, float]" = OrderedDict()+_recently_enqueued_lock = threading.Lock()+++def _take_unqueued(payloads: list[str]) -> list[str]:+ """Filter out payloads this process queued recently, and record the rest."""+ now = time.monotonic()+ with _recently_enqueued_lock:+ while _recently_enqueued:+ payload, queued_at = next(iter(_recently_enqueued.items()))+ if now - queued_at < _RECENTLY_ENQUEUED_TTL:+ break+ _recently_enqueued.popitem(last=False)++ fresh = [p for p in payloads if p not in _recently_enqueued]+ for payload in fresh:+ _recently_enqueued[payload] = now+ while len(_recently_enqueued) > _MAX_RECENTLY_ENQUEUED:+ _recently_enqueued.popitem(last=False)+ return fresh+++_redis: Optional[redis.Redis] = None+++def get_redis() -> redis.Redis:+ global _redis+ if _redis is None:+ _redis = redis.from_url(settings.REDIS_URL, decode_responses=True)+ return _redis+++def _payload(document: Document) -> str:+ return json.dumps({+ "url": document.url,+ "title": document.title,+ "extract": document.extract,+ "score": document.score,+ }, sort_keys=True)+++def enqueue_for_purge(documents: Iterable[Document], redis_client: Optional[redis.Redis] = None) -> int:+ """Queue these documents for removal from the index. Returns the number of *new*+ entries added (already-queued duplicates do not count). Never raises."""+ payloads = _take_unqueued([_payload(document) for document in documents])+ if not payloads:+ return 0++ try:+ client = redis_client if redis_client is not None else get_redis()+ if client.scard(PURGE_QUEUE_KEY) >= MAX_QUEUE_SIZE:+ logger.warning("Blacklist purge queue is full (%d); dropping %d documents",+ MAX_QUEUE_SIZE, len(payloads))+ return 0+ return client.sadd(PURGE_QUEUE_KEY, *payloads)+ except Exception:+ logger.warning("Could not enqueue %d documents for purging", len(payloads), exc_info=True)+ return 0+++def drain_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) -> list[Document]:+ """Remove and return up to `limit` queued documents."""+ client = redis_client if redis_client is not None else get_redis()+ payloads = client.spop(PURGE_QUEUE_KEY, limit) or []++ documents = []+ for payload in payloads:+ try:+ fields = json.loads(payload)+ documents.append(Document(title=fields["title"], url=fields["url"],+ extract=fields["extract"], score=fields["score"]))+ except (ValueError, KeyError, TypeError):+ logger.warning("Discarding unreadable purge queue entry: %r", payload)+ return documents+++def queue_size(redis_client: Optional[redis.Redis] = None) -> int:+ client = redis_client if redis_client is not None else get_redis()+ return client.scard(PURGE_QUEUE_KEY)diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 7a6786c..edad557 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -6,6 +6,7 @@ from django.core.cache import cachefrom redis import Redisfrom mwmbl.indexer.batch_cache import BatchCache+from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklistfrom mwmbl.models import DomainSubmissionfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completer@@ -34,6 +35,10 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAMEtiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()+# Pull the blacklist snapshot in at worker startup rather than letting the first query+# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.+get_snapshot_blacklist()+ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))# Diversity is applied by the wrapping MMRRanker, which demotes (rather than drops)# same-domain / near-duplicate results. Unwrap to disable diversity.diff --git a/mwmbl/settings.py b/mwmbl/settings.pyindex 018ba57..fd777c8 100644--- a/mwmbl/settings.py+++ b/mwmbl/settings.py@@ -31,8 +31,17 @@ SCORE_FOR_DIFFERENT_DOMAIN = 1.0SCORE_FOR_SAME_DOMAIN = 0.01EXTRA_LINK_MULTIPLIER = 0.001UNKNOWN_DOMAIN_MULTIPLIER = 0.001-EXCLUDED_DOMAINS = {'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com', 'www.lukas-renggli.ch'}-DOMAIN_BLACKLIST_REGEX = re.compile(r"porn|xxx|adult|jksu\.org|lwhyl\.org$|rgcd\.cn$|hzqwyou\.cn$|omgoat\.org$|pussyboy\.net$")+EXCLUDED_DOMAINS = {+ 'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com', 'www.lukas-renggli.ch',+ # Adult content sites found ranking for suspicious Search Console queries (2026-08-14).+ 'fineartteens.com', 'milforia.com', 'azgals.com', 'kusowanka.com',+}+# `booru.org` is a free hosting platform for imageboards; every subdomain we've found ranking+# (nyou/hypno/inflate/futaba/captions.booru.org) hosts adult/fetish content, so block the whole domain.+DOMAIN_BLACKLIST_REGEX = re.compile(+ r"porn|xxx|adult|hentai|\.booru\.org$|"+ r"jksu\.org|lwhyl\.org$|rgcd\.cn$|hzqwyou\.cn$|omgoat\.org$|pussyboy\.net$"+)CORE_DOMAINS = {'github.com','en.wikipedia.org',diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex b13d15e..f484188 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -313,3 +313,12 @@ SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rateSUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrainSUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrainsSOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages++# Blacklisted-domain filtering on the search path. Set BLACKLIST_FILTER_AT_RETRIEVAL to+# False to turn retrieval-time filtering off without a rollback; the index-time filter in+# index_documents() is unaffected.+BLACKLIST_FILTER_AT_RETRIEVAL = os.environ.get("BLACKLIST_FILTER_AT_RETRIEVAL", "true").lower() != "false"+BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis for a new snapshot+BLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote lists+BLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drained+BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge rundiff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 81a58d9..b800c98 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -12,6 +12,7 @@ from pathlib import Pathfrom urllib.parse import urlparseimport numpy as np+from django.conf import settingsfrom requests.adapters import HTTPAdapter, Retryfrom mwmbl.format import get_query_regex@@ -19,7 +20,7 @@ from mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentStatefrom mwmbl.tokenizer import tokenize, get_bigrams-from mwmbl.utils import request_cache+from mwmbl.utils import get_domain, request_cachelogger = getLogger(__name__)@@ -207,6 +208,34 @@ def get_wiki_score(url):return WIKI_SCORES.get(title, 0.0) / WIKI_MAX_SCORE+def find_blacklisted_urls(documents: list[Document]) -> set[str]:+ """URLs among these documents whose domain is blacklisted.++ Enforcing the blacklist here, on the read path, is what actually guarantees a+ blacklisted domain is never shown. The write-path checks (crawl assignment, link+ discovery, index_documents) only stop *new* content: they do nothing about the+ documents already in the index from before a domain was blacklisted, and the index is+ far too large to sweep on demand. Filtering at retrieval makes a newly blacklisted+ domain disappear from results immediately, and the purge queue then cleans up the+ index in the background so the work is not repeated forever.++ Cost is ~94us for a typical query's ~200 distinct domains - see blacklist_snapshot.+ """+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist++ domains_to_urls = defaultdict(set)+ for document in documents:+ try:+ domain = get_domain(document.url)+ except ValueError:+ # Unparseable URL - keep it, matching index_batches.filter_blacklisted_documents.+ continue+ domains_to_urls[domain].add(document.url)++ blacklisted_domains = get_snapshot_blacklist().filter_blacklisted(domains_to_urls)+ return {url for domain in blacklisted_domains for url in domains_to_urls[domain]}++def deduplicate(results, seen_titles):deduplicated_results = []for result in results:@@ -315,18 +344,53 @@ class Ranker:pages += itemsexternal_search_items = self.external_search(q)- ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)+ candidates = pages + additional_results + external_search_items+ candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)++ ordered_results = self.order_results(terms, candidates, is_complete)deduplicated_results = deduplicate(curated_items + ordered_results, set())state_fixed = [fix_document_state(result) for result in deduplicated_results]return state_fixed, terms, completions+ @staticmethod+ def _remove_blacklisted(candidates: list[Document], curated_items: list[Document],+ index_items: list[Document]) -> tuple[list[Document], list[Document]]:+ """Drop blacklisted documents from a result set and queue the indexed ones for purging.++ Curated items are filtered separately because they bypass order_results entirely -+ they are prepended straight into deduplicate() - so filtering only `candidates`+ would leave a hole for any blacklisted domain that had been curated for a term.++ Only index-sourced documents go on the purge queue: there is nothing to purge for+ a Wikipedia or Super Search result, and nothing to purge for the caller-supplied+ additional_results either.+ """+ if not getattr(settings, "BLACKLIST_FILTER_AT_RETRIEVAL", True):+ return candidates, curated_items++ blacklisted_urls = find_blacklisted_urls(candidates + curated_items)+ if not blacklisted_urls:+ return candidates, curated_items++ logger.info("Filtered %d blacklisted documents from results", len(blacklisted_urls))++ from mwmbl.indexer.purge_queue import enqueue_for_purge+ enqueue_for_purge([d for d in index_items if d.url in blacklisted_urls])++ return ([d for d in candidates if d.url not in blacklisted_urls],+ [d for d in curated_items if d.url not in blacklisted_urls])+def external_search(self, q: str):return []def get_raw_results(self, query: str):tokens = tokenize(query)term = ' '.join(tokens)- return self.tiny_index.retrieve(term)+ results = self.tiny_index.retrieve(term)+ # /raw bypasses get_results(), so it needs its own filter or it is a way to read+ # blacklisted documents straight out of the index.+ filtered, _ = self._remove_blacklisted(results, [], index_items=results)+ return filteredclass HeuristicRanker(Ranker):diff --git a/test/test_apps_background_scheduling.py b/test/test_apps_background_scheduling.pynew file mode 100644index 0000000..62af95f--- /dev/null+++ b/test/test_apps_background_scheduling.py@@ -0,0 +1,39 @@+"""The periodic tasks only exist if apps.ready() registers them, and registering the same+task twice means it runs twice as often on every deploy."""+import pytest+from background_task.models import Task+from django.conf import settings++from mwmbl.apps import MwmblConfig++BLACKLIST_TASKS = {+ "mwmbl.background.refresh_blacklist_snapshot",+ "mwmbl.background.purge_blacklisted_from_queue",+}+++def test_blacklist_tasks_are_scheduled():+ MwmblConfig._schedule_background_tasks()++ assert BLACKLIST_TASKS <= set(Task.objects.values_list("task_name", flat=True))+++def test_scheduling_twice_does_not_duplicate_tasks():+ MwmblConfig._schedule_background_tasks()+ MwmblConfig._schedule_background_tasks()++ for task_name in BLACKLIST_TASKS:+ assert Task.objects.filter(task_name=task_name).count() == 1+++def test_blacklist_tasks_repeat_at_the_configured_intervals():+ MwmblConfig._schedule_background_tasks()++ snapshot = Task.objects.get(task_name="mwmbl.background.refresh_blacklist_snapshot")+ purge = Task.objects.get(task_name="mwmbl.background.purge_blacklisted_from_queue")++ assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS+ assert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDSdiff --git a/test/test_blacklist.py b/test/test_blacklist.pyindex 5088b57..9225c22 100644--- a/test/test_blacklist.py+++ b/test/test_blacklist.py@@ -5,8 +5,6 @@ def test_blacklist_excludes_bad_pattern():"""Test that built-in rules blacklist bad patterns."""provider = BuiltInRulesBlacklistProvider()bad_domains = [- "brofqpxj.uelinc.com",- "gzsmjc.fba01.com","59648.etnomurcia.com","something.hzqwyou.cn",]@@ -15,6 +13,23 @@ def test_blacklist_excludes_bad_pattern():assert provider.is_domain_blacklisted(domain)+def test_blacklist_allows_numeric_apex_domain():+ """A numeric *apex* domain like 350.org is the actual site name, not a generated+ subdomain - the numeric-subdomain heuristic below should only catch len > 2."""+ provider = BuiltInRulesBlacklistProvider()+ assert not provider.is_domain_blacklisted("350.org")+++def test_blacklist_allows_ordinary_short_subdomains():+ """Regression test: a length-based heuristic that used to flag any 3-part .com+ domain with a 6 or 8 character first label was removed after it flagged dozens of+ legitimate subdomains (topics.nytimes.com, search.google.com, podcasts.apple.com,+ starwars.fandom.com, ...) with zero confirmed true positives in testing."""+ provider = BuiltInRulesBlacklistProvider()+ for domain in ["topics.nytimes.com", "search.google.com", "podcasts.apple.com", "starwars.fandom.com"]:+ assert not provider.is_domain_blacklisted(domain), domain++def test_blacklist_allows_top_domains():"""Test that built-in rules allow legitimate domains."""provider = BuiltInRulesBlacklistProvider()@@ -25,3 +40,27 @@ def test_blacklist_allows_other_domains():"""Test that built-in rules allow other legitimate domains."""provider = BuiltInRulesBlacklistProvider()assert not provider.is_domain_blacklisted("something.com")+++def test_blacklist_excludes_adult_sites_found_via_search_console():+ """Domains surfaced by suspicious Search Console queries (2026-08-14)."""+ provider = BuiltInRulesBlacklistProvider()+ bad_domains = [+ "fineartteens.com",+ "milforia.com",+ "azgals.com",+ "kusowanka.com",+ "nyou.booru.org",+ "hypno.booru.org",+ "inflatebooru.booru.org",+ "futabooru.booru.org",+ "captions.booru.org",+ ]+ for domain in bad_domains:+ assert provider.is_domain_blacklisted(domain), domain+++def test_blacklist_allows_legitimate_imageboard():+ """mathchan.org ('the scientific imageboard') is not adult content."""+ provider = BuiltInRulesBlacklistProvider()+ assert not provider.is_domain_blacklisted("mathchan.org")diff --git a/test/test_blacklist_providers.py b/test/test_blacklist_providers.pyindex e0284ca..a19dea7 100644--- a/test/test_blacklist_providers.py+++ b/test/test_blacklist_providers.py@@ -3,16 +3,30 @@ Tests for blacklist providers and the abstraction system."""import pytest+import requestsfrom unittest.mock import patch, MagicMock+from mwmbl.indexer import blacklist_providersfrom mwmbl.indexer.blacklist_providers import (- StaticBlacklistProvider,- HaGeZiBlacklistProvider,+ StaticBlacklistProvider,+ HaGeZiBlacklistProvider,+ AdultContentBlacklistProvider,CombinedBlacklistProvider)from mwmbl.indexer.blacklist import get_default_blacklist_provider[email protected](autouse=True)+def clear_shared_domains_cache():+ """_parsed_domains_cache is a module-level dict shared across every+ RemoteListBlacklistProvider instance (see blacklist_providers.py) so that+ index_documents() can call get_default_blacklist_provider() cheaply on every call.+ Clear it around each test so a mocked response in one test can't leak into another."""+ blacklist_providers._parsed_domains_cache.clear()+ yield+ blacklist_providers._parsed_domains_cache.clear()++def test_static_blacklist_provider():"""Test StaticBlacklistProvider with is_domain_blacklisted method."""test_domains = {'spam.com', 'malware.example'}@@ -28,8 +42,7 @@ def test_static_blacklist_provider():def test_hagezi_blacklist_provider_success():"""Test HaGeZiBlacklistProvider with successful HTTP response."""- with patch('mwmbl.indexer.blacklist_providers.request_cache') as mock_cache:- mock_session = MagicMock()+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:mock_response = MagicMock()mock_response.text = '''# HaGeZi DNS Blocklist# Comments should be ignored@@ -39,11 +52,10 @@ malware.example# Another commentbadsite.net'''- mock_session.get.return_value = mock_response- mock_cache.return_value.__enter__.return_value = mock_session-+ mock_get.return_value = mock_response+provider = HaGeZiBlacklistProvider('light')-+# Test domains that should be blacklistedassert provider.is_domain_blacklisted('spam.com') == Trueassert provider.is_domain_blacklisted('malware.example') == True@@ -53,6 +65,25 @@ badsite.netassert provider.is_domain_blacklisted('github.com') == False+def test_adult_content_blacklist_provider_success():+ """Test AdultContentBlacklistProvider parses hosts-format lines correctly."""+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ mock_response = MagicMock()+ mock_response.text = '''# Title: Porn Block List+# Homepage: https://github.com/blocklistproject/Lists++0.0.0.0 fineartteens.com+0.0.0.0 milforia.com+'''+ mock_get.return_value = mock_response++ provider = AdultContentBlacklistProvider()++ assert provider.is_domain_blacklisted('fineartteens.com') == True+ assert provider.is_domain_blacklisted('milforia.com') == True+ assert provider.is_domain_blacklisted('github.com') == False++def test_hagezi_blacklist_provider_invalid_type():"""Test HaGeZiBlacklistProvider rejects invalid list types."""with pytest.raises(ValueError):@@ -100,3 +131,43 @@ def test_integration_with_blacklist_module():# Test with a domain that should not be blacklistedassert default_provider.is_domain_blacklisted('github.com') == False+++def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():+ """These lists are tens of megabytes. mwmbl.utils.request_cache is a *filesystem* cache+ rooted at settings.REQUEST_CACHE_PATH, on the same volume as the index - caching them+ there fills that volume, and once it is full every other user of request_cache breaks,+ including get_wiki_results() on the live search path (which then retries a live fetch+ 4x per uncached query until the worker dies). Regression test: fetch must not go+ through request_cache."""+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ mock_response = MagicMock()+ mock_response.text = "spam.com\n"+ mock_get.return_value = mock_response++ # If the module still imported request_cache, patching it would succeed; assert the+ # name is gone from the module so this can't silently regress.+ import mwmbl.indexer.blacklist_providers as bp+ assert not hasattr(bp, "request_cache"), \+ "blacklist_providers must not use the shared filesystem request_cache"++ provider = HaGeZiBlacklistProvider('light')+ assert provider.is_domain_blacklisted('spam.com') is True+ assert mock_get.called+++def test_fetch_failure_falls_back_to_last_good_list():+ """A transient fetch failure must not silently blank out the provider's coverage."""+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:+ mock_response = MagicMock()+ mock_response.text = "spam.com\n"+ mock_get.return_value = mock_response+ provider = HaGeZiBlacklistProvider('light')+ assert provider.is_domain_blacklisted('spam.com') is True++ # Expire the module-level cache so the next call refetches, and make that fetch fail.+ blacklist_providers._parsed_domains_cache[provider.url] = (+ -10**9, blacklist_providers._parsed_domains_cache[provider.url][1])+ with patch('mwmbl.indexer.blacklist_providers.requests.get',+ side_effect=requests.RequestException("boom")):+ assert provider.is_domain_blacklisted('spam.com') is True # still covereddiff --git a/test/test_blacklist_snapshot.py b/test/test_blacklist_snapshot.pynew file mode 100644index 0000000..ca7fd3c--- /dev/null+++ b/test/test_blacklist_snapshot.py@@ -0,0 +1,163 @@+import fakeredis+import numpy as np+import pytest++from mwmbl.indexer import blacklist_snapshot+from mwmbl.indexer.blacklist_snapshot import (+ SNAPSHOT_KEY,+ SNAPSHOT_VERSION_KEY,+ SnapshotBlacklist,+ build_snapshot,+ collect_remote_domains,+ publish_snapshot,+ refresh_snapshot,+)+from mwmbl.indexer.blacklist_providers import (+ CombinedBlacklistProvider,+ RemoteListBlacklistProvider,+ StaticBlacklistProvider,+)+++class FakeRemoteProvider(RemoteListBlacklistProvider):+ """A remote list provider that serves a fixed set instead of fetching."""++ def __init__(self, domains):+ super().__init__(f"http://fake/{id(self)}")+ self._domains = set(domains)++ def _get_blacklisted_domains(self):+ return self._domains+++def redis_client():+ return fakeredis.FakeRedis()+++def provider():+ return CombinedBlacklistProvider([+ StaticBlacklistProvider({"not-remote.test"}),+ FakeRemoteProvider({"badsite.test", "evil.test"}),+ FakeRemoteProvider({"evil.test", "spam.test"}),+ ])+++def test_collect_remote_domains_walks_combined_providers_and_skips_local_ones(provider):+ # Only remote lists are snapshotted; local rule providers are evaluated in-process,+ # so "not-remote.test" must not appear here.+ assert collect_remote_domains(provider) == {"badsite.test", "evil.test", "spam.test"}+++def test_build_snapshot_is_sorted_and_deduplicated(provider):+ array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)+ assert len(array) == 3 # evil.test appears in both lists+ assert list(array) == sorted(array)+++def test_publish_and_load_round_trip(provider, redis_client):+ version = refresh_snapshot(provider, redis_client)++ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=redis_client)+ assert blacklist.load_now() is True+ assert blacklist.filter_blacklisted(+ ["badsite.test", "evil.test", "spam.test", "good.test"]+ ) == {"badsite.test", "evil.test", "spam.test"}+ assert redis_client.get(SNAPSHOT_VERSION_KEY).decode() == version+++def test_load_is_a_no_op_when_the_version_is_unchanged(provider, redis_client):+ refresh_snapshot(provider, redis_client)+ blacklist = SnapshotBlacklist(redis_client=redis_client)+ assert blacklist.load_now() is True+ assert blacklist.load_now() is False+++def test_load_picks_up_a_republished_snapshot(redis_client):+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=redis_client)++ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"first.test"})]), redis_client)+ blacklist.load_now()+ assert blacklist.filter_blacklisted(["first.test", "second.test"]) == {"first.test"}++ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"second.test"})]), redis_client)+ assert blacklist.load_now() is True+ assert blacklist.filter_blacklisted(["first.test", "second.test"]) == {"second.test"}+++def test_built_in_rules_apply_without_a_snapshot(redis_client):+ """The local rules must work even before the first snapshot lands, so a domain added+ to mwmbl/settings.py takes effect on deploy rather than at the next refresh."""+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({"local.test"}),+ redis_client=redis_client)++ assert blacklist.loaded is False+ assert blacklist.filter_blacklisted(["local.test", "good.test"]) == {"local.test"}+++def test_eviction_keeps_the_loaded_snapshot(provider, redis_client):+ """Production Redis runs allkeys-lru, so the snapshot can vanish. Losing it must not+ silently disable filtering."""+ refresh_snapshot(provider, redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=redis_client)+ blacklist.load_now()++ redis_client.delete(SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY)++ assert blacklist.load_now() is False+ assert blacklist.filter_blacklisted(["badsite.test", "good.test"]) == {"badsite.test"}+++def test_redis_failure_keeps_the_loaded_snapshot(provider):+ class BrokenRedis:+ def get(self, key):+ raise ConnectionError("redis is down")++ refresh_snapshot(provider, (working := fakeredis.FakeRedis()))+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=working)+ blacklist.load_now()++ blacklist._redis = BrokenRedis()+ assert blacklist.load_now() is False+ assert blacklist.filter_blacklisted(["badsite.test"]) == {"badsite.test"}+++def test_publishing_the_same_domains_twice_keeps_the_same_version(provider, redis_client):+ """Workers skip the 11 MB download when the version is unchanged, so an unchanged+ blocklist must not churn the version."""+ assert refresh_snapshot(provider, redis_client) == refresh_snapshot(provider, redis_client)+++def test_filter_blacklisted_handles_an_empty_input(provider, redis_client):+ blacklist = SnapshotBlacklist(redis_client=redis_client)+ publish_snapshot(build_snapshot(provider), redis_client)+ blacklist.load_now()++ assert blacklist.filter_blacklisted([]) == set()+++def test_is_domain_blacklisted(provider, redis_client):+ refresh_snapshot(provider, redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=redis_client)+ blacklist.load_now()++ assert blacklist.is_domain_blacklisted("badsite.test") is True+ assert blacklist.is_domain_blacklisted("good.test") is False+++def test_hashes_above_every_entry_do_not_index_out_of_bounds(redis_client):+ """np.searchsorted returns len(array) for a hash larger than everything in it."""+ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"only.test"})]), redis_client)+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),+ redis_client=redis_client)+ blacklist.load_now()++ # Try enough domains that at least one hashes above and one below the single entry.+ candidates = [f"candidate{i}.test" for i in range(100)]+ assert blacklist.filter_blacklisted(candidates) == set()diff --git a/test/test_index_batches.py b/test/test_index_batches.pyindex bc262b6..9137d0d 100644--- a/test/test_index_batches.py+++ b/test/test_index_batches.py@@ -1,11 +1,15 @@from pathlib import Pathfrom tempfile import TemporaryDirectory+from unittest.mock import patch+import pytest++from mwmbl.indexer.blacklist_providers import StaticBlacklistProviderfrom mwmbl.indexer.index_batches import (sort_documents, combine_documents, _merge_user_ids, MAX_USER_IDS,- index_results_against_query,+ index_results_against_query, index_documents,)-from mwmbl.tinysearchengine.indexer import Document, DocumentState, TinyIndex+from mwmbl.tinysearchengine.indexer import Document, DocumentState, PAGE_SIZE, TinyIndexclass UrlRanker:@@ -200,3 +204,54 @@ def test_combine_documents_propagates_user_ids_to_winner():assert len(combined) == 1assert 1 in combined[0].user_idsassert 2 in combined[0].user_ids+++# ---------------------------------------------------------------------------+# index_documents: blacklist enforcement+# ---------------------------------------------------------------------------+#+# index_documents() is the common choke point for every path that adds content to the+# index (offline batch processing, the trusted-crawler POST /results endpoint, the+# standalone crawl tool) - crawling/link-discovery only stop *new* crawling of a+# blacklisted domain, they don't stop a submitted batch that already contains one of its+# pages, so indexing needs its own blacklist check.++PATCH_TARGET = "mwmbl.indexer.index_batches.get_default_blacklist_provider"+++def index_path(tmp_path):+ path = tmp_path / "test.tinysearch"+ TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=10, page_size=PAGE_SIZE)+ return str(path)+++def _all_urls(index_path, num_pages=10):+ urls = []+ with TinyIndex(item_factory=Document, index_path=index_path, mode="r") as index:+ for page_index in range(num_pages):+ urls.extend(doc.url for doc in index.get_page(page_index))+ return urls+++def test_index_documents_skips_blacklisted_domain(index_path):+ documents = [+ Document(title="Bad", url="https://fineartteens.com/x", extract="teen gallery"),+ Document(title="Good", url="https://example.com/y", extract="a good page"),+ ]++ with patch(PATCH_TARGET, return_value=StaticBlacklistProvider({"fineartteens.com"})):+ index_documents(documents, index_path)++ urls = _all_urls(index_path)+ assert "https://fineartteens.com/x" not in urls+ assert "https://example.com/y" in urls+++def test_index_documents_keeps_everything_when_nothing_blacklisted(index_path):+ documents = [Document(title="Good", url="https://example.com/y", extract="a good page")]++ with patch(PATCH_TARGET, return_value=StaticBlacklistProvider(set())):+ index_documents(documents, index_path)++ assert "https://example.com/y" in _all_urls(index_path)diff --git a/test/test_purge_blacklisted.py b/test/test_purge_blacklisted.pynew file mode 100644index 0000000..7c2ff90--- /dev/null+++ b/test/test_purge_blacklisted.py@@ -0,0 +1,106 @@+import pytest++from mwmbl.indexer.index import tokenize_document+from mwmbl.indexer.index_batches import index_pages, preprocess_documents+from mwmbl.indexer.purge_blacklisted import pages_for_document, purge_documents+from mwmbl.tinysearchengine.indexer import Document, PAGE_SIZE, TinyIndex++NUM_PAGES = 32++BAD = Document(title="Bad page about apples", url="https://badsite.test/apples",+ extract="apples and oranges and bananas", score=1.0)+GOOD = Document(title="Good page about apples", url="https://example.test/apples",+ extract="apples and oranges and bananas", score=1.0)+++def index_path(tmp_path):+ path = str(tmp_path / "purge.tinysearch")+ TinyIndex.create(item_factory=Document, index_path=path, num_pages=NUM_PAGES, page_size=PAGE_SIZE)+ return path+++def index(index_path, documents):+ index_pages(index_path, preprocess_documents(documents, index_path))+++def urls_in_index(index_path):+ with TinyIndex(Document, index_path, 'r') as index:+ return [d.url for page in range(NUM_PAGES) for d in index.get_page(page)]+++def test_purge_removes_a_document_from_every_page_it_is_filed_under(index_path):+ index(index_path, [BAD, GOOD])++ with TinyIndex(Document, index_path, 'w') as tiny_index:+ # The document must be filed under several terms, or this proves nothing.+ assert len(pages_for_document(tiny_index, BAD)) > 1+ removed = purge_documents(tiny_index, [BAD])++ assert removed == {"badsite.test": len(pages_for_document_count(index_path, BAD))}+ assert "https://badsite.test/apples" not in urls_in_index(index_path)+++def pages_for_document_count(index_path, document):+ with TinyIndex(Document, index_path, 'r') as tiny_index:+ return pages_for_document(tiny_index, document)+++def test_purge_leaves_other_documents_on_shared_pages_alone(index_path):+ """The bad and good documents share almost all their tokens, so they share pages.+ Purging must remove by URL, not by re-testing the page against the blacklist -+ otherwise it sweeps up documents it was never asked to touch."""+ index(index_path, [BAD, GOOD])++ with TinyIndex(Document, index_path, 'w') as tiny_index:+ shared = pages_for_document(tiny_index, BAD) & pages_for_document(tiny_index, GOOD)+ assert shared, "expected these documents to share at least one page"+ purge_documents(tiny_index, [BAD])++ urls = urls_in_index(index_path)+ assert "https://badsite.test/apples" not in urls+ assert "https://example.test/apples" in urls+++def test_purge_re_checks_the_blacklist_before_removing(index_path):+ """A domain taken off the blacklist between being queued and being purged must+ survive."""+ index(index_path, [BAD, GOOD])++ with TinyIndex(Document, index_path, 'w') as tiny_index:+ removed = purge_documents(tiny_index, [BAD], is_blacklisted=lambda domain: False)++ assert removed == {}+ assert "https://badsite.test/apples" in urls_in_index(index_path)+++def test_purge_removes_only_the_documents_that_are_still_blacklisted(index_path):+ index(index_path, [BAD, GOOD])++ with TinyIndex(Document, index_path, 'w') as tiny_index:+ removed = purge_documents(tiny_index, [BAD, GOOD],+ is_blacklisted=lambda domain: domain == "badsite.test")++ assert set(removed) == {"badsite.test"}+ urls = urls_in_index(index_path)+ assert "https://badsite.test/apples" not in urls+ assert "https://example.test/apples" in urls+++def test_purge_of_an_absent_document_is_a_no_op(index_path):+ index(index_path, [GOOD])+ absent = Document(title="Nowhere", url="https://badsite.test/nowhere", extract="nothing", score=1.0)++ with TinyIndex(Document, index_path, 'w') as tiny_index:+ assert purge_documents(tiny_index, [absent]) == {}++ assert "https://example.test/apples" in urls_in_index(index_path)+++def test_pages_for_document_matches_the_indexer(index_path):+ """purge_documents relies on tokenize_document being a pure function of the stored+ fields, so that recomputing it recovers exactly the pages the indexer wrote to."""+ with TinyIndex(Document, index_path, 'r') as tiny_index:+ tokens = tokenize_document(BAD.url, BAD.title, BAD.extract, BAD.score).tokens+ expected = {tiny_index.get_key_page_index(token) for token in tokens}+ assert pages_for_document(tiny_index, BAD) == expecteddiff --git a/test/test_rank_blacklist.py b/test/test_rank_blacklist.pynew file mode 100644index 0000000..b5b07af--- /dev/null+++ b/test/test_rank_blacklist.py@@ -0,0 +1,180 @@+"""Retrieval-time blacklist filtering: the guarantee that a blacklisted domain is never+shown, plus the enqueueing that lets the background purge clean the index up afterwards."""+from unittest.mock import patch++import fakeredis+import pytest++from mwmbl.indexer import purge_queue+from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider+from mwmbl.indexer.blacklist_snapshot import SnapshotBlacklist+from mwmbl.indexer.purge_queue import PURGE_QUEUE_KEY, drain_purge_queue+from mwmbl.tinysearchengine.indexer import Document, DocumentState+from mwmbl.tinysearchengine.rank import HeuristicRanker++BLACKLISTED_DOMAIN = "badsite.test"++BAD = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/bananas",+ extract="bananas and apples", score=1.0)+GOOD = Document(title="Bananas", url="https://example.test/bananas",+ extract="bananas and apples", score=1.0)+++class FakeIndex:+ """Returns the same documents for every term, like a page every query term hits."""++ def __init__(self, documents):+ self.documents = documents++ def retrieve(self, key):+ return list(self.documents)+++class FakeCompleter:+ def complete(self, term):+ return []++[email protected](autouse=True)+def clear_recently_enqueued():+ """purge_queue keeps a process-level record of what it has already queued, so without+ this each test would inherit the previous one's suppressions."""+ purge_queue._recently_enqueued.clear()+ yield+ purge_queue._recently_enqueued.clear()+++def redis_client():+ return fakeredis.FakeRedis(decode_responses=True)+++def blacklist(redis_client):+ """A snapshot-free blacklist: the built-in rules alone are enough to decide here, so+ these tests exercise the ranker rather than the snapshot machinery."""+ return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({BLACKLISTED_DOMAIN}),+ redis_client=redis_client)+++def ranker(blacklist, redis_client, request):+ documents = getattr(request, "param", [BAD, GOOD])+ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \+ patch.object(purge_queue, "get_redis", return_value=redis_client):+ yield HeuristicRanker(FakeIndex(documents), FakeCompleter())+++def test_search_drops_blacklisted_results(ranker):+ urls = [result.url for result in ranker.search("bananas", [])]++ assert BAD.url not in urls+ assert GOOD.url in urls+++def test_search_queues_blacklisted_results_for_purging(ranker, redis_client):+ ranker.search("bananas", [])++ queued = drain_purge_queue(10, redis_client)+ assert [d.url for d in queued] == [BAD.url]+ # The queued payload carries what tokenize_document needs to find the document's pages+ assert queued[0].title == BAD.title+ assert queued[0].extract == BAD.extract+++def test_a_clean_result_set_queues_nothing(ranker, redis_client):+ with patch.object(ranker, "tiny_index", FakeIndex([GOOD])):+ assert [r.url for r in ranker.search("bananas", [])] == [GOOD.url]++ assert redis_client.scard(PURGE_QUEUE_KEY) == 0+++def test_the_same_document_is_queued_once_across_queries(ranker, redis_client):+ """A blacklisted document stays in the index until the purge runs, so it keeps+ turning up. It must neither pile up in the queue nor cost a Redis round trip on+ every query in the meantime."""+ for _ in range(5):+ ranker.search("bananas", [])++ assert redis_client.scard(PURGE_QUEUE_KEY) == 1+ assert len(purge_queue._recently_enqueued) == 1+++def test_a_document_is_re_queued_once_the_suppression_expires(ranker, redis_client):+ """If a purge is lost the document has to get another chance, or it stays in the+ index for as long as this process lives."""+ ranker.search("bananas", [])+ assert drain_purge_queue(10, redis_client)++ purge_queue._recently_enqueued.clear() # stands in for the TTL expiring+ ranker.search("bananas", [])++ assert [d.url for d in drain_purge_queue(10, redis_client)] == [BAD.url]+++def test_get_raw_results_drops_blacklisted_results(ranker):+ """/raw bypasses get_results(), so it needs filtering of its own."""+ urls = [result.url for result in ranker.get_raw_results("bananas")]++ assert BAD.url not in urls+ assert GOOD.url in urls+++def test_complete_does_not_suggest_blacklisted_urls(ranker):+ _, completions = ranker.complete("banana")++ assert not any(BLACKLISTED_DOMAIN in completion for completion in completions)+++def test_curated_blacklisted_documents_are_dropped(blacklist, redis_client):+ """Curated items bypass order_results entirely - they are prepended straight into+ deduplicate() - so they need filtering separately from the ranked candidates."""+ curated = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/curated",+ extract="bananas", score=1.0, term="bananas",+ state=DocumentState.ORGANIC_APPROVED)++ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \+ patch.object(purge_queue, "get_redis", return_value=redis_client):+ ranker = HeuristicRanker(FakeIndex([curated, GOOD]), FakeCompleter())+ urls = [result.url for result in ranker.search("bananas", [])]++ assert curated.url not in urls+ assert GOOD.url in urls+++def test_additional_results_are_filtered_but_not_queued(blacklist, redis_client):+ """Caller-supplied results (the Google-sourced documents the search page passes in)+ must be filtered too, but there is nothing in our index to purge for them."""+ additional = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/from-google",+ extract="bananas", score=1.0, state=DocumentState.FROM_GOOGLE)++ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \+ patch.object(purge_queue, "get_redis", return_value=redis_client):+ ranker = HeuristicRanker(FakeIndex([GOOD]), FakeCompleter())+ urls = [result.url for result in ranker.search("bananas", [additional])]++ assert additional.url not in urls+ assert redis_client.scard(PURGE_QUEUE_KEY) == 0+++def test_filtering_can_be_switched_off(ranker, settings):+ """The kill switch has to work without a rollback if filtering misbehaves in prod."""+ settings.BLACKLIST_FILTER_AT_RETRIEVAL = False++ assert BAD.url in [result.url for result in ranker.search("bananas", [])]+++def test_a_redis_failure_does_not_break_search(blacklist):+ """Enqueueing is best-effort: losing an entry just means the next query re-queues it,+ but a search response must never fail because of it."""+ class BrokenRedis:+ def scard(self, key):+ raise ConnectionError("redis is down")++ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \+ patch.object(purge_queue, "get_redis", return_value=BrokenRedis()):+ ranker = HeuristicRanker(FakeIndex([BAD, GOOD]), FakeCompleter())+ urls = [result.url for result in ranker.search("bananas", [])]++ assert BAD.url not in urls+ assert GOOD.url in urls