Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Filter blacklisted domains at retrieval, then purge them in the background

Daoud Clarkece627e385f1d
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index e5f1b07..60a17be 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -67,14 +67,31 @@ class MwmblConfig(AppConfig):
try:
from background_task.models import Task
- from mwmbl.background import sync_search_counts
+ from mwmbl.background import (
+ purge_blacklisted_from_queue, refresh_blacklist_snapshot, sync_search_counts,
+ )
SYNC_TASK = "mwmbl.background.sync_search_counts"
+ BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"
+ BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"
# Sync search counts once per hour (3600 seconds)
if not Task.objects.filter(task_name=SYNC_TASK).exists():
sync_search_counts(repeat=3600, repeat_until=None)
+ # Rebuild the blacklist snapshot the search path filters against. schedule=0
+ # on the task means the first run happens immediately rather than one full
+ # refresh interval after deploy, which matters because until it lands the
+ # search workers have only the built-in rules to go on.
+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():
+ refresh_blacklist_snapshot(
+ repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)
+
+ # Drain the queue of blacklisted documents that retrieval has filtered out
+ if not Task.objects.filter(task_name=BLACKLIST_PURGE_TASK).exists():
+ purge_blacklisted_from_queue(
+ repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)
+
except Exception:
# Don't prevent startup if background task scheduling fails
log.exception("Failed to schedule background tasks")
diff --git a/mwmbl/background.py b/mwmbl/background.py
index 50cca81..062bb0d 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -1,8 +1,10 @@
"""
Script that updates data in a background process.
-Also contains Django Background Tasks for periodic quota maintenance:
+Also contains Django Background Tasks for periodic maintenance:
- sync_search_counts: syncs Redis monthly counters → DB once per hour
+ - refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against
+ - purge_blacklisted_from_queue: removes retrieval-filtered documents from the index
"""
import logging
import sys
@@ -131,3 +133,48 @@ def sync_search_counts():
logger.exception("Error syncing search count for key %s", key)
+# ---------------------------------------------------------------------------
+# Blacklisted-domain maintenance (Django Background Tasks)
+# ---------------------------------------------------------------------------
+
+@background(schedule=0)
+def refresh_blacklist_snapshot():
+ """
+ Rebuild the blacklist snapshot that the search workers filter against.
+
+ This is the only place the multi-megabyte remote blocklists are downloaded and
+ parsed. Search workers only ever read the ~11 MB hash array this publishes to Redis,
+ so no query ever waits on a blocklist fetch. See mwmbl.indexer.blacklist_snapshot.
+ """
+ from mwmbl.indexer.blacklist_snapshot import refresh_snapshot
+
+ refresh_snapshot()
+
+
+@background(schedule=0)
+def purge_blacklisted_from_queue():
+ """
+ Remove the documents that retrieval filtered out from the index itself.
+
+ Retrieval already guarantees these are never shown; this closes the loop so the same
+ documents are not re-filtered on every future query. The queue is best-effort, and
+ that is fine: anything lost is re-queued the next time a query surfaces it.
+ """
+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
+ from mwmbl.indexer.purge_blacklisted import purge_documents
+ from mwmbl.indexer.purge_queue import drain_purge_queue, queue_size
+ from mwmbl.tinysearchengine.indexer import Document, TinyIndex
+
+ documents = drain_purge_queue(settings.BLACKLIST_PURGE_BATCH_SIZE)
+ if not documents:
+ return
+
+ blacklist = get_snapshot_blacklist()
+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
+ with TinyIndex(Document, str(index_path), 'w') as index:
+ removed_by_domain = purge_documents(index, documents, blacklist.is_domain_blacklisted)
+
+ logger.info("Purged %d documents from the index across %d domains; %d still queued",
+ sum(removed_by_domain.values()), len(removed_by_domain), queue_size())
+
+
diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.py
index 5aa9c36..08f4652 100644
--- a/mwmbl/indexer/blacklist.py
+++ b/mwmbl/indexer/blacklist.py
@@ -6,8 +6,9 @@ The main logic has been moved to blacklist_providers.py for better modularity.
"""
from mwmbl.indexer.blacklist_providers import (
- BuiltInRulesBlacklistProvider,
- HaGeZiBlacklistProvider,
+ BuiltInRulesBlacklistProvider,
+ HaGeZiBlacklistProvider,
+ AdultContentBlacklistProvider,
CombinedBlacklistProvider,
BlacklistProvider
)
@@ -17,5 +18,6 @@ def get_default_blacklist_provider() -> BlacklistProvider:
"""Get the default blacklist provider configuration."""
return CombinedBlacklistProvider([
BuiltInRulesBlacklistProvider(),
- HaGeZiBlacklistProvider('light')
+ HaGeZiBlacklistProvider('tif_medium'),
+ AdultContentBlacklistProvider(),
])
diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.py
index 37075ee..aea61e6 100644
--- a/mwmbl/indexer/blacklist_providers.py
+++ b/mwmbl/indexer/blacklist_providers.py
@@ -5,11 +5,16 @@ This module provides different ways to check if domains should be blacklisted,
making the system more flexible and testable.
"""
+import time
from abc import ABC, abstractmethod
-from datetime import timedelta
from typing import Set
import requests
-from mwmbl.utils import request_cache
+
+# Parsed remote lists (~950k+ lines) are expensive to re-parse, and get_default_blacklist_provider()
+# is now called on every index_documents() call (indexing needs to check the blacklist too, not just
+# crawling), so the parsed result is cached here at module level - shared across every
+# RemoteListBlacklistProvider instance, however many get constructed - keyed by URL.
+_parsed_domains_cache: dict[str, tuple[float, Set[str]]] = {}
class BlacklistProvider(ABC):
@@ -57,73 +62,127 @@ class BuiltInRulesBlacklistProvider(BlacklistProvider):
# Trusted domains are never blacklisted
if domain in self.trusted_domains:
return False
-
+
# Spam detection heuristics for SEO spam domains
domain_parts = domain.split('.')
-
- # Domains like: brofqpxj.uelinc.com, gzsmjc.fba01.com, 59648.etnomurcia.com
- if (len(domain_parts) == 3 and
- domain_parts[2] == "com" and
- len(domain_parts[0]) in {6, 8}):
- return True
-
- # Domains with all numeric first parts
- if len(domain_parts) > 0 and set(domain_parts[0]) <= set("1234567890"):
+
+ # Domains with all-numeric generated-looking subdomains, e.g. 59648.etnomurcia.com.
+ # Restricted to len > 2 so this doesn't catch a numeric *apex* domain like 350.org,
+ # where the numeric part is the actual site name rather than a generated subdomain.
+ if len(domain_parts) > 2 and set(domain_parts[0]) <= set("1234567890"):
return True
return False
-class HaGeZiBlacklistProvider(BlacklistProvider):
- """Provider that fetches HaGeZi blocklist."""
-
- # HaGeZi provides several lists, this is their main threat intelligence feeds
- HAGEZI_URLS = {
- 'light': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/light.txt',
- 'normal': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/normal.txt',
- 'pro': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/pro.txt',
- 'ultimate': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/ultimate.txt',
- }
-
- def __init__(self, list_type: str = 'light', cache_expire_days: int = 1):
- if list_type not in self.HAGEZI_URLS:
- raise ValueError(f"Invalid list_type. Must be one of: {list(self.HAGEZI_URLS.keys())}")
-
- self.url = self.HAGEZI_URLS[list_type]
+class RemoteListBlacklistProvider(BlacklistProvider):
+ """Provider that fetches a remote newline-delimited domain list, with caching.
+
+ Handles both plain domain-per-line lists and hosts-file format
+ (``0.0.0.0 domain.com``), which is what most public blocklists use.
+ """
+
+ def __init__(self, url: str, cache_expire_days: int = 1):
+ self.url = url
self.cache_expire_days = cache_expire_days
- self._cached_domains = None
-
+
def _get_blacklisted_domains(self) -> Set[str]:
- """Fetch HaGeZi blacklist with caching."""
- if self._cached_domains is not None:
- return self._cached_domains
-
- with request_cache(expire_after=timedelta(days=self.cache_expire_days)) as session:
- try:
- response = session.get(self.url)
- response.raise_for_status()
-
- # Parse HaGeZi format - one domain per line, skip comments and empty lines
- domains = set()
- for line in response.text.split('\n'):
- line = line.strip()
- if line and not line.startswith('#'):
- domains.add(line)
-
- self._cached_domains = domains
+ cached = _parsed_domains_cache.get(self.url)
+ if cached is not None:
+ fetched_at, domains = cached
+ if time.monotonic() - fetched_at < self.cache_expire_days * 86400:
return domains
- except requests.RequestException as e:
- # Log the error but don't fail - return empty set as fallback
- print(f"Warning: Failed to fetch HaGeZi blacklist from {self.url}: {e}")
- self._cached_domains = set()
- return set()
-
+
+ # Deliberately NOT using mwmbl.utils.request_cache here. That is a *filesystem*
+ # cache rooted at settings.REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache" - the
+ # same volume that holds the multi-hundred-GB index. These blocklists are tens of
+ # megabytes each, so caching them there fills that volume, and once it is full
+ # *every* user of request_cache breaks - including get_wiki_results(), which runs
+ # on the normal search path and then retries a live fetch 4 times per uncached
+ # query (WIKI_RETRY, 5s timeout each) until the worker is killed. A big periodic
+ # download has no business in a shared small-response cache on the index volume;
+ # the module-level parsed cache above already gives us reuse without touching disk.
+ try:
+ response = requests.get(self.url, timeout=60)
+ response.raise_for_status()
+
+ domains = set()
+ for line in response.text.split('\n'):
+ line = line.strip()
+ if not line or line.startswith('#'):
+ continue
+ # Hosts format: "0.0.0.0 domain.com" or "127.0.0.1 domain.com"
+ parts = line.split()
+ domains.add(parts[1] if len(parts) == 2 and parts[0] in ('0.0.0.0', '127.0.0.1') else parts[0])
+
+ _parsed_domains_cache[self.url] = (time.monotonic(), domains)
+ return domains
+ except requests.RequestException as e:
+ # Log the error but don't fail. Fall back to the last known-good parsed set
+ # rather than caching an empty one - a transient fetch error shouldn't silently
+ # disable this provider's coverage until the next successful fetch.
+ print(f"Warning: Failed to fetch blacklist from {self.url}: {e}")
+ return cached[1] if cached is not None else set()
+
def is_domain_blacklisted(self, domain: str) -> bool:
- """Check if domain is in the HaGeZi blacklist."""
domains = self._get_blacklisted_domains()
return domain in domains
+class HaGeZiBlacklistProvider(RemoteListBlacklistProvider):
+ """Provider that fetches a HaGeZi blocklist.
+
+ NB: upstream restructured their repo from domains/{tier}.txt to
+ wildcard/{tier}-onlydomains.txt at some point - the old domains/*.txt URLs now
+ 404 silently (request_cache treats that as "list temporarily empty", not an
+ error), so whichever tier was previously configured here had been contributing
+ nothing.
+
+ The 'light'/'normal'/'pro'/'ultimate' tiers are HaGeZi's general "Multi" lists:
+ ads, affiliate, tracking, telemetry, and only *also* phishing/malware/scam mixed
+ in. They're built for personal ad-blocking, not "should this be excluded from a
+ search index" - they flag plenty of legitimate businesses purely for running
+ analytics/trackers (e.g. baremetrics.com, covidtracking.com), which is a false
+ positive for our purposes even though it's correct behaviour for a DNS ad-blocker.
+
+ The 'tif'/'tif_medium'/'tif_mini' tiers are HaGeZi's dedicated Threat
+ Intelligence Feed - malware, phishing, scam and C2 domains specifically, no
+ general ad/tracker noise. That's the actually-relevant category here, and
+ tif_medium (390k entries) tested clean against every legitimate domain in our
+ Search Console sample. get_default_blacklist_provider() uses 'tif_medium'.
+ """
+
+ HAGEZI_URLS = {
+ 'light': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt',
+ 'normal': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/multi-onlydomains.txt',
+ 'pro': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/pro-onlydomains.txt',
+ 'ultimate': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/ultimate-onlydomains.txt',
+ 'tif': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif-onlydomains.txt',
+ 'tif_medium': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.medium-onlydomains.txt',
+ 'tif_mini': 'https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/tif.mini-onlydomains.txt',
+ }
+
+ def __init__(self, list_type: str = 'tif_medium', cache_expire_days: int = 1):
+ if list_type not in self.HAGEZI_URLS:
+ raise ValueError(f"Invalid list_type. Must be one of: {list(self.HAGEZI_URLS.keys())}")
+
+ super().__init__(self.HAGEZI_URLS[list_type], cache_expire_days)
+
+
+class AdultContentBlacklistProvider(RemoteListBlacklistProvider):
+ """Provider that fetches The Block List Project's maintained adult-content domain list.
+
+ See https://github.com/blocklistproject/Lists - hosts-format, ~950k domains,
+ updated regularly. Covers the general adult-content gap that the built-in
+ regex/domain rules (aimed at spam and a handful of known-bad domains) don't.
+ """
+
+ URL = 'https://raw.githubusercontent.com/blocklistproject/Lists/master/porn.txt'
+
+ def __init__(self, cache_expire_days: int = 7):
+ super().__init__(self.URL, cache_expire_days)
+
+
class CombinedBlacklistProvider(BlacklistProvider):
"""Provider that combines multiple blacklist providers."""
diff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.py
new file mode 100644
index 0000000..9ef7309
--- /dev/null
+++ b/mwmbl/indexer/blacklist_snapshot.py
@@ -0,0 +1,269 @@
+"""A blacklist representation the search path can afford to consult on every query.
+
+The blacklist providers in blacklist_providers.py are built for the indexing path: the
+remote-list ones download tens of megabytes and parse them into a Python set of ~1.3M
+domain strings. That is fine for a batch job but wrong for search in two ways:
+
+ * Memory. The combined HaGeZi tif_medium + Block List Project adult list is 1,338,063
+ unique domains, which costs ~156 MB as a set[str] - and gunicorn runs
+ (cpu_count * 2 + 1) workers, so on an 8-core box that would be ~2.6 GB of duplicated
+ domain strings.
+ * Latency. RemoteListBlacklistProvider fetches lazily, so the first query in each
+ worker - and the first query after the module cache expires - would block on a
+ multi-megabyte HTTP download with a 60 second timeout. That is exactly the failure
+ mode that commit c28be4e was fixing.
+
+So the query path never fetches, parses or holds the domain strings. A background task
+(mwmbl.background.refresh_blacklist_snapshot) does that work once, hashes every domain to
+a 64-bit integer, and publishes the sorted array to Redis as one ~11 MB blob. Workers
+load the blob into a numpy array and answer membership with a vectorised binary search:
+10.7 MB resident, and ~94 us to classify the ~200 distinct domains of a typical query -
+noise next to the mmap page reads and the Wikipedia call the same request already makes.
+
+Hashing rather than storing the domains is what buys the 15x memory saving. It costs
+exactness in principle: two different domains could collide and a clean domain be treated
+as blacklisted. With 64-bit hashes over 1.3M entries the chance any given lookup collides
+is ~7e-14, i.e. it will not happen.
+
+Only the *remote* lists go in the snapshot. BuiltInRulesBlacklistProvider (the regex,
+EXCLUDED_DOMAINS, the numeric-subdomain heuristic and the hn_top_domains trust exemption)
+is cheap and stays a local in-process check, so editing mwmbl/settings.py takes effect on
+deploy instead of waiting for the next snapshot refresh. Evaluating them as
+`built_in_rules OR snapshot` reproduces CombinedBlacklistProvider's semantics exactly.
+"""
+import hashlib
+import threading
+import time
+from logging import getLogger
+from typing import Iterable, Optional
+
+import mmh3
+import numpy as np
+import redis
+from django.conf import settings
+
+from mwmbl.indexer.blacklist_providers import (
+ BlacklistProvider,
+ BuiltInRulesBlacklistProvider,
+ CombinedBlacklistProvider,
+ RemoteListBlacklistProvider,
+)
+
+logger = getLogger(__name__)
+
+
+SNAPSHOT_KEY = "blacklist:domain-hashes:v1"
+SNAPSHOT_VERSION_KEY = "blacklist:domain-hashes:v1:version"
+
+# Hashes are stored little-endian so the blob is portable between machines.
+HASH_DTYPE = np.dtype(np.uint64).newbyteorder('<')
+
+
+_redis: Optional[redis.Redis] = None
+
+
+def get_redis() -> redis.Redis:
+ """Binary-safe Redis connection - the snapshot blob is not text."""
+ global _redis
+ if _redis is None:
+ _redis = redis.from_url(settings.REDIS_URL)
+ return _redis
+
+
+def hash_domain(domain: str) -> int:
+ return mmh3.hash64(domain, signed=False)[0]
+
+
+def hash_domains(domains: Iterable[str]) -> np.ndarray:
+ return np.fromiter((hash_domain(d) for d in domains), dtype=HASH_DTYPE)
+
+
+# ---------------------------------------------------------------------------
+# Write side - runs in the background task only
+# ---------------------------------------------------------------------------
+
+def collect_remote_domains(provider: BlacklistProvider) -> set[str]:
+ """Every domain from the remote lists reachable from this provider.
+
+ Walks CombinedBlacklistProvider recursively so callers can just hand over
+ get_default_blacklist_provider() without knowing how it is composed. Providers that
+ are not remote lists (the built-in rules) have no enumerable domain set and are
+ skipped - they are evaluated locally instead, see the module docstring.
+ """
+ if isinstance(provider, CombinedBlacklistProvider):
+ domains = set()
+ for sub_provider in provider.providers:
+ domains |= collect_remote_domains(sub_provider)
+ return domains
+ if isinstance(provider, RemoteListBlacklistProvider):
+ return provider._get_blacklisted_domains()
+ return set()
+
+
+def build_snapshot(provider: BlacklistProvider) -> bytes:
+ """Download/parse the remote lists and return the sorted hash array as bytes."""
+ domains = collect_remote_domains(provider)
+ hashes = np.unique(hash_domains(domains)) # np.unique sorts, which is what we need
+ logger.info("Built blacklist snapshot: %d domains, %d unique hashes, %.1f MB",
+ len(domains), len(hashes), hashes.nbytes / 1e6)
+ return hashes.astype(HASH_DTYPE).tobytes()
+
+
+def publish_snapshot(blob: bytes, redis_client: Optional[redis.Redis] = None) -> str:
+ """Publish the blob and return its version.
+
+ The version is a content hash, so a refresh that produces an identical snapshot does
+ not make every worker re-download 11 MB. Blob first, then version: a worker that sees
+ a new version is then guaranteed to find the matching blob.
+ """
+ redis_client = redis_client if redis_client is not None else get_redis()
+ version = hashlib.sha256(blob).hexdigest()
+ redis_client.set(SNAPSHOT_KEY, blob)
+ redis_client.set(SNAPSHOT_VERSION_KEY, version)
+ logger.info("Published blacklist snapshot version %s (%.1f MB)", version[:12], len(blob) / 1e6)
+ return version
+
+
+def refresh_snapshot(provider: Optional[BlacklistProvider] = None,
+ redis_client: Optional[redis.Redis] = None) -> str:
+ if provider is None:
+ from mwmbl.indexer.blacklist import get_default_blacklist_provider
+ provider = get_default_blacklist_provider()
+ return publish_snapshot(build_snapshot(provider), redis_client)
+
+
+# ---------------------------------------------------------------------------
+# Read side - runs in every search worker
+# ---------------------------------------------------------------------------
+
+class SnapshotBlacklist:
+ """Membership queries against the published snapshot, plus the local built-in rules.
+
+ One instance per process. Nothing here ever blocks a request on the network: the
+ snapshot is refreshed by a daemon thread, and the array is swapped in by plain
+ attribute assignment (atomic in CPython), so an in-flight query simply finishes
+ against the previous array.
+ """
+
+ def __init__(self, built_in_rules: Optional[BlacklistProvider] = None,
+ redis_client: Optional[redis.Redis] = None):
+ self._built_in_rules = built_in_rules if built_in_rules is not None else BuiltInRulesBlacklistProvider()
+ self._redis = redis_client
+ self._array: Optional[np.ndarray] = None
+ self._version: Optional[str] = None
+ self._last_checked: float = 0.0
+ self._loading = False
+ self._lock = threading.Lock()
+
+ @property
+ def loaded(self) -> bool:
+ return self._array is not None
+
+ def _get_redis(self) -> redis.Redis:
+ return self._redis if self._redis is not None else get_redis()
+
+ def load_now(self) -> bool:
+ """Synchronously fetch the snapshot if its version has changed. Returns True if
+ the in-memory array was replaced."""
+ try:
+ client = self._get_redis()
+ version = client.get(SNAPSHOT_VERSION_KEY)
+ except Exception:
+ logger.warning("Could not read blacklist snapshot version from Redis", exc_info=True)
+ return False
+
+ if version is None:
+ # Production Redis runs allkeys-lru, so the snapshot can be evicted. Keep
+ # whatever we already have rather than falling back to an empty array, which
+ # would silently disable filtering; the refresh task will republish it.
+ logger.warning("Blacklist snapshot missing from Redis (evicted?); keeping the loaded copy")
+ return False
+
+ version = version.decode() if isinstance(version, bytes) else version
+ if version == self._version:
+ return False
+
+ try:
+ blob = client.get(SNAPSHOT_KEY)
+ except Exception:
+ logger.warning("Could not read blacklist snapshot from Redis", exc_info=True)
+ return False
+
+ if not blob:
+ logger.warning("Blacklist snapshot version %s present but blob is missing", version[:12])
+ return False
+
+ array = np.frombuffer(blob, dtype=HASH_DTYPE)
+ self._array = array
+ self._version = version
+ logger.info("Loaded blacklist snapshot version %s: %d domains", version[:12], len(array))
+ return True
+
+ def _maybe_refresh(self):
+ """Kick off a refresh in the background if it has been long enough."""
+ interval = getattr(settings, "BLACKLIST_SNAPSHOT_CHECK_SECONDS", 300)
+ if time.monotonic() - self._last_checked < interval:
+ return
+
+ with self._lock:
+ if time.monotonic() - self._last_checked < interval or self._loading:
+ return
+ self._last_checked = time.monotonic()
+ self._loading = True
+
+ if self._array is None:
+ # Logged here rather than per query: this is once per check interval, which
+ # is often enough to be noticed and rare enough not to drown the logs.
+ logger.error("No blacklist snapshot has loaded; filtering on built-in rules only")
+
+ def load():
+ try:
+ self.load_now()
+ finally:
+ self._loading = False
+
+ threading.Thread(target=load, name="blacklist-snapshot-refresh", daemon=True).start()
+
+ def filter_blacklisted(self, domains: Iterable[str]) -> set[str]:
+ """Return the subset of these domains that are blacklisted."""
+ domains = list(dict.fromkeys(domains)) # de-duplicate, keeping order for zip below
+ if not domains:
+ return set()
+
+ self._maybe_refresh()
+
+ blacklisted = {d for d in domains if self._built_in_rules.is_domain_blacklisted(d)}
+
+ array = self._array # read once: a background refresh may swap it mid-call
+ if array is None:
+ return blacklisted
+
+ remaining = [d for d in domains if d not in blacklisted]
+ if not remaining:
+ return blacklisted
+
+ hashes = hash_domains(remaining)
+ positions = np.searchsorted(array, hashes)
+ # searchsorted can return len(array) for a hash above every entry; clamp so the
+ # lookup below stays in bounds. Index 0 is always a safe stand-in because the
+ # equality check is what decides membership.
+ positions[positions >= len(array)] = 0
+ found = array[positions] == hashes
+
+ blacklisted.update(domain for domain, hit in zip(remaining, found) if hit)
+ return blacklisted
+
+ def is_domain_blacklisted(self, domain: str) -> bool:
+ return domain in self.filter_blacklisted([domain])
+
+
+_snapshot_blacklist: Optional[SnapshotBlacklist] = None
+
+
+def get_snapshot_blacklist() -> SnapshotBlacklist:
+ """The per-process singleton, shared by search_setup and the background tasks."""
+ global _snapshot_blacklist
+ if _snapshot_blacklist is None:
+ _snapshot_blacklist = SnapshotBlacklist()
+ _snapshot_blacklist.load_now()
+ return _snapshot_blacklist
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index c952fa9..290f45e 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -13,12 +13,13 @@ from mwmbl.crawler.batch import HashedBatch, Item
from mwmbl.crawler.urls import URLStatus
from mwmbl.indexer import process_batch
from mwmbl.indexer.batch_cache import BatchCache
+from mwmbl.indexer.blacklist import get_default_blacklist_provider
from mwmbl.indexer.index import tokenize_document, prepare_url_for_tokenizing
from mwmbl.indexer.indexdb import BatchStatus
from mwmbl.tinysearchengine.indexer import Document, TinyIndex, DocumentState, CURATED_STATES
from mwmbl.tinysearchengine.rank import score_result, DOCUMENT_FREQUENCIES, N_DOCUMENTS, HeuristicRanker
from mwmbl.tokenizer import tokenize, get_bigrams
-from mwmbl.utils import add_term_infos
+from mwmbl.utils import add_term_infos, get_domain
logger = getLogger(__name__)
@@ -69,12 +70,37 @@ def index_batches(batch_data: Collection[HashedBatch], index_path: str) -> Count
def index_documents(documents, index_path):
+ """The common choke point every indexing path (offline batch processing, the
+ trusted-crawler POST /results endpoint, the standalone crawl tool) goes through, so
+ this is where the blacklist is enforced. Crawling/link-discovery also check the
+ blacklist (RedisURLQueue, update_urls.process_link) but only to stop *new* crawling -
+ a submitted batch or a direct /results submission can contain a blacklisted domain's
+ pages regardless of whether that domain was ever handed out to be crawled (e.g. a
+ browser-extension user organically visiting the site), so indexing needs its own check
+ rather than relying on those upstream gates."""
+ documents = filter_blacklisted_documents(documents)
page_documents = preprocess_documents(documents, index_path)
new_page_doc_counts = index_pages(index_path, page_documents)
end_time = datetime.utcnow()
return end_time, new_page_doc_counts
+def filter_blacklisted_documents(documents: list[Document]) -> list[Document]:
+ blacklist_provider = get_default_blacklist_provider()
+ kept = []
+ for document in documents:
+ try:
+ domain = get_domain(document.url)
+ except ValueError:
+ kept.append(document)
+ continue
+ if blacklist_provider.is_domain_blacklisted(domain):
+ logger.info(f"Skipping indexing for blacklisted domain {domain}: {document.url}")
+ continue
+ kept.append(document)
+ return kept
+
+
def index_pages(index_path: str, page_documents: dict[int, list[Document]], mark_synced: bool = False) -> Counter:
term_new_doc_counts = Counter()
with TinyIndex(Document, index_path, 'w') as indexer:
diff --git a/mwmbl/indexer/purge_blacklisted.py b/mwmbl/indexer/purge_blacklisted.py
new file mode 100644
index 0000000..a076418
--- /dev/null
+++ b/mwmbl/indexer/purge_blacklisted.py
@@ -0,0 +1,80 @@
+"""Removal of specific already-indexed documents from the index.
+
+TinyIndex has no per-document delete: a page is a compressed blob of documents, so
+removal means reading the page, dropping the document from the list, and writing the
+whole page back. The only hard part is knowing *which* pages to rewrite.
+
+A document is filed under every token that tokenize_document() derives from it - each
+token hashes to one page - so a document typically lives on a dozen or more pages, and
+removing it from just the page you happened to find it on leaves it retrievable by every
+other term. tokenize_document() is a pure function of the url/title/extract/score stored
+in the index, so recomputing it recovers that full token set exactly.
+"""
+from collections import defaultdict
+from logging import getLogger
+from typing import Callable, Iterable, Optional
+
+from mwmbl.indexer.index import tokenize_document
+from mwmbl.tinysearchengine.indexer import Document, TinyIndex
+from mwmbl.utils import get_domain
+
+logger = getLogger(__name__)
+
+
+def pages_for_document(index: TinyIndex, document: Document) -> set[int]:
+ """Every page this document is filed under."""
+ tokenized = tokenize_document(document.url, document.title, document.extract, document.score)
+ return {index.get_key_page_index(token) for token in tokenized.tokens}
+
+
+def purge_documents(index: TinyIndex, documents: Iterable[Document],
+ is_blacklisted: Optional[Callable[[str], bool]] = None) -> dict[str, int]:
+ """Remove exactly these documents, by URL, from every page they are filed under.
+
+ Returns a count of removed documents by domain.
+
+ Removal is by URL membership, never by re-applying the blacklist to the whole page.
+ Common tokens - "com", "you", "the" - hash to pages shared with thousands of
+ unrelated documents, so re-testing the page would sweep away blacklisted documents
+ that were never in the set we were asked to remove and that nobody has looked at.
+ Those will come round again on their own once a query surfaces them.
+
+ `is_blacklisted` re-checks each document before removal, so a domain that came off
+ the blacklist between being queued and being purged is left alone. Pass None to skip
+ the re-check and remove everything given.
+ """
+ target_urls = set()
+ pages_to_urls: dict[int, set[str]] = defaultdict(set)
+ for document in documents:
+ if is_blacklisted is not None:
+ try:
+ domain = get_domain(document.url)
+ except ValueError:
+ continue
+ if not is_blacklisted(domain):
+ logger.info("Skipping purge of %s: no longer blacklisted", document.url)
+ continue
+
+ target_urls.add(document.url)
+ for page_index in pages_for_document(index, document):
+ pages_to_urls[page_index].add(document.url)
+
+ removed_by_domain: dict[str, int] = defaultdict(int)
+ for page_index, urls in pages_to_urls.items():
+ page = index.get_page(page_index)
+ kept = [d for d in page if d.url not in urls]
+ if len(kept) == len(page):
+ continue
+
+ for document in page:
+ if document.url in urls:
+ try:
+ removed_by_domain[get_domain(document.url)] += 1
+ except ValueError:
+ removed_by_domain[document.url] += 1
+ index.store_in_page(page_index, kept)
+
+ if removed_by_domain:
+ logger.info("Purged %d documents across %d pages from %d URLs",
+ sum(removed_by_domain.values()), len(pages_to_urls), len(target_urls))
+ return dict(removed_by_domain)
diff --git a/mwmbl/indexer/purge_queue.py b/mwmbl/indexer/purge_queue.py
new file mode 100644
index 0000000..d6c390e
--- /dev/null
+++ b/mwmbl/indexer/purge_queue.py
@@ -0,0 +1,128 @@
+"""The hand-off between the search path and the background index purge.
+
+When retrieval filters a blacklisted document out of a result set, that document is still
+sitting in the index and will be re-filtered on every future query that touches its page.
+The search worker cannot remove it there and then - that is a read-modify-write of an
+index page - so it drops it on this queue and a background task
+(mwmbl.background.purge_blacklisted_from_queue) does the removal.
+
+Two properties matter:
+
+ * Enqueueing must never be able to affect a search response. Every Redis call here is
+ wrapped, and a failure is logged and swallowed. Losing an entry is harmless: the
+ document is still filtered out of the results, and the next query that surfaces it
+ enqueues it again. The whole loop is self-healing, which is also why it is safe that
+ production Redis runs allkeys-lru and can evict this key outright.
+ * It must not grow without bound. A Redis SET dedupes identical payloads for free -
+ the same popular blacklisted document surfacing on a thousand queries is one entry -
+ and MAX_QUEUE_SIZE caps the pathological case where the purge task is not running.
+
+The payload carries exactly the four fields tokenize_document() needs to work out which
+index pages a document is filed under. They are read straight out of the index, so the
+serialised JSON is byte-identical between retrievals and SET dedupe works on it.
+"""
+import json
+import threading
+import time
+from collections import OrderedDict
+from logging import getLogger
+from typing import Iterable, Optional
+
+import redis
+from django.conf import settings
+
+from mwmbl.tinysearchengine.indexer import Document
+
+logger = getLogger(__name__)
+
+
+PURGE_QUEUE_KEY = "blacklist:purge-queue"
+MAX_QUEUE_SIZE = 100_000
+
+# A blacklisted document stays in the index until the next purge run, so every query that
+# surfaces it in the meantime tries to queue it again - two Redis round trips to re-add
+# something already in the SET. This remembers what this process has queued recently and
+# skips those, which matters most for exactly the documents that matter most: the ones
+# turning up on popular queries. The TTL means a document whose purge failed or was lost
+# is retried rather than suppressed forever.
+_RECENTLY_ENQUEUED_TTL = 30 * 60
+_MAX_RECENTLY_ENQUEUED = 10_000
+_recently_enqueued: "OrderedDict[str, float]" = OrderedDict()
+_recently_enqueued_lock = threading.Lock()
+
+
+def _take_unqueued(payloads: list[str]) -> list[str]:
+ """Filter out payloads this process queued recently, and record the rest."""
+ now = time.monotonic()
+ with _recently_enqueued_lock:
+ while _recently_enqueued:
+ payload, queued_at = next(iter(_recently_enqueued.items()))
+ if now - queued_at < _RECENTLY_ENQUEUED_TTL:
+ break
+ _recently_enqueued.popitem(last=False)
+
+ fresh = [p for p in payloads if p not in _recently_enqueued]
+ for payload in fresh:
+ _recently_enqueued[payload] = now
+ while len(_recently_enqueued) > _MAX_RECENTLY_ENQUEUED:
+ _recently_enqueued.popitem(last=False)
+ return fresh
+
+
+_redis: Optional[redis.Redis] = None
+
+
+def get_redis() -> redis.Redis:
+ global _redis
+ if _redis is None:
+ _redis = redis.from_url(settings.REDIS_URL, decode_responses=True)
+ return _redis
+
+
+def _payload(document: Document) -> str:
+ return json.dumps({
+ "url": document.url,
+ "title": document.title,
+ "extract": document.extract,
+ "score": document.score,
+ }, sort_keys=True)
+
+
+def enqueue_for_purge(documents: Iterable[Document], redis_client: Optional[redis.Redis] = None) -> int:
+ """Queue these documents for removal from the index. Returns the number of *new*
+ entries added (already-queued duplicates do not count). Never raises."""
+ payloads = _take_unqueued([_payload(document) for document in documents])
+ if not payloads:
+ return 0
+
+ try:
+ client = redis_client if redis_client is not None else get_redis()
+ if client.scard(PURGE_QUEUE_KEY) >= MAX_QUEUE_SIZE:
+ logger.warning("Blacklist purge queue is full (%d); dropping %d documents",
+ MAX_QUEUE_SIZE, len(payloads))
+ return 0
+ return client.sadd(PURGE_QUEUE_KEY, *payloads)
+ except Exception:
+ logger.warning("Could not enqueue %d documents for purging", len(payloads), exc_info=True)
+ return 0
+
+
+def drain_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) -> list[Document]:
+ """Remove and return up to `limit` queued documents."""
+ client = redis_client if redis_client is not None else get_redis()
+ payloads = client.spop(PURGE_QUEUE_KEY, limit) or []
+
+ documents = []
+ for payload in payloads:
+ try:
+ fields = json.loads(payload)
+ documents.append(Document(title=fields["title"], url=fields["url"],
+ extract=fields["extract"], score=fields["score"]))
+ except (ValueError, KeyError, TypeError):
+ logger.warning("Discarding unreadable purge queue entry: %r", payload)
+ return documents
+
+
+def queue_size(redis_client: Optional[redis.Redis] = None) -> int:
+ client = redis_client if redis_client is not None else get_redis()
+ return client.scard(PURGE_QUEUE_KEY)
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 7a6786c..edad557 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -6,6 +6,7 @@ from django.core.cache import cache
from redis import Redis
from mwmbl.indexer.batch_cache import BatchCache
+from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
from mwmbl.models import DomainSubmission
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
@@ -34,6 +35,10 @@ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
+# Pull the blacklist snapshot in at worker startup rather than letting the first query
+# pay for the ~11 MB Redis read. Subsequent refreshes happen on a daemon thread.
+get_snapshot_blacklist()
+
ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
# Diversity is applied by the wrapping MMRRanker, which demotes (rather than drops)
# same-domain / near-duplicate results. Unwrap to disable diversity.
diff --git a/mwmbl/settings.py b/mwmbl/settings.py
index 018ba57..fd777c8 100644
--- a/mwmbl/settings.py
+++ b/mwmbl/settings.py
@@ -31,8 +31,17 @@ SCORE_FOR_DIFFERENT_DOMAIN = 1.0
SCORE_FOR_SAME_DOMAIN = 0.01
EXTRA_LINK_MULTIPLIER = 0.001
UNKNOWN_DOMAIN_MULTIPLIER = 0.001
-EXCLUDED_DOMAINS = {'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com', 'www.lukas-renggli.ch'}
-DOMAIN_BLACKLIST_REGEX = re.compile(r"porn|xxx|adult|jksu\.org|lwhyl\.org$|rgcd\.cn$|hzqwyou\.cn$|omgoat\.org$|pussyboy\.net$")
+EXCLUDED_DOMAINS = {
+ 'web.archive.org', 'forums.giantitp.com', 'www.crutchfield.com', 'plus.google.com', 'www.lukas-renggli.ch',
+ # Adult content sites found ranking for suspicious Search Console queries (2026-08-14).
+ 'fineartteens.com', 'milforia.com', 'azgals.com', 'kusowanka.com',
+}
+# `booru.org` is a free hosting platform for imageboards; every subdomain we've found ranking
+# (nyou/hypno/inflate/futaba/captions.booru.org) hosts adult/fetish content, so block the whole domain.
+DOMAIN_BLACKLIST_REGEX = re.compile(
+ r"porn|xxx|adult|hentai|\.booru\.org$|"
+ r"jksu\.org|lwhyl\.org$|rgcd\.cn$|hzqwyou\.cn$|omgoat\.org$|pussyboy\.net$"
+)
CORE_DOMAINS = {
'github.com',
'en.wikipedia.org',
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index b13d15e..f484188 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -313,3 +313,12 @@ SUPER_SEARCH_XGB_EPSILON = 0.1 # epsilon-greedy exploration rate
SUPER_SEARCH_XGB_MIN_TRAIN_ROWS = 2000 # (source, reward) pairs required before an online retrain
SUPER_SEARCH_XGB_TRAIN_WINDOW_DAYS = 90 # impression window for online retrains
SOURCE_PROVENANCE_MAX_DEPTH = 3 # max crawl hops a Super Search source propagates to descendant pages
+
+# Blacklisted-domain filtering on the search path. Set BLACKLIST_FILTER_AT_RETRIEVAL to
+# False to turn retrieval-time filtering off without a rollback; the index-time filter in
+# index_documents() is unaffected.
+BLACKLIST_FILTER_AT_RETRIEVAL = os.environ.get("BLACKLIST_FILTER_AT_RETRIEVAL", "true").lower() != "false"
+BLACKLIST_SNAPSHOT_CHECK_SECONDS = 300 # how often a worker checks Redis for a new snapshot
+BLACKLIST_SNAPSHOT_REFRESH_SECONDS = 6 * 60 * 60 # how often the snapshot is rebuilt from the remote lists
+BLACKLIST_PURGE_INTERVAL_SECONDS = 300 # how often the purge queue is drained
+BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index per purge run
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 81a58d9..b800c98 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -12,6 +12,7 @@ from pathlib import Path
from urllib.parse import urlparse
import numpy as np
+from django.conf import settings
from requests.adapters import HTTPAdapter, Retry
from mwmbl.format import get_query_regex
@@ -19,7 +20,7 @@ from mwmbl.hn_top_domains_filtered import DOMAINS
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
from mwmbl.tokenizer import tokenize, get_bigrams
-from mwmbl.utils import request_cache
+from mwmbl.utils import get_domain, request_cache
logger = getLogger(__name__)
@@ -207,6 +208,34 @@ def get_wiki_score(url):
return WIKI_SCORES.get(title, 0.0) / WIKI_MAX_SCORE
+def find_blacklisted_urls(documents: list[Document]) -> set[str]:
+ """URLs among these documents whose domain is blacklisted.
+
+ Enforcing the blacklist here, on the read path, is what actually guarantees a
+ blacklisted domain is never shown. The write-path checks (crawl assignment, link
+ discovery, index_documents) only stop *new* content: they do nothing about the
+ documents already in the index from before a domain was blacklisted, and the index is
+ far too large to sweep on demand. Filtering at retrieval makes a newly blacklisted
+ domain disappear from results immediately, and the purge queue then cleans up the
+ index in the background so the work is not repeated forever.
+
+ Cost is ~94us for a typical query's ~200 distinct domains - see blacklist_snapshot.
+ """
+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
+
+ domains_to_urls = defaultdict(set)
+ for document in documents:
+ try:
+ domain = get_domain(document.url)
+ except ValueError:
+ # Unparseable URL - keep it, matching index_batches.filter_blacklisted_documents.
+ continue
+ domains_to_urls[domain].add(document.url)
+
+ blacklisted_domains = get_snapshot_blacklist().filter_blacklisted(domains_to_urls)
+ return {url for domain in blacklisted_domains for url in domains_to_urls[domain]}
+
+
def deduplicate(results, seen_titles):
deduplicated_results = []
for result in results:
@@ -315,18 +344,53 @@ class Ranker:
pages += items
external_search_items = self.external_search(q)
- ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)
+ candidates = pages + additional_results + external_search_items
+ candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)
+
+ ordered_results = self.order_results(terms, candidates, is_complete)
deduplicated_results = deduplicate(curated_items + ordered_results, set())
state_fixed = [fix_document_state(result) for result in deduplicated_results]
return state_fixed, terms, completions
+ @staticmethod
+ def _remove_blacklisted(candidates: list[Document], curated_items: list[Document],
+ index_items: list[Document]) -> tuple[list[Document], list[Document]]:
+ """Drop blacklisted documents from a result set and queue the indexed ones for purging.
+
+ Curated items are filtered separately because they bypass order_results entirely -
+ they are prepended straight into deduplicate() - so filtering only `candidates`
+ would leave a hole for any blacklisted domain that had been curated for a term.
+
+ Only index-sourced documents go on the purge queue: there is nothing to purge for
+ a Wikipedia or Super Search result, and nothing to purge for the caller-supplied
+ additional_results either.
+ """
+ if not getattr(settings, "BLACKLIST_FILTER_AT_RETRIEVAL", True):
+ return candidates, curated_items
+
+ blacklisted_urls = find_blacklisted_urls(candidates + curated_items)
+ if not blacklisted_urls:
+ return candidates, curated_items
+
+ logger.info("Filtered %d blacklisted documents from results", len(blacklisted_urls))
+
+ from mwmbl.indexer.purge_queue import enqueue_for_purge
+ enqueue_for_purge([d for d in index_items if d.url in blacklisted_urls])
+
+ return ([d for d in candidates if d.url not in blacklisted_urls],
+ [d for d in curated_items if d.url not in blacklisted_urls])
+
def external_search(self, q: str):
return []
def get_raw_results(self, query: str):
tokens = tokenize(query)
term = ' '.join(tokens)
- return self.tiny_index.retrieve(term)
+ results = self.tiny_index.retrieve(term)
+ # /raw bypasses get_results(), so it needs its own filter or it is a way to read
+ # blacklisted documents straight out of the index.
+ filtered, _ = self._remove_blacklisted(results, [], index_items=results)
+ return filtered
class HeuristicRanker(Ranker):
diff --git a/test/test_apps_background_scheduling.py b/test/test_apps_background_scheduling.py
new file mode 100644
index 0000000..62af95f
--- /dev/null
+++ b/test/test_apps_background_scheduling.py
@@ -0,0 +1,39 @@
+"""The periodic tasks only exist if apps.ready() registers them, and registering the same
+task twice means it runs twice as often on every deploy."""
+import pytest
+from background_task.models import Task
+from django.conf import settings
+
+from mwmbl.apps import MwmblConfig
+
+BLACKLIST_TASKS = {
+ "mwmbl.background.refresh_blacklist_snapshot",
+ "mwmbl.background.purge_blacklisted_from_queue",
+}
+
+
+def test_blacklist_tasks_are_scheduled():
+ MwmblConfig._schedule_background_tasks()
+
+ assert BLACKLIST_TASKS <= set(Task.objects.values_list("task_name", flat=True))
+
+
+def test_scheduling_twice_does_not_duplicate_tasks():
+ MwmblConfig._schedule_background_tasks()
+ MwmblConfig._schedule_background_tasks()
+
+ for task_name in BLACKLIST_TASKS:
+ assert Task.objects.filter(task_name=task_name).count() == 1
+
+
+def test_blacklist_tasks_repeat_at_the_configured_intervals():
+ MwmblConfig._schedule_background_tasks()
+
+ snapshot = Task.objects.get(task_name="mwmbl.background.refresh_blacklist_snapshot")
+ purge = Task.objects.get(task_name="mwmbl.background.purge_blacklisted_from_queue")
+
+ assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS
+ assert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDS
diff --git a/test/test_blacklist.py b/test/test_blacklist.py
index 5088b57..9225c22 100644
--- a/test/test_blacklist.py
+++ b/test/test_blacklist.py
@@ -5,8 +5,6 @@ def test_blacklist_excludes_bad_pattern():
"""Test that built-in rules blacklist bad patterns."""
provider = BuiltInRulesBlacklistProvider()
bad_domains = [
- "brofqpxj.uelinc.com",
- "gzsmjc.fba01.com",
"59648.etnomurcia.com",
"something.hzqwyou.cn",
]
@@ -15,6 +13,23 @@ def test_blacklist_excludes_bad_pattern():
assert provider.is_domain_blacklisted(domain)
+def test_blacklist_allows_numeric_apex_domain():
+ """A numeric *apex* domain like 350.org is the actual site name, not a generated
+ subdomain - the numeric-subdomain heuristic below should only catch len > 2."""
+ provider = BuiltInRulesBlacklistProvider()
+ assert not provider.is_domain_blacklisted("350.org")
+
+
+def test_blacklist_allows_ordinary_short_subdomains():
+ """Regression test: a length-based heuristic that used to flag any 3-part .com
+ domain with a 6 or 8 character first label was removed after it flagged dozens of
+ legitimate subdomains (topics.nytimes.com, search.google.com, podcasts.apple.com,
+ starwars.fandom.com, ...) with zero confirmed true positives in testing."""
+ provider = BuiltInRulesBlacklistProvider()
+ for domain in ["topics.nytimes.com", "search.google.com", "podcasts.apple.com", "starwars.fandom.com"]:
+ assert not provider.is_domain_blacklisted(domain), domain
+
+
def test_blacklist_allows_top_domains():
"""Test that built-in rules allow legitimate domains."""
provider = BuiltInRulesBlacklistProvider()
@@ -25,3 +40,27 @@ def test_blacklist_allows_other_domains():
"""Test that built-in rules allow other legitimate domains."""
provider = BuiltInRulesBlacklistProvider()
assert not provider.is_domain_blacklisted("something.com")
+
+
+def test_blacklist_excludes_adult_sites_found_via_search_console():
+ """Domains surfaced by suspicious Search Console queries (2026-08-14)."""
+ provider = BuiltInRulesBlacklistProvider()
+ bad_domains = [
+ "fineartteens.com",
+ "milforia.com",
+ "azgals.com",
+ "kusowanka.com",
+ "nyou.booru.org",
+ "hypno.booru.org",
+ "inflatebooru.booru.org",
+ "futabooru.booru.org",
+ "captions.booru.org",
+ ]
+ for domain in bad_domains:
+ assert provider.is_domain_blacklisted(domain), domain
+
+
+def test_blacklist_allows_legitimate_imageboard():
+ """mathchan.org ('the scientific imageboard') is not adult content."""
+ provider = BuiltInRulesBlacklistProvider()
+ assert not provider.is_domain_blacklisted("mathchan.org")
diff --git a/test/test_blacklist_providers.py b/test/test_blacklist_providers.py
index e0284ca..a19dea7 100644
--- a/test/test_blacklist_providers.py
+++ b/test/test_blacklist_providers.py
@@ -3,16 +3,30 @@ Tests for blacklist providers and the abstraction system.
"""
import pytest
+import requests
from unittest.mock import patch, MagicMock
+from mwmbl.indexer import blacklist_providers
from mwmbl.indexer.blacklist_providers import (
- StaticBlacklistProvider,
- HaGeZiBlacklistProvider,
+ StaticBlacklistProvider,
+ HaGeZiBlacklistProvider,
+ AdultContentBlacklistProvider,
CombinedBlacklistProvider
)
from mwmbl.indexer.blacklist import get_default_blacklist_provider
[email protected](autouse=True)
+def clear_shared_domains_cache():
+ """_parsed_domains_cache is a module-level dict shared across every
+ RemoteListBlacklistProvider instance (see blacklist_providers.py) so that
+ index_documents() can call get_default_blacklist_provider() cheaply on every call.
+ Clear it around each test so a mocked response in one test can't leak into another."""
+ blacklist_providers._parsed_domains_cache.clear()
+ yield
+ blacklist_providers._parsed_domains_cache.clear()
+
+
def test_static_blacklist_provider():
"""Test StaticBlacklistProvider with is_domain_blacklisted method."""
test_domains = {'spam.com', 'malware.example'}
@@ -28,8 +42,7 @@ def test_static_blacklist_provider():
def test_hagezi_blacklist_provider_success():
"""Test HaGeZiBlacklistProvider with successful HTTP response."""
- with patch('mwmbl.indexer.blacklist_providers.request_cache') as mock_cache:
- mock_session = MagicMock()
+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:
mock_response = MagicMock()
mock_response.text = '''# HaGeZi DNS Blocklist
# Comments should be ignored
@@ -39,11 +52,10 @@ malware.example
# Another comment
badsite.net
'''
- mock_session.get.return_value = mock_response
- mock_cache.return_value.__enter__.return_value = mock_session
-
+ mock_get.return_value = mock_response
+
provider = HaGeZiBlacklistProvider('light')
-
+
# Test domains that should be blacklisted
assert provider.is_domain_blacklisted('spam.com') == True
assert provider.is_domain_blacklisted('malware.example') == True
@@ -53,6 +65,25 @@ badsite.net
assert provider.is_domain_blacklisted('github.com') == False
+def test_adult_content_blacklist_provider_success():
+ """Test AdultContentBlacklistProvider parses hosts-format lines correctly."""
+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:
+ mock_response = MagicMock()
+ mock_response.text = '''# Title: Porn Block List
+# Homepage: https://github.com/blocklistproject/Lists
+
+0.0.0.0 fineartteens.com
+0.0.0.0 milforia.com
+'''
+ mock_get.return_value = mock_response
+
+ provider = AdultContentBlacklistProvider()
+
+ assert provider.is_domain_blacklisted('fineartteens.com') == True
+ assert provider.is_domain_blacklisted('milforia.com') == True
+ assert provider.is_domain_blacklisted('github.com') == False
+
+
def test_hagezi_blacklist_provider_invalid_type():
"""Test HaGeZiBlacklistProvider rejects invalid list types."""
with pytest.raises(ValueError):
@@ -100,3 +131,43 @@ def test_integration_with_blacklist_module():
# Test with a domain that should not be blacklisted
assert default_provider.is_domain_blacklisted('github.com') == False
+
+
+def test_remote_lists_do_not_use_the_shared_filesystem_request_cache():
+ """These lists are tens of megabytes. mwmbl.utils.request_cache is a *filesystem* cache
+ rooted at settings.REQUEST_CACHE_PATH, on the same volume as the index - caching them
+ there fills that volume, and once it is full every other user of request_cache breaks,
+ including get_wiki_results() on the live search path (which then retries a live fetch
+ 4x per uncached query until the worker dies). Regression test: fetch must not go
+ through request_cache."""
+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:
+ mock_response = MagicMock()
+ mock_response.text = "spam.com\n"
+ mock_get.return_value = mock_response
+
+ # If the module still imported request_cache, patching it would succeed; assert the
+ # name is gone from the module so this can't silently regress.
+ import mwmbl.indexer.blacklist_providers as bp
+ assert not hasattr(bp, "request_cache"), \
+ "blacklist_providers must not use the shared filesystem request_cache"
+
+ provider = HaGeZiBlacklistProvider('light')
+ assert provider.is_domain_blacklisted('spam.com') is True
+ assert mock_get.called
+
+
+def test_fetch_failure_falls_back_to_last_good_list():
+ """A transient fetch failure must not silently blank out the provider's coverage."""
+ with patch('mwmbl.indexer.blacklist_providers.requests.get') as mock_get:
+ mock_response = MagicMock()
+ mock_response.text = "spam.com\n"
+ mock_get.return_value = mock_response
+ provider = HaGeZiBlacklistProvider('light')
+ assert provider.is_domain_blacklisted('spam.com') is True
+
+ # Expire the module-level cache so the next call refetches, and make that fetch fail.
+ blacklist_providers._parsed_domains_cache[provider.url] = (
+ -10**9, blacklist_providers._parsed_domains_cache[provider.url][1])
+ with patch('mwmbl.indexer.blacklist_providers.requests.get',
+ side_effect=requests.RequestException("boom")):
+ assert provider.is_domain_blacklisted('spam.com') is True # still covered
diff --git a/test/test_blacklist_snapshot.py b/test/test_blacklist_snapshot.py
new file mode 100644
index 0000000..ca7fd3c
--- /dev/null
+++ b/test/test_blacklist_snapshot.py
@@ -0,0 +1,163 @@
+import fakeredis
+import numpy as np
+import pytest
+
+from mwmbl.indexer import blacklist_snapshot
+from mwmbl.indexer.blacklist_snapshot import (
+ SNAPSHOT_KEY,
+ SNAPSHOT_VERSION_KEY,
+ SnapshotBlacklist,
+ build_snapshot,
+ collect_remote_domains,
+ publish_snapshot,
+ refresh_snapshot,
+)
+from mwmbl.indexer.blacklist_providers import (
+ CombinedBlacklistProvider,
+ RemoteListBlacklistProvider,
+ StaticBlacklistProvider,
+)
+
+
+class FakeRemoteProvider(RemoteListBlacklistProvider):
+ """A remote list provider that serves a fixed set instead of fetching."""
+
+ def __init__(self, domains):
+ super().__init__(f"http://fake/{id(self)}")
+ self._domains = set(domains)
+
+ def _get_blacklisted_domains(self):
+ return self._domains
+
+
+def redis_client():
+ return fakeredis.FakeRedis()
+
+
+def provider():
+ return CombinedBlacklistProvider([
+ StaticBlacklistProvider({"not-remote.test"}),
+ FakeRemoteProvider({"badsite.test", "evil.test"}),
+ FakeRemoteProvider({"evil.test", "spam.test"}),
+ ])
+
+
+def test_collect_remote_domains_walks_combined_providers_and_skips_local_ones(provider):
+ # Only remote lists are snapshotted; local rule providers are evaluated in-process,
+ # so "not-remote.test" must not appear here.
+ assert collect_remote_domains(provider) == {"badsite.test", "evil.test", "spam.test"}
+
+
+def test_build_snapshot_is_sorted_and_deduplicated(provider):
+ array = np.frombuffer(build_snapshot(provider), dtype=blacklist_snapshot.HASH_DTYPE)
+ assert len(array) == 3 # evil.test appears in both lists
+ assert list(array) == sorted(array)
+
+
+def test_publish_and_load_round_trip(provider, redis_client):
+ version = refresh_snapshot(provider, redis_client)
+
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=redis_client)
+ assert blacklist.load_now() is True
+ assert blacklist.filter_blacklisted(
+ ["badsite.test", "evil.test", "spam.test", "good.test"]
+ ) == {"badsite.test", "evil.test", "spam.test"}
+ assert redis_client.get(SNAPSHOT_VERSION_KEY).decode() == version
+
+
+def test_load_is_a_no_op_when_the_version_is_unchanged(provider, redis_client):
+ refresh_snapshot(provider, redis_client)
+ blacklist = SnapshotBlacklist(redis_client=redis_client)
+ assert blacklist.load_now() is True
+ assert blacklist.load_now() is False
+
+
+def test_load_picks_up_a_republished_snapshot(redis_client):
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=redis_client)
+
+ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"first.test"})]), redis_client)
+ blacklist.load_now()
+ assert blacklist.filter_blacklisted(["first.test", "second.test"]) == {"first.test"}
+
+ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"second.test"})]), redis_client)
+ assert blacklist.load_now() is True
+ assert blacklist.filter_blacklisted(["first.test", "second.test"]) == {"second.test"}
+
+
+def test_built_in_rules_apply_without_a_snapshot(redis_client):
+ """The local rules must work even before the first snapshot lands, so a domain added
+ to mwmbl/settings.py takes effect on deploy rather than at the next refresh."""
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({"local.test"}),
+ redis_client=redis_client)
+
+ assert blacklist.loaded is False
+ assert blacklist.filter_blacklisted(["local.test", "good.test"]) == {"local.test"}
+
+
+def test_eviction_keeps_the_loaded_snapshot(provider, redis_client):
+ """Production Redis runs allkeys-lru, so the snapshot can vanish. Losing it must not
+ silently disable filtering."""
+ refresh_snapshot(provider, redis_client)
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=redis_client)
+ blacklist.load_now()
+
+ redis_client.delete(SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY)
+
+ assert blacklist.load_now() is False
+ assert blacklist.filter_blacklisted(["badsite.test", "good.test"]) == {"badsite.test"}
+
+
+def test_redis_failure_keeps_the_loaded_snapshot(provider):
+ class BrokenRedis:
+ def get(self, key):
+ raise ConnectionError("redis is down")
+
+ refresh_snapshot(provider, (working := fakeredis.FakeRedis()))
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=working)
+ blacklist.load_now()
+
+ blacklist._redis = BrokenRedis()
+ assert blacklist.load_now() is False
+ assert blacklist.filter_blacklisted(["badsite.test"]) == {"badsite.test"}
+
+
+def test_publishing_the_same_domains_twice_keeps_the_same_version(provider, redis_client):
+ """Workers skip the 11 MB download when the version is unchanged, so an unchanged
+ blocklist must not churn the version."""
+ assert refresh_snapshot(provider, redis_client) == refresh_snapshot(provider, redis_client)
+
+
+def test_filter_blacklisted_handles_an_empty_input(provider, redis_client):
+ blacklist = SnapshotBlacklist(redis_client=redis_client)
+ publish_snapshot(build_snapshot(provider), redis_client)
+ blacklist.load_now()
+
+ assert blacklist.filter_blacklisted([]) == set()
+
+
+def test_is_domain_blacklisted(provider, redis_client):
+ refresh_snapshot(provider, redis_client)
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=redis_client)
+ blacklist.load_now()
+
+ assert blacklist.is_domain_blacklisted("badsite.test") is True
+ assert blacklist.is_domain_blacklisted("good.test") is False
+
+
+def test_hashes_above_every_entry_do_not_index_out_of_bounds(redis_client):
+ """np.searchsorted returns len(array) for a hash larger than everything in it."""
+ refresh_snapshot(CombinedBlacklistProvider([FakeRemoteProvider({"only.test"})]), redis_client)
+ blacklist = SnapshotBlacklist(built_in_rules=StaticBlacklistProvider(set()),
+ redis_client=redis_client)
+ blacklist.load_now()
+
+ # Try enough domains that at least one hashes above and one below the single entry.
+ candidates = [f"candidate{i}.test" for i in range(100)]
+ assert blacklist.filter_blacklisted(candidates) == set()
diff --git a/test/test_index_batches.py b/test/test_index_batches.py
index bc262b6..9137d0d 100644
--- a/test/test_index_batches.py
+++ b/test/test_index_batches.py
@@ -1,11 +1,15 @@
from pathlib import Path
from tempfile import TemporaryDirectory
+from unittest.mock import patch
+import pytest
+
+from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
from mwmbl.indexer.index_batches import (
sort_documents, combine_documents, _merge_user_ids, MAX_USER_IDS,
- index_results_against_query,
+ index_results_against_query, index_documents,
)
-from mwmbl.tinysearchengine.indexer import Document, DocumentState, TinyIndex
+from mwmbl.tinysearchengine.indexer import Document, DocumentState, PAGE_SIZE, TinyIndex
class UrlRanker:
@@ -200,3 +204,54 @@ def test_combine_documents_propagates_user_ids_to_winner():
assert len(combined) == 1
assert 1 in combined[0].user_ids
assert 2 in combined[0].user_ids
+
+
+# ---------------------------------------------------------------------------
+# index_documents: blacklist enforcement
+# ---------------------------------------------------------------------------
+#
+# index_documents() is the common choke point for every path that adds content to the
+# index (offline batch processing, the trusted-crawler POST /results endpoint, the
+# standalone crawl tool) - crawling/link-discovery only stop *new* crawling of a
+# blacklisted domain, they don't stop a submitted batch that already contains one of its
+# pages, so indexing needs its own blacklist check.
+
+PATCH_TARGET = "mwmbl.indexer.index_batches.get_default_blacklist_provider"
+
+
+def index_path(tmp_path):
+ path = tmp_path / "test.tinysearch"
+ TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=10, page_size=PAGE_SIZE)
+ return str(path)
+
+
+def _all_urls(index_path, num_pages=10):
+ urls = []
+ with TinyIndex(item_factory=Document, index_path=index_path, mode="r") as index:
+ for page_index in range(num_pages):
+ urls.extend(doc.url for doc in index.get_page(page_index))
+ return urls
+
+
+def test_index_documents_skips_blacklisted_domain(index_path):
+ documents = [
+ Document(title="Bad", url="https://fineartteens.com/x", extract="teen gallery"),
+ Document(title="Good", url="https://example.com/y", extract="a good page"),
+ ]
+
+ with patch(PATCH_TARGET, return_value=StaticBlacklistProvider({"fineartteens.com"})):
+ index_documents(documents, index_path)
+
+ urls = _all_urls(index_path)
+ assert "https://fineartteens.com/x" not in urls
+ assert "https://example.com/y" in urls
+
+
+def test_index_documents_keeps_everything_when_nothing_blacklisted(index_path):
+ documents = [Document(title="Good", url="https://example.com/y", extract="a good page")]
+
+ with patch(PATCH_TARGET, return_value=StaticBlacklistProvider(set())):
+ index_documents(documents, index_path)
+
+ assert "https://example.com/y" in _all_urls(index_path)
diff --git a/test/test_purge_blacklisted.py b/test/test_purge_blacklisted.py
new file mode 100644
index 0000000..7c2ff90
--- /dev/null
+++ b/test/test_purge_blacklisted.py
@@ -0,0 +1,106 @@
+import pytest
+
+from mwmbl.indexer.index import tokenize_document
+from mwmbl.indexer.index_batches import index_pages, preprocess_documents
+from mwmbl.indexer.purge_blacklisted import pages_for_document, purge_documents
+from mwmbl.tinysearchengine.indexer import Document, PAGE_SIZE, TinyIndex
+
+NUM_PAGES = 32
+
+BAD = Document(title="Bad page about apples", url="https://badsite.test/apples",
+ extract="apples and oranges and bananas", score=1.0)
+GOOD = Document(title="Good page about apples", url="https://example.test/apples",
+ extract="apples and oranges and bananas", score=1.0)
+
+
+def index_path(tmp_path):
+ path = str(tmp_path / "purge.tinysearch")
+ TinyIndex.create(item_factory=Document, index_path=path, num_pages=NUM_PAGES, page_size=PAGE_SIZE)
+ return path
+
+
+def index(index_path, documents):
+ index_pages(index_path, preprocess_documents(documents, index_path))
+
+
+def urls_in_index(index_path):
+ with TinyIndex(Document, index_path, 'r') as index:
+ return [d.url for page in range(NUM_PAGES) for d in index.get_page(page)]
+
+
+def test_purge_removes_a_document_from_every_page_it_is_filed_under(index_path):
+ index(index_path, [BAD, GOOD])
+
+ with TinyIndex(Document, index_path, 'w') as tiny_index:
+ # The document must be filed under several terms, or this proves nothing.
+ assert len(pages_for_document(tiny_index, BAD)) > 1
+ removed = purge_documents(tiny_index, [BAD])
+
+ assert removed == {"badsite.test": len(pages_for_document_count(index_path, BAD))}
+ assert "https://badsite.test/apples" not in urls_in_index(index_path)
+
+
+def pages_for_document_count(index_path, document):
+ with TinyIndex(Document, index_path, 'r') as tiny_index:
+ return pages_for_document(tiny_index, document)
+
+
+def test_purge_leaves_other_documents_on_shared_pages_alone(index_path):
+ """The bad and good documents share almost all their tokens, so they share pages.
+ Purging must remove by URL, not by re-testing the page against the blacklist -
+ otherwise it sweeps up documents it was never asked to touch."""
+ index(index_path, [BAD, GOOD])
+
+ with TinyIndex(Document, index_path, 'w') as tiny_index:
+ shared = pages_for_document(tiny_index, BAD) & pages_for_document(tiny_index, GOOD)
+ assert shared, "expected these documents to share at least one page"
+ purge_documents(tiny_index, [BAD])
+
+ urls = urls_in_index(index_path)
+ assert "https://badsite.test/apples" not in urls
+ assert "https://example.test/apples" in urls
+
+
+def test_purge_re_checks_the_blacklist_before_removing(index_path):
+ """A domain taken off the blacklist between being queued and being purged must
+ survive."""
+ index(index_path, [BAD, GOOD])
+
+ with TinyIndex(Document, index_path, 'w') as tiny_index:
+ removed = purge_documents(tiny_index, [BAD], is_blacklisted=lambda domain: False)
+
+ assert removed == {}
+ assert "https://badsite.test/apples" in urls_in_index(index_path)
+
+
+def test_purge_removes_only_the_documents_that_are_still_blacklisted(index_path):
+ index(index_path, [BAD, GOOD])
+
+ with TinyIndex(Document, index_path, 'w') as tiny_index:
+ removed = purge_documents(tiny_index, [BAD, GOOD],
+ is_blacklisted=lambda domain: domain == "badsite.test")
+
+ assert set(removed) == {"badsite.test"}
+ urls = urls_in_index(index_path)
+ assert "https://badsite.test/apples" not in urls
+ assert "https://example.test/apples" in urls
+
+
+def test_purge_of_an_absent_document_is_a_no_op(index_path):
+ index(index_path, [GOOD])
+ absent = Document(title="Nowhere", url="https://badsite.test/nowhere", extract="nothing", score=1.0)
+
+ with TinyIndex(Document, index_path, 'w') as tiny_index:
+ assert purge_documents(tiny_index, [absent]) == {}
+
+ assert "https://example.test/apples" in urls_in_index(index_path)
+
+
+def test_pages_for_document_matches_the_indexer(index_path):
+ """purge_documents relies on tokenize_document being a pure function of the stored
+ fields, so that recomputing it recovers exactly the pages the indexer wrote to."""
+ with TinyIndex(Document, index_path, 'r') as tiny_index:
+ tokens = tokenize_document(BAD.url, BAD.title, BAD.extract, BAD.score).tokens
+ expected = {tiny_index.get_key_page_index(token) for token in tokens}
+ assert pages_for_document(tiny_index, BAD) == expected
diff --git a/test/test_rank_blacklist.py b/test/test_rank_blacklist.py
new file mode 100644
index 0000000..b5b07af
--- /dev/null
+++ b/test/test_rank_blacklist.py
@@ -0,0 +1,180 @@
+"""Retrieval-time blacklist filtering: the guarantee that a blacklisted domain is never
+shown, plus the enqueueing that lets the background purge clean the index up afterwards."""
+from unittest.mock import patch
+
+import fakeredis
+import pytest
+
+from mwmbl.indexer import purge_queue
+from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
+from mwmbl.indexer.blacklist_snapshot import SnapshotBlacklist
+from mwmbl.indexer.purge_queue import PURGE_QUEUE_KEY, drain_purge_queue
+from mwmbl.tinysearchengine.indexer import Document, DocumentState
+from mwmbl.tinysearchengine.rank import HeuristicRanker
+
+BLACKLISTED_DOMAIN = "badsite.test"
+
+BAD = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/bananas",
+ extract="bananas and apples", score=1.0)
+GOOD = Document(title="Bananas", url="https://example.test/bananas",
+ extract="bananas and apples", score=1.0)
+
+
+class FakeIndex:
+ """Returns the same documents for every term, like a page every query term hits."""
+
+ def __init__(self, documents):
+ self.documents = documents
+
+ def retrieve(self, key):
+ return list(self.documents)
+
+
+class FakeCompleter:
+ def complete(self, term):
+ return []
+
+
[email protected](autouse=True)
+def clear_recently_enqueued():
+ """purge_queue keeps a process-level record of what it has already queued, so without
+ this each test would inherit the previous one's suppressions."""
+ purge_queue._recently_enqueued.clear()
+ yield
+ purge_queue._recently_enqueued.clear()
+
+
+def redis_client():
+ return fakeredis.FakeRedis(decode_responses=True)
+
+
+def blacklist(redis_client):
+ """A snapshot-free blacklist: the built-in rules alone are enough to decide here, so
+ these tests exercise the ranker rather than the snapshot machinery."""
+ return SnapshotBlacklist(built_in_rules=StaticBlacklistProvider({BLACKLISTED_DOMAIN}),
+ redis_client=redis_client)
+
+
+def ranker(blacklist, redis_client, request):
+ documents = getattr(request, "param", [BAD, GOOD])
+ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \
+ patch.object(purge_queue, "get_redis", return_value=redis_client):
+ yield HeuristicRanker(FakeIndex(documents), FakeCompleter())
+
+
+def test_search_drops_blacklisted_results(ranker):
+ urls = [result.url for result in ranker.search("bananas", [])]
+
+ assert BAD.url not in urls
+ assert GOOD.url in urls
+
+
+def test_search_queues_blacklisted_results_for_purging(ranker, redis_client):
+ ranker.search("bananas", [])
+
+ queued = drain_purge_queue(10, redis_client)
+ assert [d.url for d in queued] == [BAD.url]
+ # The queued payload carries what tokenize_document needs to find the document's pages
+ assert queued[0].title == BAD.title
+ assert queued[0].extract == BAD.extract
+
+
+def test_a_clean_result_set_queues_nothing(ranker, redis_client):
+ with patch.object(ranker, "tiny_index", FakeIndex([GOOD])):
+ assert [r.url for r in ranker.search("bananas", [])] == [GOOD.url]
+
+ assert redis_client.scard(PURGE_QUEUE_KEY) == 0
+
+
+def test_the_same_document_is_queued_once_across_queries(ranker, redis_client):
+ """A blacklisted document stays in the index until the purge runs, so it keeps
+ turning up. It must neither pile up in the queue nor cost a Redis round trip on
+ every query in the meantime."""
+ for _ in range(5):
+ ranker.search("bananas", [])
+
+ assert redis_client.scard(PURGE_QUEUE_KEY) == 1
+ assert len(purge_queue._recently_enqueued) == 1
+
+
+def test_a_document_is_re_queued_once_the_suppression_expires(ranker, redis_client):
+ """If a purge is lost the document has to get another chance, or it stays in the
+ index for as long as this process lives."""
+ ranker.search("bananas", [])
+ assert drain_purge_queue(10, redis_client)
+
+ purge_queue._recently_enqueued.clear() # stands in for the TTL expiring
+ ranker.search("bananas", [])
+
+ assert [d.url for d in drain_purge_queue(10, redis_client)] == [BAD.url]
+
+
+def test_get_raw_results_drops_blacklisted_results(ranker):
+ """/raw bypasses get_results(), so it needs filtering of its own."""
+ urls = [result.url for result in ranker.get_raw_results("bananas")]
+
+ assert BAD.url not in urls
+ assert GOOD.url in urls
+
+
+def test_complete_does_not_suggest_blacklisted_urls(ranker):
+ _, completions = ranker.complete("banana")
+
+ assert not any(BLACKLISTED_DOMAIN in completion for completion in completions)
+
+
+def test_curated_blacklisted_documents_are_dropped(blacklist, redis_client):
+ """Curated items bypass order_results entirely - they are prepended straight into
+ deduplicate() - so they need filtering separately from the ranked candidates."""
+ curated = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/curated",
+ extract="bananas", score=1.0, term="bananas",
+ state=DocumentState.ORGANIC_APPROVED)
+
+ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \
+ patch.object(purge_queue, "get_redis", return_value=redis_client):
+ ranker = HeuristicRanker(FakeIndex([curated, GOOD]), FakeCompleter())
+ urls = [result.url for result in ranker.search("bananas", [])]
+
+ assert curated.url not in urls
+ assert GOOD.url in urls
+
+
+def test_additional_results_are_filtered_but_not_queued(blacklist, redis_client):
+ """Caller-supplied results (the Google-sourced documents the search page passes in)
+ must be filtered too, but there is nothing in our index to purge for them."""
+ additional = Document(title="Bananas", url=f"https://{BLACKLISTED_DOMAIN}/from-google",
+ extract="bananas", score=1.0, state=DocumentState.FROM_GOOGLE)
+
+ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \
+ patch.object(purge_queue, "get_redis", return_value=redis_client):
+ ranker = HeuristicRanker(FakeIndex([GOOD]), FakeCompleter())
+ urls = [result.url for result in ranker.search("bananas", [additional])]
+
+ assert additional.url not in urls
+ assert redis_client.scard(PURGE_QUEUE_KEY) == 0
+
+
+def test_filtering_can_be_switched_off(ranker, settings):
+ """The kill switch has to work without a rollback if filtering misbehaves in prod."""
+ settings.BLACKLIST_FILTER_AT_RETRIEVAL = False
+
+ assert BAD.url in [result.url for result in ranker.search("bananas", [])]
+
+
+def test_a_redis_failure_does_not_break_search(blacklist):
+ """Enqueueing is best-effort: losing an entry just means the next query re-queues it,
+ but a search response must never fail because of it."""
+ class BrokenRedis:
+ def scard(self, key):
+ raise ConnectionError("redis is down")
+
+ with patch("mwmbl.indexer.blacklist_snapshot.get_snapshot_blacklist", return_value=blacklist), \
+ patch.object(purge_queue, "get_redis", return_value=BrokenRedis()):
+ ranker = HeuristicRanker(FakeIndex([BAD, GOOD]), FakeCompleter())
+ urls = [result.url for result in ranker.search("bananas", [])]
+
+ assert BAD.url not in urls
+ assert GOOD.url in urls