An open source, non-profit web search engine
Merge main into filter-blacklisted-at-retrieval
Daoud Clarkee76b034b25b6
diff --cc mwmbl/apps.pyindex 60a17be,decf3e0..524d2db--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@@ -65,33 -70,36 +70,54 @@@ class MwmblConfig(AppConfig)except RuntimeError:pass+ # The gunicorn server runs multiple worker processes, each of which calls+ # this method independently on startup. The "does a Task row already+ # exist?" checks below are plain reads with no transaction or unique+ # constraint backing them, so without this lock, workers starting at the+ # same time (every deploy/restart) can race past the check before any of+ # them commits its Task row, each creating its own duplicate periodic+ # task. cache.add() is atomic (same pattern as the rate limiter in+ # mwmbl.quota), so only the first worker to reach this point proceeds;+ # the rest skip immediately, and the lock's TTL simply bounds how long+ # a crashed winner blocks a retry by a later-starting worker.+ from django.core.cache import cache+ if not cache.add(MwmblConfig._SCHEDULE_LOCK_KEY, "1", timeout=MwmblConfig._SCHEDULE_LOCK_TIMEOUT):+ log.info("Skipping background task scheduling: another worker is already handling it.")+ return+try:from background_task.models import Task- from mwmbl.background import sync_search_counts, report_usage_to_polar+ from mwmbl.background import (- purge_blacklisted_from_queue, refresh_blacklist_snapshot, sync_search_counts,++ purge_blacklisted_from_queue, refresh_blacklist_snapshot, report_usage_to_polar,++ sync_search_counts,+ )SYNC_TASK = "mwmbl.background.sync_search_counts"+ POLAR_REPORT_TASK = "mwmbl.background.report_usage_to_polar"+ BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"+ BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"# Sync search counts once per hour (3600 seconds)if not Task.objects.filter(task_name=SYNC_TASK).exists():sync_search_counts(repeat=3600, repeat_until=None)+ # Report billable usage overage to Polar once per hour (3600 seconds)+ if not Task.objects.filter(task_name=POLAR_REPORT_TASK).exists():+ report_usage_to_polar(repeat=3600, repeat_until=None)++ # Rebuild the blacklist snapshot the search path filters against. schedule=0+ # on the task means the first run happens immediately rather than one full+ # refresh interval after deploy, which matters because until it lands the+ # search workers have only the built-in rules to go on.+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():+ refresh_blacklist_snapshot(+ repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)++ # Drain the queue of blacklisted documents that retrieval has filtered out+ if not Task.objects.filter(task_name=BLACKLIST_PURGE_TASK).exists():+ purge_blacklisted_from_queue(+ repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)+except Exception:# Don't prevent startup if background task scheduling failslog.exception("Failed to schedule background tasks")diff --cc mwmbl/background.pyindex 062bb0d,8b9b282..5b5cedb--- a/mwmbl/background.py+++ b/mwmbl/background.py@@@ -1,10 -1,9 +1,11 @@@"""Script that updates data in a background process.-Also contains Django Background Tasks for periodic quota maintenance:+Also contains Django Background Tasks for periodic maintenance:- sync_search_counts: syncs Redis monthly counters → DB once per hour+ - report_usage_to_polar: reports billable usage overage to Polar once per hour+ - refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against+ - purge_blacklisted_from_queue: removes retrieval-filtered documents from the index"""import loggingimport sys@@@ -133,48 -133,49 +135,94 @@@ def sync_search_counts()logger.exception("Error syncing search count for key %s", key)+# ---------------------------------------------------------------------------+# Blacklisted-domain maintenance (Django Background Tasks)+# ---------------------------------------------------------------------------++@background(schedule=0)+def refresh_blacklist_snapshot():+ """+ Rebuild the blacklist snapshot that the search workers filter against.++ This is the only place the multi-megabyte remote blocklists are downloaded and+ parsed. Search workers only ever read the ~11 MB hash array this publishes to Redis,+ so no query ever waits on a blocklist fetch. See mwmbl.indexer.blacklist_snapshot.+ """+ from mwmbl.indexer.blacklist_snapshot import refresh_snapshot++ refresh_snapshot()+++@background(schedule=0)+def purge_blacklisted_from_queue():+ """+ Remove the documents that retrieval filtered out from the index itself.++ Retrieval already guarantees these are never shown; this closes the loop so the same+ documents are not re-filtered on every future query. The queue is best-effort, and+ that is fine: anything lost is re-queued the next time a query surfaces it.+ """+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist+ from mwmbl.indexer.purge_blacklisted import purge_documents+ from mwmbl.indexer.purge_queue import drain_purge_queue, queue_size+ from mwmbl.tinysearchengine.indexer import Document, TinyIndex++ documents = drain_purge_queue(settings.BLACKLIST_PURGE_BATCH_SIZE)+ if not documents:+ return++ blacklist = get_snapshot_blacklist()+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME+ with TinyIndex(Document, str(index_path), 'w') as index:+ removed_by_domain = purge_documents(index, documents, blacklist.is_domain_blacklisted)++ logger.info("Purged %d documents from the index across %d domains; %d still queued",+ sum(removed_by_domain.values()), len(removed_by_domain), queue_size())+++ @background(schedule=0)+ def report_usage_to_polar():+ """+ Reports each user's billable overage (requests beyond the free 2,000/month)+ to Polar as usage events, once per hour.++ Only the delta since the last report is sent (UsageBucket.reported_overage+ tracks how much has already been ingested), so re-runs are idempotent and a+ failed batch simply gets resent (larger) on the next run rather than risking+ double-counting.+ """+ from polar_sdk import Polar++ now = datetime.now(timezone.utc)++ events = []+ buckets_to_update = []+ for bucket in UsageBucket.objects.filter(year=now.year, month=now.month).select_related("user__billing"):+ billing = getattr(bucket.user, "billing", None)+ if not billing or not billing.polar_customer_id:+ continue # no Polar customer yet — nothing to report++ total_overage = pricing.billable_overage(bucket.count)+ delta = total_overage - bucket.reported_overage+ if delta <= 0:+ continue++ events.append({+ "name": "search_request",+ "external_customer_id": str(bucket.user.id),+ "metadata": {"quantity": delta},+ })+ bucket.reported_overage = total_overage+ buckets_to_update.append(bucket)++ if not events:+ return++ try:+ with Polar(access_token=settings.POLAR_ACCESS_TOKEN, server=settings.POLAR_SERVER) as polar:+ polar.events.ingest(request={"events": events})+ except Exception:+ logger.exception("Error reporting usage to Polar")+ return++ UsageBucket.objects.bulk_update(buckets_to_update, ["reported_overage"])diff --cc mwmbl/tinysearchengine/rank.pyindex b800c98,58ac061..052a4db--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@@ -12,8 -14,8 +14,9 @@@ from pathlib import Patfrom urllib.parse import urlparseimport numpy as np+from django.conf import settingsfrom requests.adapters import HTTPAdapter, Retry+ from requests.exceptions import RetryErrorfrom mwmbl.format import get_query_regexfrom mwmbl.hn_top_domains_filtered import DOMAINS@@@ -343,11 -320,8 +349,11 @@@ class Rankerif items is not None:pages += items- external_search_items = self.external_search(q)+ external_search_items = self.external_search(q) if use_external_search else []- ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)+ candidates = pages + additional_results + external_search_items+ candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)++ ordered_results = self.order_results(terms, candidates, is_complete)deduplicated_results = deduplicate(curated_items + ordered_results, set())state_fixed = [fix_document_state(result) for result in deduplicated_results]return state_fixed, terms, completionsdiff --cc test/test_apps_background_scheduling.pyindex 62af95f,f9dd4f6..453a616--- a/test/test_apps_background_scheduling.py+++ b/test/test_apps_background_scheduling.py@@@ -1,39 -1,67 +1,85 @@@- """The periodic tasks only exist if apps.ready() registers them, and registering the same- task twice means it runs twice as often on every deploy."""+ """+ Tests for MwmblConfig._schedule_background_tasks().+-Covers the cache-lock guard added to prevent duplicate periodic Task rows-when multiple gunicorn worker processes call AppConfig.ready() on startup.++Covers the cache-lock guard added to prevent duplicate periodic Task rows when++multiple gunicorn worker processes call AppConfig.ready() on startup, and the++set of periodic tasks that are registered - the tasks only exist if ready()++registers them, and registering the same task twice means it runs twice as++often on every deploy.+ """++ from unittest.mock import patch+import pytestfrom background_task.models import Task+from django.conf import settings+ from django.core.cache import cachefrom mwmbl.apps import MwmblConfig- BLACKLIST_TASKS = {- "mwmbl.background.refresh_blacklist_snapshot",- "mwmbl.background.purge_blacklisted_from_queue",- }+ SYNC_TASK = "mwmbl.background.sync_search_counts"+ POLAR_REPORT_TASK = "mwmbl.background.report_usage_to_polar"++BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"++BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"++++ALL_TASKS = {SYNC_TASK, POLAR_REPORT_TASK, BLACKLIST_SNAPSHOT_TASK, BLACKLIST_PURGE_TASK}+++ @pytest.fixture(autouse=True)+ def clear_schedule_lock():+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY)+ yield+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY)+++ @pytest.mark.django_db-def test_schedule_background_tasks_creates_both_tasks():++def test_schedule_background_tasks_creates_all_tasks():+ MwmblConfig._schedule_background_tasks()+- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1++ for task_name in ALL_TASKS:++ assert Task.objects.filter(task_name=task_name).count() == 1@pytest.mark.django_db- def test_blacklist_tasks_are_scheduled():+ def test_schedule_background_tasks_skips_when_lock_already_held():+ """Simulates a second gunicorn worker starting while the first is scheduling."""+ assert cache.add(MwmblConfig._SCHEDULE_LOCK_KEY, "1", timeout=60)+MwmblConfig._schedule_background_tasks()- assert BLACKLIST_TASKS <= set(Task.objects.values_list("task_name", flat=True))- assert Task.objects.filter(task_name=SYNC_TASK).count() == 0- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 0++ assert Task.objects.filter(task_name__in=ALL_TASKS).count() == 0@pytest.mark.django_db- def test_scheduling_twice_does_not_duplicate_tasks():+ def test_schedule_background_tasks_second_call_does_not_duplicate():MwmblConfig._schedule_background_tasks()+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY) # as if a later worker acquires the lock nextMwmblConfig._schedule_background_tasks()- for task_name in BLACKLIST_TASKS:- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1++ for task_name in ALL_TASKS:++ assert Task.objects.filter(task_name=task_name).count() == 1+++ @pytest.mark.django_db+ def test_schedule_background_tasks_releases_lock_on_failure():+ with patch("mwmbl.background.sync_search_counts", side_effect=RuntimeError("boom")):+ MwmblConfig._schedule_background_tasks()++ assert Task.objects.filter(task_name=SYNC_TASK).count() == 0+ # Lock was released so a later-starting worker isn't blocked for the full TTL.+ assert cache.get(MwmblConfig._SCHEDULE_LOCK_KEY) is None++ MwmblConfig._schedule_background_tasks()- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1++ for task_name in ALL_TASKS:+ assert Task.objects.filter(task_name=task_name).count() == 1+++def test_blacklist_tasks_repeat_at_the_configured_intervals():+ MwmblConfig._schedule_background_tasks()+- snapshot = Task.objects.get(task_name="mwmbl.background.refresh_blacklist_snapshot")- purge = Task.objects.get(task_name="mwmbl.background.purge_blacklisted_from_queue")++ snapshot = Task.objects.get(task_name=BLACKLIST_SNAPSHOT_TASK)++ purge = Task.objects.get(task_name=BLACKLIST_PURGE_TASK)++ assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS+ assert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDS