Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Merge main into filter-blacklisted-at-retrieval

Daoud Clarkee76b034b25b6
diff --cc mwmbl/apps.py
index 60a17be,decf3e0..524d2db
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@@ -65,33 -70,36 +70,54 @@@ class MwmblConfig(AppConfig)
except RuntimeError:
pass
+ # The gunicorn server runs multiple worker processes, each of which calls
+ # this method independently on startup. The "does a Task row already
+ # exist?" checks below are plain reads with no transaction or unique
+ # constraint backing them, so without this lock, workers starting at the
+ # same time (every deploy/restart) can race past the check before any of
+ # them commits its Task row, each creating its own duplicate periodic
+ # task. cache.add() is atomic (same pattern as the rate limiter in
+ # mwmbl.quota), so only the first worker to reach this point proceeds;
+ # the rest skip immediately, and the lock's TTL simply bounds how long
+ # a crashed winner blocks a retry by a later-starting worker.
+ from django.core.cache import cache
+ if not cache.add(MwmblConfig._SCHEDULE_LOCK_KEY, "1", timeout=MwmblConfig._SCHEDULE_LOCK_TIMEOUT):
+ log.info("Skipping background task scheduling: another worker is already handling it.")
+ return
+
try:
from background_task.models import Task
- from mwmbl.background import sync_search_counts, report_usage_to_polar
+ from mwmbl.background import (
- purge_blacklisted_from_queue, refresh_blacklist_snapshot, sync_search_counts,
++ purge_blacklisted_from_queue, refresh_blacklist_snapshot, report_usage_to_polar,
++ sync_search_counts,
+ )
SYNC_TASK = "mwmbl.background.sync_search_counts"
+ POLAR_REPORT_TASK = "mwmbl.background.report_usage_to_polar"
+ BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"
+ BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"
# Sync search counts once per hour (3600 seconds)
if not Task.objects.filter(task_name=SYNC_TASK).exists():
sync_search_counts(repeat=3600, repeat_until=None)
+ # Report billable usage overage to Polar once per hour (3600 seconds)
+ if not Task.objects.filter(task_name=POLAR_REPORT_TASK).exists():
+ report_usage_to_polar(repeat=3600, repeat_until=None)
+
+ # Rebuild the blacklist snapshot the search path filters against. schedule=0
+ # on the task means the first run happens immediately rather than one full
+ # refresh interval after deploy, which matters because until it lands the
+ # search workers have only the built-in rules to go on.
+ if not Task.objects.filter(task_name=BLACKLIST_SNAPSHOT_TASK).exists():
+ refresh_blacklist_snapshot(
+ repeat=settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS, repeat_until=None)
+
+ # Drain the queue of blacklisted documents that retrieval has filtered out
+ if not Task.objects.filter(task_name=BLACKLIST_PURGE_TASK).exists():
+ purge_blacklisted_from_queue(
+ repeat=settings.BLACKLIST_PURGE_INTERVAL_SECONDS, repeat_until=None)
+
except Exception:
# Don't prevent startup if background task scheduling fails
log.exception("Failed to schedule background tasks")
diff --cc mwmbl/background.py
index 062bb0d,8b9b282..5b5cedb
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@@ -1,10 -1,9 +1,11 @@@
"""
Script that updates data in a background process.
-Also contains Django Background Tasks for periodic quota maintenance:
+Also contains Django Background Tasks for periodic maintenance:
- sync_search_counts: syncs Redis monthly counters → DB once per hour
+ - report_usage_to_polar: reports billable usage overage to Polar once per hour
+ - refresh_blacklist_snapshot: rebuilds the blacklist the search path filters against
+ - purge_blacklisted_from_queue: removes retrieval-filtered documents from the index
"""
import logging
import sys
@@@ -133,48 -133,49 +135,94 @@@ def sync_search_counts()
logger.exception("Error syncing search count for key %s", key)
+# ---------------------------------------------------------------------------
+# Blacklisted-domain maintenance (Django Background Tasks)
+# ---------------------------------------------------------------------------
+
+@background(schedule=0)
+def refresh_blacklist_snapshot():
+ """
+ Rebuild the blacklist snapshot that the search workers filter against.
+
+ This is the only place the multi-megabyte remote blocklists are downloaded and
+ parsed. Search workers only ever read the ~11 MB hash array this publishes to Redis,
+ so no query ever waits on a blocklist fetch. See mwmbl.indexer.blacklist_snapshot.
+ """
+ from mwmbl.indexer.blacklist_snapshot import refresh_snapshot
+
+ refresh_snapshot()
+
+
+@background(schedule=0)
+def purge_blacklisted_from_queue():
+ """
+ Remove the documents that retrieval filtered out from the index itself.
+
+ Retrieval already guarantees these are never shown; this closes the loop so the same
+ documents are not re-filtered on every future query. The queue is best-effort, and
+ that is fine: anything lost is re-queued the next time a query surfaces it.
+ """
+ from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
+ from mwmbl.indexer.purge_blacklisted import purge_documents
+ from mwmbl.indexer.purge_queue import drain_purge_queue, queue_size
+ from mwmbl.tinysearchengine.indexer import Document, TinyIndex
+
+ documents = drain_purge_queue(settings.BLACKLIST_PURGE_BATCH_SIZE)
+ if not documents:
+ return
+
+ blacklist = get_snapshot_blacklist()
+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
+ with TinyIndex(Document, str(index_path), 'w') as index:
+ removed_by_domain = purge_documents(index, documents, blacklist.is_domain_blacklisted)
+
+ logger.info("Purged %d documents from the index across %d domains; %d still queued",
+ sum(removed_by_domain.values()), len(removed_by_domain), queue_size())
+
+
+ @background(schedule=0)
+ def report_usage_to_polar():
+ """
+ Reports each user's billable overage (requests beyond the free 2,000/month)
+ to Polar as usage events, once per hour.
+
+ Only the delta since the last report is sent (UsageBucket.reported_overage
+ tracks how much has already been ingested), so re-runs are idempotent and a
+ failed batch simply gets resent (larger) on the next run rather than risking
+ double-counting.
+ """
+ from polar_sdk import Polar
+
+ now = datetime.now(timezone.utc)
+
+ events = []
+ buckets_to_update = []
+ for bucket in UsageBucket.objects.filter(year=now.year, month=now.month).select_related("user__billing"):
+ billing = getattr(bucket.user, "billing", None)
+ if not billing or not billing.polar_customer_id:
+ continue # no Polar customer yet — nothing to report
+
+ total_overage = pricing.billable_overage(bucket.count)
+ delta = total_overage - bucket.reported_overage
+ if delta <= 0:
+ continue
+
+ events.append({
+ "name": "search_request",
+ "external_customer_id": str(bucket.user.id),
+ "metadata": {"quantity": delta},
+ })
+ bucket.reported_overage = total_overage
+ buckets_to_update.append(bucket)
+
+ if not events:
+ return
+
+ try:
+ with Polar(access_token=settings.POLAR_ACCESS_TOKEN, server=settings.POLAR_SERVER) as polar:
+ polar.events.ingest(request={"events": events})
+ except Exception:
+ logger.exception("Error reporting usage to Polar")
+ return
+
+ UsageBucket.objects.bulk_update(buckets_to_update, ["reported_overage"])
diff --cc mwmbl/tinysearchengine/rank.py
index b800c98,58ac061..052a4db
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@@ -12,8 -14,8 +14,9 @@@ from pathlib import Pat
from urllib.parse import urlparse
import numpy as np
+from django.conf import settings
from requests.adapters import HTTPAdapter, Retry
+ from requests.exceptions import RetryError
from mwmbl.format import get_query_regex
from mwmbl.hn_top_domains_filtered import DOMAINS
@@@ -343,11 -320,8 +349,11 @@@ class Ranker
if items is not None:
pages += items
- external_search_items = self.external_search(q)
+ external_search_items = self.external_search(q) if use_external_search else []
- ordered_results = self.order_results(terms, pages + additional_results + external_search_items, is_complete)
+ candidates = pages + additional_results + external_search_items
+ candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)
+
+ ordered_results = self.order_results(terms, candidates, is_complete)
deduplicated_results = deduplicate(curated_items + ordered_results, set())
state_fixed = [fix_document_state(result) for result in deduplicated_results]
return state_fixed, terms, completions
diff --cc test/test_apps_background_scheduling.py
index 62af95f,f9dd4f6..453a616
--- a/test/test_apps_background_scheduling.py
+++ b/test/test_apps_background_scheduling.py
@@@ -1,39 -1,67 +1,85 @@@
- """The periodic tasks only exist if apps.ready() registers them, and registering the same
- task twice means it runs twice as often on every deploy."""
+ """
+ Tests for MwmblConfig._schedule_background_tasks().
+
-Covers the cache-lock guard added to prevent duplicate periodic Task rows
-when multiple gunicorn worker processes call AppConfig.ready() on startup.
++Covers the cache-lock guard added to prevent duplicate periodic Task rows when
++multiple gunicorn worker processes call AppConfig.ready() on startup, and the
++set of periodic tasks that are registered - the tasks only exist if ready()
++registers them, and registering the same task twice means it runs twice as
++often on every deploy.
+ """
+
+ from unittest.mock import patch
+
import pytest
from background_task.models import Task
+from django.conf import settings
+ from django.core.cache import cache
from mwmbl.apps import MwmblConfig
- BLACKLIST_TASKS = {
- "mwmbl.background.refresh_blacklist_snapshot",
- "mwmbl.background.purge_blacklisted_from_queue",
- }
+ SYNC_TASK = "mwmbl.background.sync_search_counts"
+ POLAR_REPORT_TASK = "mwmbl.background.report_usage_to_polar"
++BLACKLIST_SNAPSHOT_TASK = "mwmbl.background.refresh_blacklist_snapshot"
++BLACKLIST_PURGE_TASK = "mwmbl.background.purge_blacklisted_from_queue"
++
++ALL_TASKS = {SYNC_TASK, POLAR_REPORT_TASK, BLACKLIST_SNAPSHOT_TASK, BLACKLIST_PURGE_TASK}
+
+
+ @pytest.fixture(autouse=True)
+ def clear_schedule_lock():
+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY)
+ yield
+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY)
+
+
+ @pytest.mark.django_db
-def test_schedule_background_tasks_creates_both_tasks():
++def test_schedule_background_tasks_creates_all_tasks():
+ MwmblConfig._schedule_background_tasks()
+
- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1
- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1
++ for task_name in ALL_TASKS:
++ assert Task.objects.filter(task_name=task_name).count() == 1
@pytest.mark.django_db
- def test_blacklist_tasks_are_scheduled():
+ def test_schedule_background_tasks_skips_when_lock_already_held():
+ """Simulates a second gunicorn worker starting while the first is scheduling."""
+ assert cache.add(MwmblConfig._SCHEDULE_LOCK_KEY, "1", timeout=60)
+
MwmblConfig._schedule_background_tasks()
- assert BLACKLIST_TASKS <= set(Task.objects.values_list("task_name", flat=True))
- assert Task.objects.filter(task_name=SYNC_TASK).count() == 0
- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 0
++ assert Task.objects.filter(task_name__in=ALL_TASKS).count() == 0
@pytest.mark.django_db
- def test_scheduling_twice_does_not_duplicate_tasks():
+ def test_schedule_background_tasks_second_call_does_not_duplicate():
MwmblConfig._schedule_background_tasks()
+ cache.delete(MwmblConfig._SCHEDULE_LOCK_KEY) # as if a later worker acquires the lock next
MwmblConfig._schedule_background_tasks()
- for task_name in BLACKLIST_TASKS:
- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1
- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1
++ for task_name in ALL_TASKS:
++ assert Task.objects.filter(task_name=task_name).count() == 1
+
+
+ @pytest.mark.django_db
+ def test_schedule_background_tasks_releases_lock_on_failure():
+ with patch("mwmbl.background.sync_search_counts", side_effect=RuntimeError("boom")):
+ MwmblConfig._schedule_background_tasks()
+
+ assert Task.objects.filter(task_name=SYNC_TASK).count() == 0
+ # Lock was released so a later-starting worker isn't blocked for the full TTL.
+ assert cache.get(MwmblConfig._SCHEDULE_LOCK_KEY) is None
+
+ MwmblConfig._schedule_background_tasks()
- assert Task.objects.filter(task_name=SYNC_TASK).count() == 1
- assert Task.objects.filter(task_name=POLAR_REPORT_TASK).count() == 1
++ for task_name in ALL_TASKS:
+ assert Task.objects.filter(task_name=task_name).count() == 1
+
+
+def test_blacklist_tasks_repeat_at_the_configured_intervals():
+ MwmblConfig._schedule_background_tasks()
+
- snapshot = Task.objects.get(task_name="mwmbl.background.refresh_blacklist_snapshot")
- purge = Task.objects.get(task_name="mwmbl.background.purge_blacklisted_from_queue")
++ snapshot = Task.objects.get(task_name=BLACKLIST_SNAPSHOT_TASK)
++ purge = Task.objects.get(task_name=BLACKLIST_PURGE_TASK)
+
+ assert snapshot.repeat == settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS
+ assert purge.repeat == settings.BLACKLIST_PURGE_INTERVAL_SECONDS