Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Show the blacklist filtering state in the admin

Daoud Clarkea90e3599747d
diff --git a/mwmbl/admin.py b/mwmbl/admin.py
index 1658056..13aa3c1 100644
--- a/mwmbl/admin.py
+++ b/mwmbl/admin.py
@@ -65,3 +65,7 @@ class UserBillingAdmin(ModelAdmin):
admin.site.register(ApiKey, ApiKeyAdmin)
admin.site.register(MarketingConsent, MarketingConsentAdmin)
admin.site.register(UserBilling, UserBillingAdmin)
+
+# Adds the "Status" panel linking to the blacklist status page (mwmbl.admin_views) to the
+# admin front page.
+admin.site.index_template = "admin/mwmbl_index.html"
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.py
new file mode 100644
index 0000000..5303883
--- /dev/null
+++ b/mwmbl/admin_views.py
@@ -0,0 +1,147 @@
+"""Admin-only visibility on the blacklist filtering state that lives in Redis.
+
+Retrieval filtering, the snapshot refresh and the index purge are three processes talking
+to each other through Redis keys, and none of that state is reachable from the Django
+admin: a snapshot that never publishes and a purge queue that never drains look exactly
+like everything working, because retrieval quietly falls back to the built-in rules either
+way. This puts the whole loop - the published snapshot, what this worker actually loaded,
+the queue waiting to be purged, and the background tasks that maintain both - on one page.
+
+The page is read-only, and deliberately cheap: the snapshot's size comes from STRLEN so a
+page load never pulls the ~11 MB blob out of Redis, and the queue is sampled with
+SRANDMEMBER so looking at it cannot consume it.
+"""
+import os
+from datetime import datetime, timedelta
+from logging import getLogger
+
+from background_task.models import CompletedTask, Task
+from django.conf import settings
+from django.contrib.admin.views.decorators import staff_member_required
+from django.shortcuts import render
+from redis import RedisError
+
+from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY
+from mwmbl.indexer import blacklist_snapshot, purge_queue
+from mwmbl.indexer.blacklist_snapshot import (
+ HASH_DTYPE, SNAPSHOT_KEY, SNAPSHOT_VERSION_KEY, get_snapshot_blacklist,
+)
+from mwmbl.indexer.purge_queue import MAX_QUEUE_SIZE, PURGE_QUEUE_KEY, peek_purge_queue
+
+logger = getLogger(__name__)
+
+
+SNAPSHOT_TASK_NAME = "mwmbl.background.refresh_blacklist_snapshot"
+PURGE_TASK_NAME = "mwmbl.background.purge_blacklisted_from_queue"
+
+QUEUE_SAMPLE_SIZE = 50
+REMOVED_COUNT_DAYS = 14
+
+
+def _snapshot_status() -> dict:
+ """What is published, and what this worker is actually filtering against.
+
+ The two can differ, and the difference is the interesting part. Each gunicorn worker
+ loads the snapshot independently and only re-checks every
+ BLACKLIST_SNAPSHOT_CHECK_SECONDS, so a freshly published snapshot legitimately shows
+ as out of date here for a few minutes. A worker stuck on an old version for longer
+ than that is a real problem.
+ """
+ client = blacklist_snapshot.get_redis()
+ published_version = client.get(SNAPSHOT_VERSION_KEY)
+ if published_version is not None:
+ published_version = published_version.decode()
+
+ # STRLEN, not GET: the blob is ~11 MB and this page is only reporting on its size.
+ # Missing keys have length 0, which is also how the blob's absence is detected.
+ blob_size = client.strlen(SNAPSHOT_KEY)
+
+ snapshot = get_snapshot_blacklist()
+ return {
+ "published_version": published_version,
+ "blob_present": blob_size > 0,
+ "blob_size": blob_size,
+ "blob_domains": blob_size // HASH_DTYPE.itemsize,
+ # load_now() refuses a blob that is not a whole number of hashes, so surface the
+ # same check rather than leaving a rejected snapshot looking healthy here.
+ "blob_size_valid": blob_size % HASH_DTYPE.itemsize == 0,
+ "loaded_version": snapshot.loaded_version,
+ "loaded_domains": snapshot.num_domains,
+ "up_to_date": snapshot.loaded_version == published_version,
+ "check_seconds": settings.BLACKLIST_SNAPSHOT_CHECK_SECONDS,
+ "refresh_seconds": settings.BLACKLIST_SNAPSHOT_REFRESH_SECONDS,
+ }
+
+
+def _queue_status() -> dict:
+ client = purge_queue.get_redis()
+ size = client.scard(PURGE_QUEUE_KEY)
+ return {
+ "size": size,
+ "max_size": MAX_QUEUE_SIZE,
+ "full": size >= MAX_QUEUE_SIZE,
+ "batch_size": settings.BLACKLIST_PURGE_BATCH_SIZE,
+ "interval_seconds": settings.BLACKLIST_PURGE_INTERVAL_SECONDS,
+ "sample": peek_purge_queue(QUEUE_SAMPLE_SIZE, client),
+ }
+
+
+def _removed_counts(days: int) -> list[dict]:
+ """Documents the purge has removed from the index, per day, most recent first.
+
+ Retrieval filters blacklisted domains out of results whether or not the removal ever
+ happens, so a queue that is filling while this stays at zero is the signature of a
+ broken loop - see StatsManager.record_blacklisted_removed.
+ """
+ client = purge_queue.get_redis()
+ today = datetime.utcnow().date()
+ dates = [today - timedelta(days=i) for i in range(days)]
+ counts = client.mget([BLACKLISTED_REMOVED_COUNT_KEY.format(date=date) for date in dates])
+ return [{"date": date, "count": int(count) if count else 0} for date, count in zip(dates, counts)]
+
+
+def _task_status() -> list[dict]:
+ """The two background tasks that maintain the state above.
+
+ Nothing in this loop runs without a `manage.py process_tasks` worker, and one running
+ an image that predates these tasks skips them silently by name rather than failing -
+ so "pending, run_at long past, never completed" is what a missing or stale worker
+ looks like, and it is otherwise invisible.
+ """
+ task_names = [SNAPSHOT_TASK_NAME, PURGE_TASK_NAME]
+ pending = {task.task_name: task for task in Task.objects.filter(task_name__in=task_names)}
+
+ statuses = []
+ for task_name in task_names:
+ last_completed = CompletedTask.objects.filter(task_name=task_name).order_by("-run_at").first()
+ statuses.append({
+ "name": task_name,
+ "task": pending.get(task_name),
+ "last_completed": last_completed,
+ })
+ return statuses
+
+
+@staff_member_required
+def blacklist_status_view(request):
+ context = {
+ "title": "Blacklist status",
+ # This page reports on one gunicorn worker's in-memory snapshot, and the request
+ # lands on whichever worker the load balancer picked. Named so the template can
+ # say which one, because "loaded version" is otherwise ambiguous across workers.
+ "worker_pid": os.getpid(),
+ }
+
+ # A status page for Redis state has to render when Redis is down - that is precisely
+ # the failure it exists to report. Everything below this line comes from Redis, so one
+ # handler covers the lot.
+ try:
+ context["snapshot"] = _snapshot_status()
+ context["queue"] = _queue_status()
+ context["removed_counts"] = _removed_counts(REMOVED_COUNT_DAYS)
+ except RedisError as e:
+ logger.exception("Could not read blacklist status from Redis")
+ context["redis_error"] = str(e)
+
+ context["tasks"] = _task_status()
+ return render(request, "admin/blacklist_status.html", context)
diff --git a/mwmbl/indexer/blacklist_snapshot.py b/mwmbl/indexer/blacklist_snapshot.py
index 4115e7f..fb709a4 100644
--- a/mwmbl/indexer/blacklist_snapshot.py
+++ b/mwmbl/indexer/blacklist_snapshot.py
@@ -166,6 +166,17 @@ class SnapshotBlacklist:
def loaded(self) -> bool:
return self._array is not None
+ @property
+ def loaded_version(self) -> Optional[str]:
+ """The version of the snapshot this process is filtering against, if any."""
+ return self._version
+
+ @property
+ def num_domains(self) -> int:
+ """How many domain hashes this process has loaded."""
+ array = self._array
+ return 0 if array is None else len(array)
+
def _get_redis(self) -> redis.Redis:
return self._redis if self._redis is not None else get_redis()
diff --git a/mwmbl/indexer/purge_queue.py b/mwmbl/indexer/purge_queue.py
index b1a7a67..9a5eb3c 100644
--- a/mwmbl/indexer/purge_queue.py
+++ b/mwmbl/indexer/purge_queue.py
@@ -123,11 +123,7 @@ def enqueue_for_purge(documents: Iterable[Document], redis_client: Optional[redi
return added
-def drain_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) -> list[Document]:
- """Remove and return up to `limit` queued documents."""
- client = redis_client if redis_client is not None else get_redis()
- payloads = client.spop(PURGE_QUEUE_KEY, limit) or []
-
+def _documents_from_payloads(payloads: Iterable[str]) -> list[Document]:
documents = []
for payload in payloads:
try:
@@ -142,6 +138,25 @@ def drain_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) ->
return documents
+def drain_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) -> list[Document]:
+ """Remove and return up to `limit` queued documents."""
+ client = redis_client if redis_client is not None else get_redis()
+ payloads = client.spop(PURGE_QUEUE_KEY, limit) or []
+ return _documents_from_payloads(payloads)
+
+
+def peek_purge_queue(limit: int, redis_client: Optional[redis.Redis] = None) -> list[Document]:
+ """Up to `limit` queued documents, left on the queue.
+
+ SRANDMEMBER rather than SPOP: this exists for the admin status view, where reading the
+ queue must not consume it. The queue is a SET and has no order, so this is a sample of
+ what is waiting rather than the entries the next purge run will take.
+ """
+ client = redis_client if redis_client is not None else get_redis()
+ payloads = client.srandmember(PURGE_QUEUE_KEY, limit) or []
+ return _documents_from_payloads(payloads)
+
+
def queue_size(redis_client: Optional[redis.Redis] = None) -> int:
client = redis_client if redis_client is not None else get_redis()
return client.scard(PURGE_QUEUE_KEY)
diff --git a/mwmbl/templates/admin/blacklist_status.html b/mwmbl/templates/admin/blacklist_status.html
new file mode 100644
index 0000000..b2565cf
--- /dev/null
+++ b/mwmbl/templates/admin/blacklist_status.html
@@ -0,0 +1,211 @@
+{% extends "admin/base_site.html" %}
+{% load humanbytes %}
+
+{% block title %}Blacklist status | {{ site_title|default:"Django site admin" }}{% endblock %}
+
+{% block breadcrumbs %}
+ <div class="breadcrumbs">
+ <a href="{% url 'admin:index' %}">Home</a>
+ &rsaquo; Blacklist status
+ </div>
+{% endblock %}
+
+{% block content %}
+ <style>
+ .blacklist-status table { width: 100%; margin-bottom: 10px; }
+ .blacklist-status td.key { width: 30%; font-weight: bold; }
+ .blacklist-status .ok { color: #417690; font-weight: bold; }
+ .blacklist-status .bad { color: #ba2121; font-weight: bold; }
+ .blacklist-status .url { word-break: break-all; }
+ .blacklist-status .note { color: #666; font-size: 0.9em; }
+ </style>
+
+ <div id="content-main" class="blacklist-status">
+
+ {% if redis_error %}
+ <div class="module">
+ <h2>Redis unreachable</h2>
+ <p class="bad">{{ redis_error }}</p>
+ <p class="note">
+ Everything below the background tasks comes from Redis. With Redis down, search
+ workers filter on the built-in rules only and nothing is queued for purging.
+ </p>
+ </div>
+ {% else %}
+
+ <div class="module">
+ <h2>Snapshot</h2>
+ <table>
+ <tr>
+ <td class="key">Blob present</td>
+ <td>
+ {% if snapshot.blob_present %}
+ <span class="ok">yes</span>
+ {% else %}
+ <span class="bad">no</span> &mdash; retrieval is filtering on the built-in
+ rules only (the regex, EXCLUDED_DOMAINS and the numeric-subdomain
+ heuristic). The remote blocklists are not being applied.
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td class="key">Blob size</td>
+ <td>
+ {{ snapshot.blob_size|human_bytes }} ({{ snapshot.blob_size }} bytes)
+ {% if not snapshot.blob_size_valid %}
+ <span class="bad">&mdash; not a whole number of hashes, so workers will
+ refuse to load it</span>
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td class="key">Domains in blob</td>
+ <td>{{ snapshot.blob_domains }}</td>
+ </tr>
+ <tr>
+ <td class="key">Published version</td>
+ <td>
+ {% if snapshot.published_version %}
+ {# Truncated to the same 12 characters the publish/load log lines use, so
+ the page and the logs can be matched up by eye. #}
+ <code title="{{ snapshot.published_version }}">{{ snapshot.published_version|slice:":12" }}</code>
+ {% if not snapshot.blob_present %}
+ <span class="bad">&mdash; version present but blob missing; it was
+ probably evicted (Redis runs allkeys-lru). Workers keep the copy they
+ already loaded until the refresh task republishes.</span>
+ {% endif %}
+ {% else %}
+ <span class="bad">never published</span>
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td class="key">Loaded by this worker</td>
+ <td>
+ {% if snapshot.loaded_version %}
+ <code title="{{ snapshot.loaded_version }}">{{ snapshot.loaded_version|slice:":12" }}</code>
+ &mdash; {{ snapshot.loaded_domains }} domains,
+ {% if snapshot.up_to_date %}
+ <span class="ok">up to date</span>
+ {% else %}
+ <span class="bad">out of date</span>
+ {% endif %}
+ {% else %}
+ <span class="bad">nothing loaded</span>
+ {% endif %}
+ </td>
+ </tr>
+ </table>
+ <p class="note">
+ This is worker pid {{ worker_pid }} only &mdash; each gunicorn worker loads the
+ snapshot independently and re-checks every {{ snapshot.check_seconds }}s, so for
+ a few minutes after a publish some workers legitimately show as out of date.
+ Reload to land on another worker. The snapshot is rebuilt every
+ {{ snapshot.refresh_seconds }}s.
+ </p>
+ </div>
+
+ <div class="module">
+ <h2>Purge queue</h2>
+ <table>
+ <tr>
+ <td class="key">Queued documents</td>
+ <td>
+ {{ queue.size }} of {{ queue.max_size }} max
+ {% if queue.full %}
+ <span class="bad">&mdash; full; further documents are being dropped rather
+ than queued</span>
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <td class="key">Drain rate</td>
+ <td>{{ queue.batch_size }} documents every {{ queue.interval_seconds }}s</td>
+ </tr>
+ </table>
+ <p class="note">
+ Queued by retrieval when it filters a blacklisted document out of results, and
+ drained by the purge task, which removes them from the index. A queue that grows
+ while the removals below stay at zero means the loop is broken. Results are
+ filtered either way, so this backlog does not affect what users see.
+ </p>
+
+ <h3>Sample of {{ queue.sample|length }} queued document{{ queue.sample|pluralize }}</h3>
+ {% if queue.sample %}
+ <table>
+ <thead>
+ <tr><th>URL</th><th>Title</th><th>Filed under term</th></tr>
+ </thead>
+ <tbody>
+ {% for document in queue.sample %}
+ <tr>
+ <td class="url">{{ document.url }}</td>
+ <td>{{ document.title }}</td>
+ <td>{% if document.term %}{{ document.term }}{% else %}<span class="note">own tokens</span>{% endif %}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ <p class="note">
+ A random sample &mdash; the queue is a Redis SET and has no order. Reading it
+ here does not consume it.
+ </p>
+ {% else %}
+ <p class="note">The queue is empty.</p>
+ {% endif %}
+ </div>
+
+ <div class="module">
+ <h2>Documents removed from the index</h2>
+ <table>
+ <thead><tr><th>Date (UTC)</th><th>Removed</th></tr></thead>
+ <tbody>
+ {% for day in removed_counts %}
+ <tr><td>{{ day.date }}</td><td>{{ day.count }}</td></tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ </div>
+
+ {% endif %}
+
+ <div class="module">
+ <h2>Background tasks</h2>
+ <table>
+ <thead>
+ <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>
+ </thead>
+ <tbody>
+ {% for task in tasks %}
+ <tr>
+ <td><code>{{ task.name }}</code></td>
+ <td>
+ {% if task.task %}
+ {{ task.task.run_at }}
+ {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}
+ {% else %}
+ <span class="bad">not scheduled</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>
+ <td>
+ {% if task.last_completed %}
+ {{ task.last_completed.run_at }}
+ {% else %}
+ <span class="bad">never</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ <p class="note">
+ Both tasks only run under a <code>manage.py process_tasks</code> worker. A worker
+ running an image that predates these tasks skips them silently by name, which shows
+ up here as a next run in the past that never completes.
+ </p>
+ </div>
+
+ </div>
+{% endblock %}
diff --git a/mwmbl/templates/admin/mwmbl_index.html b/mwmbl/templates/admin/mwmbl_index.html
new file mode 100644
index 0000000..3e92eed
--- /dev/null
+++ b/mwmbl/templates/admin/mwmbl_index.html
@@ -0,0 +1,19 @@
+{% extends "admin/index.html" %}
+
+{# Wired up as admin.site.index_template. A file named admin/index.html here would not
+ win: django.contrib.admin comes before mwmbl in INSTALLED_APPS, so the app-directories
+ loader finds Django's copy first. A distinct name sidesteps that, and lets this template
+ extend the original rather than duplicate it. #}
+
+{% block content %}
+ {{ block.super }}
+ <div class="module">
+ <table>
+ <caption>Status</caption>
+ <tr>
+ <th scope="row"><a href="{% url 'blacklist_status' %}">Blacklist status</a></th>
+ <td>The published snapshot, the purge queue and the tasks that maintain them.</td>
+ </tr>
+ </table>
+ </div>
+{% endblock %}
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 4008b36..6d8cb49 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -20,6 +20,7 @@ from django.views.generic import RedirectView
from debug_toolbar.toolbar import debug_toolbar_urls
import mwmbl.crawler.app as crawler
+from mwmbl.admin_views import blacklist_status_view
from mwmbl.api import api as v1_api, v2_api, register_routers
from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
@@ -37,6 +38,9 @@ def trigger_error(request):
urlpatterns = [
+ # Before admin.site.urls so the admin catch-all does not shadow it. Access is gated by
+ # staff_member_required on the view, matching the rest of the admin.
+ path('admin/blacklist-status/', blacklist_status_view, name="blacklist_status"),
path('admin/', admin.site.urls),
path('accounts/', include('allauth.urls')),
diff --git a/test/test_admin_blacklist_status.py b/test/test_admin_blacklist_status.py
new file mode 100644
index 0000000..d079e61
--- /dev/null
+++ b/test/test_admin_blacklist_status.py
@@ -0,0 +1,224 @@
+"""Tests for the blacklist status admin page (mwmbl.admin_views)."""
+import fakeredis
+import numpy as np
+import pytest
+from django.contrib.auth import get_user_model
+from redis import ConnectionError as RedisConnectionError
+
+from mwmbl import admin_views
+from mwmbl.indexer import blacklist_snapshot, purge_queue
+from mwmbl.indexer.blacklist_snapshot import HASH_DTYPE, SnapshotBlacklist, publish_snapshot
+from mwmbl.indexer.purge_queue import drain_purge_queue, enqueue_for_purge, peek_purge_queue, queue_size
+from mwmbl.tinysearchengine.indexer import Document
+
+User = get_user_model()
+
+STATUS_URL = "/admin/blacklist-status/"
+
+
[email protected](autouse=True)
+def clear_recently_enqueued():
+ """purge_queue keeps a process-level record of what it has already queued, so without
+ this each test would inherit the previous one's suppressions."""
+ purge_queue._recently_enqueued.clear()
+ yield
+ purge_queue._recently_enqueued.clear()
+
+
+def redis_server():
+ return fakeredis.FakeServer()
+
+
+def redis_client(redis_server):
+ return fakeredis.FakeRedis(server=redis_server, decode_responses=True)
+
+
+def documents():
+ return [
+ Document(title="Bad", url="https://badsite.test/a", extract="x", score=1.0),
+ Document(title="Worse", url="https://badsite.test/b", extract="y", score=2.0, term="query term"),
+ ]
+
+
+def wired_redis(monkeypatch, redis_server, redis_client):
+ """Point both Redis-backed modules at one fake server, and reset the snapshot singleton.
+
+ Two clients over one server, as in production: blacklist_snapshot's is binary because
+ the snapshot blob is not text, purge_queue's decodes because its payloads are JSON.
+ Sharing the server is what lets a test set up a snapshot and a queue and see both on
+ the same page.
+ """
+ binary_client = fakeredis.FakeRedis(server=redis_server)
+ monkeypatch.setattr(blacklist_snapshot, "_redis", binary_client)
+ monkeypatch.setattr(purge_queue, "_redis", redis_client)
+ monkeypatch.setattr(blacklist_snapshot, "_snapshot_blacklist",
+ SnapshotBlacklist(redis_client=binary_client))
+ return redis_client
+
+
+def staff_client(client, db):
+ user = User.objects.create_user(username="staff_member_1", password="correctpassword", is_staff=True)
+ client.force_login(user)
+ return client
+
+
+# ---------------------------------------------------------------------------
+# peek_purge_queue
+# ---------------------------------------------------------------------------
+
+def test_peek_does_not_consume_the_queue(redis_client, documents):
+ enqueue_for_purge(documents, redis_client)
+
+ peeked = peek_purge_queue(10, redis_client)
+
+ assert {d.url for d in peeked} == {"https://badsite.test/a", "https://badsite.test/b"}
+ assert queue_size(redis_client) == 2
+ # And the entries are still there for the purge task to take.
+ assert len(drain_purge_queue(10, redis_client)) == 2
+
+
+def test_peek_returns_the_queued_term(redis_client, documents):
+ enqueue_for_purge(documents, redis_client)
+
+ terms = {d.url: d.term for d in peek_purge_queue(10, redis_client)}
+
+ assert terms["https://badsite.test/b"] == "query term"
+ assert terms["https://badsite.test/a"] is None
+
+
+def test_peek_on_an_empty_queue(redis_client):
+ assert peek_purge_queue(10, redis_client) == []
+
+
+# ---------------------------------------------------------------------------
+# Status gathering
+# ---------------------------------------------------------------------------
+
+def test_snapshot_status_reports_a_published_blob(wired_redis):
+ blob = np.array([1, 2, 3], dtype=HASH_DTYPE).tobytes()
+ version = publish_snapshot(blob, wired_redis)
+
+ status = admin_views._snapshot_status()
+
+ assert status["blob_present"]
+ assert status["blob_size"] == 24
+ assert status["blob_domains"] == 3
+ assert status["blob_size_valid"]
+ assert status["published_version"] == version
+
+
+def test_snapshot_status_when_nothing_is_published(wired_redis):
+ status = admin_views._snapshot_status()
+
+ assert not status["blob_present"]
+ assert status["blob_size"] == 0
+ assert status["published_version"] is None
+ assert status["loaded_version"] is None
+ assert status["loaded_domains"] == 0
+
+
+def test_snapshot_status_flags_a_blob_that_is_not_a_whole_number_of_hashes(wired_redis):
+ publish_snapshot(b"12345", wired_redis)
+
+ status = admin_views._snapshot_status()
+
+ assert status["blob_present"]
+ assert not status["blob_size_valid"]
+
+
+def test_snapshot_status_distinguishes_published_from_loaded(wired_redis):
+ publish_snapshot(np.array([1, 2], dtype=HASH_DTYPE).tobytes(), wired_redis)
+ status_before_load = admin_views._snapshot_status()
+
+ blacklist_snapshot.get_snapshot_blacklist().load_now()
+ status_after_load = admin_views._snapshot_status()
+
+ assert not status_before_load["up_to_date"]
+ assert status_after_load["up_to_date"]
+ assert status_after_load["loaded_domains"] == 2
+
+
+def test_queue_status_samples_without_draining(wired_redis, documents):
+ enqueue_for_purge(documents, wired_redis)
+
+ status = admin_views._queue_status()
+
+ assert status["size"] == 2
+ assert len(status["sample"]) == 2
+ assert not status["full"]
+ assert queue_size(wired_redis) == 2
+
+
+def test_removed_counts_covers_the_requested_window(wired_redis):
+ counts = admin_views._removed_counts(14)
+
+ assert len(counts) == 14
+ assert all(day["count"] == 0 for day in counts)
+ # Most recent first, so the page reads today-downwards.
+ assert counts[0]["date"] > counts[-1]["date"]
+
+
+# ---------------------------------------------------------------------------
+# The view
+# ---------------------------------------------------------------------------
+
+def test_view_requires_staff(client, db):
+ user = User.objects.create_user(username="ordinary_user_2", password="correctpassword")
+ client.force_login(user)
+
+ response = client.get(STATUS_URL)
+
+ assert response.status_code == 302
+ assert "/admin/login/" in response.url
+
+
+def test_view_renders_the_snapshot_and_queue(staff_client, wired_redis, documents):
+ publish_snapshot(np.array([1, 2, 3], dtype=HASH_DTYPE).tobytes(), wired_redis)
+ enqueue_for_purge(documents, wired_redis)
+
+ response = staff_client.get(STATUS_URL)
+ content = response.content.decode()
+
+ assert response.status_code == 200
+ assert "https://badsite.test/a" in content
+ assert "Purge queue" in content
+ assert response.context["snapshot"]["blob_domains"] == 3
+ assert response.context["queue"]["size"] == 2
+
+
+def test_view_renders_when_redis_is_down(staff_client, monkeypatch):
+ def explode():
+ raise RedisConnectionError("connection refused")
+
+ monkeypatch.setattr(admin_views.blacklist_snapshot, "get_redis", explode)
+
+ response = staff_client.get(STATUS_URL)
+
+ # A status page for Redis state is exactly what you load when Redis is down, so it
+ # must report the failure rather than 500.
+ assert response.status_code == 200
+ assert "connection refused" in response.content.decode()
+ assert "snapshot" not in response.context
+
+
+def test_view_reports_the_background_tasks(staff_client, wired_redis):
+ response = staff_client.get(STATUS_URL)
+
+ task_names = [task["name"] for task in response.context["tasks"]]
+
+ assert task_names == [admin_views.SNAPSHOT_TASK_NAME, admin_views.PURGE_TASK_NAME]
+ # Nothing scheduled in the test DB, which is what a missing process_tasks worker looks
+ # like - the page has to render that rather than assume the rows exist.
+ assert all(task["task"] is None and task["last_completed"] is None
+ for task in response.context["tasks"])
+
+
+def test_admin_index_links_to_the_status_page(staff_client):
+ response = staff_client.get("/admin/")
+
+ assert STATUS_URL in response.content.decode()