An open source, non-profit web search engine
Add an admin page showing the weekly index URL count
Claudeab4792204995
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.pyindex a000a5f..3ec5efd 100644--- a/mwmbl/admin_views.py+++ b/mwmbl/admin_views.py@@ -1,4 +1,5 @@-"""Admin-only status pages: the blacklist filtering state in Redis, and paying users.+"""Admin-only status pages: the blacklist filtering state in Redis, the weekly index count,+and paying users.The blacklist page gives visibility on the blacklist filtering state that lives in Redis.@@ -17,6 +18,7 @@ SRANDMEMBER so looking at it cannot consume it.import osfrom datetime import timedeltafrom logging import getLogger+from pathlib import Pathfrom background_task.models import CompletedTask, Taskfrom django.conf import settings@@ -26,7 +28,7 @@ from django.db.models import F, OuterRef, Q, Subqueryfrom django.shortcuts import renderfrom redis import RedisError-from mwmbl import pricing, quota+from mwmbl import count_urls, pricing, quotafrom mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEYfrom mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer import blacklist_snapshot, purge_queue@@ -46,9 +48,11 @@ logger = getLogger(__name__)SNAPSHOT_TASK_NAME = "mwmbl.background.refresh_blacklist_snapshot"PURGE_TASK_NAME = "mwmbl.background.purge_blacklisted_from_queue"+INDEX_COUNT_TASK_NAME = "mwmbl.background.count_index_urls"QUEUE_SAMPLE_SIZE = 50REMOVED_COUNT_DAYS = 14+PUBLISHED_INDEX_COUNT_DAYS = 60def _snapshot_status() -> dict:@@ -130,15 +134,14 @@ def _curated_status() -> dict:}-def _task_status() -> list[dict]:- """The two background tasks that maintain the state above.+def _task_status(task_names: list[str]) -> list[dict]:+ """The background tasks that maintain the state a status page reports on.- Nothing in this loop runs without a `manage.py process_tasks` worker, and one running- an image that predates these tasks skips them silently by name rather than failing -- so "pending, run_at long past, never completed" is what a missing or stale worker- looks like, and it is otherwise invisible.+ None of them runs without a `manage.py process_tasks` worker, and one running an image+ that predates a task skips it silently by name rather than failing - so "pending,+ run_at long past, never completed" is what a missing or stale worker looks like, and+ it is otherwise invisible."""- task_names = [SNAPSHOT_TASK_NAME, PURGE_TASK_NAME]pending = {task.task_name: task for task in Task.objects.filter(task_name__in=task_names)}statuses = []@@ -176,10 +179,28 @@ def blacklist_status_view(request):logger.exception("Could not read blacklist status from Redis")context["redis_error"] = str(e)- context["tasks"] = _task_status()+ context["tasks"] = _task_status([SNAPSHOT_TASK_NAME, PURGE_TASK_NAME])return render(request, "admin/blacklist_status.html", context)+@staff_member_required+def index_count_view(request):+ context = {"title": "Index count"}+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME+ # Rendered with Redis down, like the blacklist page: the scan stalling because Redis+ # is unreachable is one of the things this page is for.+ try:+ redis = count_urls.get_redis()+ context["scan"] = count_urls.get_scan_status(redis, index_path)+ context["published"] = count_urls.get_published_counts(redis, PUBLISHED_INDEX_COUNT_DAYS)+ except RedisError as e:+ logger.exception("Could not read the index count from Redis")+ context["redis_error"] = str(e)++ context["tasks"] = _task_status([INDEX_COUNT_TASK_NAME])+ return render(request, "admin/index_count.html", context)++def _api_users() -> list[dict]:"""Users holding a search API key, plus anyone with a Polar usage subscription.diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.pyindex 21704b7..c3d18d8 100644--- a/mwmbl/count_urls.py+++ b/mwmbl/count_urls.py@@ -204,6 +204,53 @@ def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:}+def get_scan_status(redis: Redis, index_path: Path) -> dict:+ """How far the scan in progress has got, and when the last one finished.++ For the admin status page. The counts so far are PFCOUNTs of the scan's HyperLogLogs,+ which is cheap, so looking at the scan does not slow it down.+ """+ scan = redis.hgetall(INDEX_SCAN_KEY)+ last_finished = redis.get(INDEX_SCAN_LAST_FINISHED_KEY)+ last_finished_date = None if last_finished is None else date.fromisoformat(last_finished)+ interval = timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)+ status = {+ "in_progress": bool(scan),+ "last_finished": last_finished_date,+ "next_scan_due": None if last_finished_date is None else last_finished_date + interval,+ "interval_days": settings.INDEX_COUNT_INTERVAL_DAYS,+ "run_interval_seconds": settings.INDEX_COUNT_RUN_INTERVAL_SECONDS,+ "seconds_per_run": settings.INDEX_COUNT_SECONDS_PER_RUN,+ }+ if scan:+ num_pages = TinyIndex(item_factory=Document, index_path=index_path).num_pages+ next_page = int(scan["next_page"])+ status |= {+ "next_page": next_page,+ "num_pages": num_pages,+ "percent_done": 100 * next_page / num_pages,+ "num_results": int(scan["num_results"]),+ "urls_so_far": redis.pfcount(INDEX_SCAN_URL_HLL_KEY),+ "domains_so_far": redis.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY),+ "expires_in_seconds": redis.ttl(INDEX_SCAN_KEY),+ }+ return status+++def get_published_counts(redis: Redis, num_days: int) -> list[dict]:+ """The counts published by finished scans in the last num_days, most recent first."""+ today = utc_today()+ days = [today - timedelta(days=i) for i in range(num_days)]+ keys = (INDEX_URL_COUNT_KEY, INDEX_DOMAIN_COUNT_KEY, INDEX_RESULT_COUNT_KEY)+ counts = redis.mget([key.format(date=day) for day in days for key in keys])+ published = []+ for i, day in enumerate(days):+ urls, domains, results = counts[i * len(keys) : (i + 1) * len(keys)]+ if urls is not None:+ published.append({"date": day, "urls": int(urls), "domains": int(domains), "results": int(results)})+ return published++def get_domain_result_count(domain: str) -> int:redis = get_redis()diff --git a/mwmbl/templates/admin/index_count.html b/mwmbl/templates/admin/index_count.htmlnew file mode 100644index 0000000..447e103--- /dev/null+++ b/mwmbl/templates/admin/index_count.html@@ -0,0 +1,136 @@+{% extends "admin/base_site.html" %}+{% load humanize %}++{% block title %}Index count | {{ site_title|default:"Django site admin" }}{% endblock %}++{% block breadcrumbs %}+ <div class="breadcrumbs">+ <a href="{% url 'admin:index' %}">Home</a>+ › Index count+ </div>+{% endblock %}++{% block content %}+ <style>+ .index-count table { width: 100%; margin-bottom: 10px; }+ .index-count td.key { width: 30%; font-weight: bold; }+ .index-count .ok { color: #417690; font-weight: bold; }+ .index-count .bad { color: #ba2121; font-weight: bold; }+ .index-count .note { color: #666; font-size: 0.9em; }+ </style>++ <div id="content-main" class="index-count">++ {% if redis_error %}+ <div class="module">+ <h2>Redis unreachable</h2>+ <p class="bad">{{ redis_error }}</p>+ <p class="note">The scan keeps its progress in Redis, so it cannot advance until Redis is back.</p>+ </div>+ {% else %}++ <div class="module">+ <h2>Current scan</h2>+ <table>+ {% if scan.in_progress %}+ <tr>+ <td class="key">Progress</td>+ <td>+ page {{ scan.next_page|intcomma }} of {{ scan.num_pages|intcomma }}+ ({{ scan.percent_done|floatformat:1 }}%)+ </td>+ </tr>+ <tr><td class="key">Results so far</td><td>{{ scan.num_results|intcomma }}</td></tr>+ <tr><td class="key">Unique URLs so far</td><td>{{ scan.urls_so_far|intcomma }}</td></tr>+ <tr><td class="key">Unique domains so far</td><td>{{ scan.domains_so_far|intcomma }}</td></tr>+ <tr>+ <td class="key">State expires in</td>+ <td>+ {{ scan.expires_in_seconds|intcomma }}s+ <span class="note">— reset by every batch, so this falling well below+ two days means the scan has stopped advancing</span>+ </td>+ </tr>+ {% else %}+ <tr><td class="key">Scan in progress</td><td>no</td></tr>+ {% endif %}+ <tr>+ <td class="key">Last scan finished</td>+ <td>{% if scan.last_finished %}{{ scan.last_finished }}{% else %}<span class="bad">never</span>{% endif %}</td>+ </tr>+ <tr>+ <td class="key">Next scan due</td>+ <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>+ </tr>+ </table>+ <p class="note">+ A scan reads every page of the index into HyperLogLogs, about 1% accurate. It runs+ for {{ scan.seconds_per_run }}s every {{ scan.run_interval_seconds }}s, and a new one+ starts {{ scan.interval_days }} days after the last finished. Counts are published+ only when a scan reaches the last page.+ </p>+ </div>++ <div class="module">+ <h2>Published counts</h2>+ {% if published %}+ <table>+ <thead><tr><th>Date (UTC)</th><th>Unique URLs</th><th>Unique domains</th><th>Results</th></tr></thead>+ <tbody>+ {% for count in published %}+ <tr>+ <td>{{ count.date }}</td>+ <td>{{ count.urls|intcomma }}</td>+ <td>{{ count.domains|intcomma }}</td>+ <td>{{ count.results|intcomma }}</td>+ </tr>+ {% endfor %}+ </tbody>+ </table>+ {% else %}+ <p class="note">No scan has finished in the counts still held in Redis.</p>+ {% endif %}+ </div>++ {% endif %}++ <div class="module">+ <h2>Background task</h2>+ <table>+ <thead>+ <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>+ </thead>+ <tbody>+ {% for task in tasks %}+ <tr>+ <td><code>{{ task.name }}</code></td>+ <td>+ {% if task.task %}+ {{ task.task.run_at }}+ {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}+ {% else %}+ <span class="bad">not scheduled</span>+ {% endif %}+ </td>+ <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>+ <td>+ {% if task.last_completed %}+ {{ task.last_completed.run_at }}+ {% else %}+ <span class="bad">never</span>+ {% endif %}+ </td>+ <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>+ </tr>+ {% endfor %}+ </tbody>+ </table>+ <p class="note">+ The task logs and swallows its own errors so that it keeps being rescheduled, so a+ failing scan shows here as runs completing without the progress above moving. Look+ for "Error counting the URLs in the index" in the worker logs.+ </p>+ </div>++ </div>+{% endblock %}diff --git a/mwmbl/templates/admin/mwmbl_index.html b/mwmbl/templates/admin/mwmbl_index.htmlindex f4b3fa2..db5d247 100644--- a/mwmbl/templates/admin/mwmbl_index.html+++ b/mwmbl/templates/admin/mwmbl_index.html@@ -14,6 +14,10 @@<th scope="row"><a href="{% url 'blacklist_status' %}">Blacklist status</a></th><td>The published snapshot, the purge queue and the tasks that maintain them.</td></tr>+ <tr>+ <th scope="row"><a href="{% url 'index_count' %}">Index count</a></th>+ <td>Progress of the weekly scan counting unique URLs in the index, and its results.</td>+ </tr><tr><th scope="row"><a href="{% url 'paying_users' %}">Paying users</a></th><td>API users and members, their quotas, usage and what they pay, and Seed Search usage.</td>diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 2fd16b0..a7f6df0 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -21,7 +21,7 @@ from django.urls import include, pathfrom django.views.generic import RedirectViewimport mwmbl.crawler.app as crawler-from mwmbl.admin_views import blacklist_status_view, paying_users_view+from mwmbl.admin_views import blacklist_status_view, index_count_view, paying_users_viewfrom mwmbl.api import api as v1_apifrom mwmbl.api import register_routers, v2_apifrom mwmbl.search_setup import combined_ranker, ranker, tiny_index@@ -57,6 +57,7 @@ urlpatterns = [# staff_member_required on the view, matching the rest of the admin.path("admin/blacklist-status/", blacklist_status_view, name="blacklist_status"),path("admin/paying-users/", paying_users_view, name="paying_users"),+ path("admin/index-count/", index_count_view, name="index_count"),path("admin/", admin.site.urls),path("accounts/", include("allauth.urls")),path("", index, name="index"),diff --git a/test/test_admin_index_count.py b/test/test_admin_index_count.pynew file mode 100644index 0000000..10876e3--- /dev/null+++ b/test/test_admin_index_count.py@@ -0,0 +1,82 @@+"""Tests for the index count admin page (mwmbl.admin_views.index_count_view)."""++import fakeredis+import pytest+from django.contrib.auth import get_user_model+from redis import ConnectionError as RedisConnectionError++from mwmbl import count_urls+from mwmbl.count_urls import count_urls_step+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex++User = get_user_model()++STATUS_URL = "/admin/index-count/"+NUM_PAGES = 4+++def redis(monkeypatch):+ client = fakeredis.FakeRedis(decode_responses=True)+ monkeypatch.setattr(count_urls, "get_redis", lambda: client)+ return client+++def index_path(tmp_path, settings):+ settings.DATA_PATH = str(tmp_path)+ path = tmp_path / settings.INDEX_NAME+ TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=NUM_PAGES, page_size=PAGE_SIZE)+ with TinyIndex(Document, str(path), "w") as index:+ index.store_in_page(0, [Document(title="A", url="https://a.test/", extract="x", score=1.0, term="a")])+ return path+++def staff_client(client, db):+ user = User.objects.create_user(username="staff_member_1", password="correctpassword", is_staff=True)+ client.force_login(user)+ return client+++def test_non_staff_are_redirected_to_login(client, db):+ response = client.get(STATUS_URL)++ assert response.status_code == 302+ assert "/admin/login/" in response.url+++def test_shows_a_scan_in_progress(staff_client, redis, index_path, monkeypatch):+ monkeypatch.setattr(count_urls, "NUM_PAGES_IN_BATCH", 1)+ count_urls_step(redis, index_path, time_budget_seconds=0)++ response = staff_client.get(STATUS_URL)++ assert response.status_code == 200+ assert response.context["scan"]["in_progress"]+ assert response.context["scan"]["next_page"] == 1+ assert b"page 1 of 4" in response.content+++def test_shows_published_counts(staff_client, redis, index_path):+ count_urls_step(redis, index_path, time_budget_seconds=60)++ response = staff_client.get(STATUS_URL)++ assert response.status_code == 200+ assert not response.context["scan"]["in_progress"]+ assert response.context["published"][0]["urls"] == 1+ task_names = [task["name"] for task in response.context["tasks"]]+ assert task_names == ["mwmbl.background.count_index_urls"]+++def test_renders_when_redis_is_down(staff_client, index_path, monkeypatch):+ def unreachable():+ raise RedisConnectionError("Connection refused")++ monkeypatch.setattr(count_urls, "get_redis", unreachable)++ response = staff_client.get(STATUS_URL)++ assert response.status_code == 200+ assert b"Redis unreachable" in response.contentdiff --git a/test/test_count_urls.py b/test/test_count_urls.pyindex 31eec26..8aad31a 100644--- a/test/test_count_urls.py+++ b/test/test_count_urls.py@@ -22,6 +22,8 @@ from mwmbl.count_urls import (INDEX_URL_COUNT_KEY,count_urls_step,get_counts,+ get_published_counts,+ get_scan_status,)from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, PageError, TinyIndexfrom mwmbl.utils import utc_today@@ -186,3 +188,36 @@ def test_the_background_task_does_not_raise_so_it_keeps_repeating():with patch.object(background, "count_urls_step", side_effect=ConnectionError("Redis is down")):background.count_index_urls.now()+++def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_path):+ with patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):+ count_urls_step(redis, index_path, time_budget_seconds=0)++ status = get_scan_status(redis, index_path)++ assert status["in_progress"]+ assert status["next_page"] == 4+ assert status["num_pages"] == NUM_PAGES+ assert status["percent_done"] == 40+ assert status["num_results"] == 4+ assert status["urls_so_far"] == 3+ assert status["domains_so_far"] == 2+ assert status["last_finished"] is None+ assert status["next_scan_due"] is None+++def test_scan_status_after_a_scan_has_finished(redis, index_path, settings):+ count_urls_step(redis, index_path, time_budget_seconds=60)++ status = get_scan_status(redis, index_path)++ assert not status["in_progress"]+ assert status["last_finished"] == utc_today()+ assert status["next_scan_due"] == utc_today() + timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)+++def test_published_counts_list_only_the_days_a_scan_finished(redis, index_path):+ count_urls_step(redis, index_path, time_budget_seconds=60)++ assert get_published_counts(redis, num_days=30) == [{"date": utc_today(), "urls": 4, "domains": 3, "results": 5}]