Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add an admin page showing the weekly index URL count

Claudeab4792204995
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.py
index a000a5f..3ec5efd 100644
--- a/mwmbl/admin_views.py
+++ b/mwmbl/admin_views.py
@@ -1,4 +1,5 @@
-"""Admin-only status pages: the blacklist filtering state in Redis, and paying users.
+"""Admin-only status pages: the blacklist filtering state in Redis, the weekly index count,
+and paying users.
The blacklist page gives visibility on the blacklist filtering state that lives in Redis.
@@ -17,6 +18,7 @@ SRANDMEMBER so looking at it cannot consume it.
import os
from datetime import timedelta
from logging import getLogger
+from pathlib import Path
from background_task.models import CompletedTask, Task
from django.conf import settings
@@ -26,7 +28,7 @@ from django.db.models import F, OuterRef, Q, Subquery
from django.shortcuts import render
from redis import RedisError
-from mwmbl import pricing, quota
+from mwmbl import count_urls, pricing, quota
from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY
from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer import blacklist_snapshot, purge_queue
@@ -46,9 +48,11 @@ logger = getLogger(__name__)
SNAPSHOT_TASK_NAME = "mwmbl.background.refresh_blacklist_snapshot"
PURGE_TASK_NAME = "mwmbl.background.purge_blacklisted_from_queue"
+INDEX_COUNT_TASK_NAME = "mwmbl.background.count_index_urls"
QUEUE_SAMPLE_SIZE = 50
REMOVED_COUNT_DAYS = 14
+PUBLISHED_INDEX_COUNT_DAYS = 60
def _snapshot_status() -> dict:
@@ -130,15 +134,14 @@ def _curated_status() -> dict:
}
-def _task_status() -> list[dict]:
- """The two background tasks that maintain the state above.
+def _task_status(task_names: list[str]) -> list[dict]:
+ """The background tasks that maintain the state a status page reports on.
- Nothing in this loop runs without a `manage.py process_tasks` worker, and one running
- an image that predates these tasks skips them silently by name rather than failing -
- so "pending, run_at long past, never completed" is what a missing or stale worker
- looks like, and it is otherwise invisible.
+ None of them runs without a `manage.py process_tasks` worker, and one running an image
+ that predates a task skips it silently by name rather than failing - so "pending,
+ run_at long past, never completed" is what a missing or stale worker looks like, and
+ it is otherwise invisible.
"""
- task_names = [SNAPSHOT_TASK_NAME, PURGE_TASK_NAME]
pending = {task.task_name: task for task in Task.objects.filter(task_name__in=task_names)}
statuses = []
@@ -176,10 +179,28 @@ def blacklist_status_view(request):
logger.exception("Could not read blacklist status from Redis")
context["redis_error"] = str(e)
- context["tasks"] = _task_status()
+ context["tasks"] = _task_status([SNAPSHOT_TASK_NAME, PURGE_TASK_NAME])
return render(request, "admin/blacklist_status.html", context)
+@staff_member_required
+def index_count_view(request):
+ context = {"title": "Index count"}
+ index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
+ # Rendered with Redis down, like the blacklist page: the scan stalling because Redis
+ # is unreachable is one of the things this page is for.
+ try:
+ redis = count_urls.get_redis()
+ context["scan"] = count_urls.get_scan_status(redis, index_path)
+ context["published"] = count_urls.get_published_counts(redis, PUBLISHED_INDEX_COUNT_DAYS)
+ except RedisError as e:
+ logger.exception("Could not read the index count from Redis")
+ context["redis_error"] = str(e)
+
+ context["tasks"] = _task_status([INDEX_COUNT_TASK_NAME])
+ return render(request, "admin/index_count.html", context)
+
+
def _api_users() -> list[dict]:
"""Users holding a search API key, plus anyone with a Polar usage subscription.
diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.py
index 21704b7..c3d18d8 100644
--- a/mwmbl/count_urls.py
+++ b/mwmbl/count_urls.py
@@ -204,6 +204,53 @@ def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:
}
+def get_scan_status(redis: Redis, index_path: Path) -> dict:
+ """How far the scan in progress has got, and when the last one finished.
+
+ For the admin status page. The counts so far are PFCOUNTs of the scan's HyperLogLogs,
+ which is cheap, so looking at the scan does not slow it down.
+ """
+ scan = redis.hgetall(INDEX_SCAN_KEY)
+ last_finished = redis.get(INDEX_SCAN_LAST_FINISHED_KEY)
+ last_finished_date = None if last_finished is None else date.fromisoformat(last_finished)
+ interval = timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)
+ status = {
+ "in_progress": bool(scan),
+ "last_finished": last_finished_date,
+ "next_scan_due": None if last_finished_date is None else last_finished_date + interval,
+ "interval_days": settings.INDEX_COUNT_INTERVAL_DAYS,
+ "run_interval_seconds": settings.INDEX_COUNT_RUN_INTERVAL_SECONDS,
+ "seconds_per_run": settings.INDEX_COUNT_SECONDS_PER_RUN,
+ }
+ if scan:
+ num_pages = TinyIndex(item_factory=Document, index_path=index_path).num_pages
+ next_page = int(scan["next_page"])
+ status |= {
+ "next_page": next_page,
+ "num_pages": num_pages,
+ "percent_done": 100 * next_page / num_pages,
+ "num_results": int(scan["num_results"]),
+ "urls_so_far": redis.pfcount(INDEX_SCAN_URL_HLL_KEY),
+ "domains_so_far": redis.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY),
+ "expires_in_seconds": redis.ttl(INDEX_SCAN_KEY),
+ }
+ return status
+
+
+def get_published_counts(redis: Redis, num_days: int) -> list[dict]:
+ """The counts published by finished scans in the last num_days, most recent first."""
+ today = utc_today()
+ days = [today - timedelta(days=i) for i in range(num_days)]
+ keys = (INDEX_URL_COUNT_KEY, INDEX_DOMAIN_COUNT_KEY, INDEX_RESULT_COUNT_KEY)
+ counts = redis.mget([key.format(date=day) for day in days for key in keys])
+ published = []
+ for i, day in enumerate(days):
+ urls, domains, results = counts[i * len(keys) : (i + 1) * len(keys)]
+ if urls is not None:
+ published.append({"date": day, "urls": int(urls), "domains": int(domains), "results": int(results)})
+ return published
+
+
def get_domain_result_count(domain: str) -> int:
redis = get_redis()
diff --git a/mwmbl/templates/admin/index_count.html b/mwmbl/templates/admin/index_count.html
new file mode 100644
index 0000000..447e103
--- /dev/null
+++ b/mwmbl/templates/admin/index_count.html
@@ -0,0 +1,136 @@
+{% extends "admin/base_site.html" %}
+{% load humanize %}
+
+{% block title %}Index count | {{ site_title|default:"Django site admin" }}{% endblock %}
+
+{% block breadcrumbs %}
+ <div class="breadcrumbs">
+ <a href="{% url 'admin:index' %}">Home</a>
+ &rsaquo; Index count
+ </div>
+{% endblock %}
+
+{% block content %}
+ <style>
+ .index-count table { width: 100%; margin-bottom: 10px; }
+ .index-count td.key { width: 30%; font-weight: bold; }
+ .index-count .ok { color: #417690; font-weight: bold; }
+ .index-count .bad { color: #ba2121; font-weight: bold; }
+ .index-count .note { color: #666; font-size: 0.9em; }
+ </style>
+
+ <div id="content-main" class="index-count">
+
+ {% if redis_error %}
+ <div class="module">
+ <h2>Redis unreachable</h2>
+ <p class="bad">{{ redis_error }}</p>
+ <p class="note">The scan keeps its progress in Redis, so it cannot advance until Redis is back.</p>
+ </div>
+ {% else %}
+
+ <div class="module">
+ <h2>Current scan</h2>
+ <table>
+ {% if scan.in_progress %}
+ <tr>
+ <td class="key">Progress</td>
+ <td>
+ page {{ scan.next_page|intcomma }} of {{ scan.num_pages|intcomma }}
+ ({{ scan.percent_done|floatformat:1 }}%)
+ </td>
+ </tr>
+ <tr><td class="key">Results so far</td><td>{{ scan.num_results|intcomma }}</td></tr>
+ <tr><td class="key">Unique URLs so far</td><td>{{ scan.urls_so_far|intcomma }}</td></tr>
+ <tr><td class="key">Unique domains so far</td><td>{{ scan.domains_so_far|intcomma }}</td></tr>
+ <tr>
+ <td class="key">State expires in</td>
+ <td>
+ {{ scan.expires_in_seconds|intcomma }}s
+ <span class="note">&mdash; reset by every batch, so this falling well below
+ two days means the scan has stopped advancing</span>
+ </td>
+ </tr>
+ {% else %}
+ <tr><td class="key">Scan in progress</td><td>no</td></tr>
+ {% endif %}
+ <tr>
+ <td class="key">Last scan finished</td>
+ <td>{% if scan.last_finished %}{{ scan.last_finished }}{% else %}<span class="bad">never</span>{% endif %}</td>
+ </tr>
+ <tr>
+ <td class="key">Next scan due</td>
+ <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>
+ </tr>
+ </table>
+ <p class="note">
+ A scan reads every page of the index into HyperLogLogs, about 1% accurate. It runs
+ for {{ scan.seconds_per_run }}s every {{ scan.run_interval_seconds }}s, and a new one
+ starts {{ scan.interval_days }} days after the last finished. Counts are published
+ only when a scan reaches the last page.
+ </p>
+ </div>
+
+ <div class="module">
+ <h2>Published counts</h2>
+ {% if published %}
+ <table>
+ <thead><tr><th>Date (UTC)</th><th>Unique URLs</th><th>Unique domains</th><th>Results</th></tr></thead>
+ <tbody>
+ {% for count in published %}
+ <tr>
+ <td>{{ count.date }}</td>
+ <td>{{ count.urls|intcomma }}</td>
+ <td>{{ count.domains|intcomma }}</td>
+ <td>{{ count.results|intcomma }}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ {% else %}
+ <p class="note">No scan has finished in the counts still held in Redis.</p>
+ {% endif %}
+ </div>
+
+ {% endif %}
+
+ <div class="module">
+ <h2>Background task</h2>
+ <table>
+ <thead>
+ <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>
+ </thead>
+ <tbody>
+ {% for task in tasks %}
+ <tr>
+ <td><code>{{ task.name }}</code></td>
+ <td>
+ {% if task.task %}
+ {{ task.task.run_at }}
+ {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}
+ {% else %}
+ <span class="bad">not scheduled</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>
+ <td>
+ {% if task.last_completed %}
+ {{ task.last_completed.run_at }}
+ {% else %}
+ <span class="bad">never</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+ </table>
+ <p class="note">
+ The task logs and swallows its own errors so that it keeps being rescheduled, so a
+ failing scan shows here as runs completing without the progress above moving. Look
+ for "Error counting the URLs in the index" in the worker logs.
+ </p>
+ </div>
+
+ </div>
+{% endblock %}
diff --git a/mwmbl/templates/admin/mwmbl_index.html b/mwmbl/templates/admin/mwmbl_index.html
index f4b3fa2..db5d247 100644
--- a/mwmbl/templates/admin/mwmbl_index.html
+++ b/mwmbl/templates/admin/mwmbl_index.html
@@ -14,6 +14,10 @@
<th scope="row"><a href="{% url 'blacklist_status' %}">Blacklist status</a></th>
<td>The published snapshot, the purge queue and the tasks that maintain them.</td>
</tr>
+ <tr>
+ <th scope="row"><a href="{% url 'index_count' %}">Index count</a></th>
+ <td>Progress of the weekly scan counting unique URLs in the index, and its results.</td>
+ </tr>
<tr>
<th scope="row"><a href="{% url 'paying_users' %}">Paying users</a></th>
<td>API users and members, their quotas, usage and what they pay, and Seed Search usage.</td>
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 2fd16b0..a7f6df0 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -21,7 +21,7 @@ from django.urls import include, path
from django.views.generic import RedirectView
import mwmbl.crawler.app as crawler
-from mwmbl.admin_views import blacklist_status_view, paying_users_view
+from mwmbl.admin_views import blacklist_status_view, index_count_view, paying_users_view
from mwmbl.api import api as v1_api
from mwmbl.api import register_routers, v2_api
from mwmbl.search_setup import combined_ranker, ranker, tiny_index
@@ -57,6 +57,7 @@ urlpatterns = [
# staff_member_required on the view, matching the rest of the admin.
path("admin/blacklist-status/", blacklist_status_view, name="blacklist_status"),
path("admin/paying-users/", paying_users_view, name="paying_users"),
+ path("admin/index-count/", index_count_view, name="index_count"),
path("admin/", admin.site.urls),
path("accounts/", include("allauth.urls")),
path("", index, name="index"),
diff --git a/test/test_admin_index_count.py b/test/test_admin_index_count.py
new file mode 100644
index 0000000..10876e3
--- /dev/null
+++ b/test/test_admin_index_count.py
@@ -0,0 +1,82 @@
+"""Tests for the index count admin page (mwmbl.admin_views.index_count_view)."""
+
+import fakeredis
+import pytest
+from django.contrib.auth import get_user_model
+from redis import ConnectionError as RedisConnectionError
+
+from mwmbl import count_urls
+from mwmbl.count_urls import count_urls_step
+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex
+
+User = get_user_model()
+
+STATUS_URL = "/admin/index-count/"
+NUM_PAGES = 4
+
+
+def redis(monkeypatch):
+ client = fakeredis.FakeRedis(decode_responses=True)
+ monkeypatch.setattr(count_urls, "get_redis", lambda: client)
+ return client
+
+
+def index_path(tmp_path, settings):
+ settings.DATA_PATH = str(tmp_path)
+ path = tmp_path / settings.INDEX_NAME
+ TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=NUM_PAGES, page_size=PAGE_SIZE)
+ with TinyIndex(Document, str(path), "w") as index:
+ index.store_in_page(0, [Document(title="A", url="https://a.test/", extract="x", score=1.0, term="a")])
+ return path
+
+
+def staff_client(client, db):
+ user = User.objects.create_user(username="staff_member_1", password="correctpassword", is_staff=True)
+ client.force_login(user)
+ return client
+
+
+def test_non_staff_are_redirected_to_login(client, db):
+ response = client.get(STATUS_URL)
+
+ assert response.status_code == 302
+ assert "/admin/login/" in response.url
+
+
+def test_shows_a_scan_in_progress(staff_client, redis, index_path, monkeypatch):
+ monkeypatch.setattr(count_urls, "NUM_PAGES_IN_BATCH", 1)
+ count_urls_step(redis, index_path, time_budget_seconds=0)
+
+ response = staff_client.get(STATUS_URL)
+
+ assert response.status_code == 200
+ assert response.context["scan"]["in_progress"]
+ assert response.context["scan"]["next_page"] == 1
+ assert b"page 1 of 4" in response.content
+
+
+def test_shows_published_counts(staff_client, redis, index_path):
+ count_urls_step(redis, index_path, time_budget_seconds=60)
+
+ response = staff_client.get(STATUS_URL)
+
+ assert response.status_code == 200
+ assert not response.context["scan"]["in_progress"]
+ assert response.context["published"][0]["urls"] == 1
+ task_names = [task["name"] for task in response.context["tasks"]]
+ assert task_names == ["mwmbl.background.count_index_urls"]
+
+
+def test_renders_when_redis_is_down(staff_client, index_path, monkeypatch):
+ def unreachable():
+ raise RedisConnectionError("Connection refused")
+
+ monkeypatch.setattr(count_urls, "get_redis", unreachable)
+
+ response = staff_client.get(STATUS_URL)
+
+ assert response.status_code == 200
+ assert b"Redis unreachable" in response.content
diff --git a/test/test_count_urls.py b/test/test_count_urls.py
index 31eec26..8aad31a 100644
--- a/test/test_count_urls.py
+++ b/test/test_count_urls.py
@@ -22,6 +22,8 @@ from mwmbl.count_urls import (
INDEX_URL_COUNT_KEY,
count_urls_step,
get_counts,
+ get_published_counts,
+ get_scan_status,
)
from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, PageError, TinyIndex
from mwmbl.utils import utc_today
@@ -186,3 +188,36 @@ def test_the_background_task_does_not_raise_so_it_keeps_repeating():
with patch.object(background, "count_urls_step", side_effect=ConnectionError("Redis is down")):
background.count_index_urls.now()
+
+
+def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_path):
+ with patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):
+ count_urls_step(redis, index_path, time_budget_seconds=0)
+
+ status = get_scan_status(redis, index_path)
+
+ assert status["in_progress"]
+ assert status["next_page"] == 4
+ assert status["num_pages"] == NUM_PAGES
+ assert status["percent_done"] == 40
+ assert status["num_results"] == 4
+ assert status["urls_so_far"] == 3
+ assert status["domains_so_far"] == 2
+ assert status["last_finished"] is None
+ assert status["next_scan_due"] is None
+
+
+def test_scan_status_after_a_scan_has_finished(redis, index_path, settings):
+ count_urls_step(redis, index_path, time_budget_seconds=60)
+
+ status = get_scan_status(redis, index_path)
+
+ assert not status["in_progress"]
+ assert status["last_finished"] == utc_today()
+ assert status["next_scan_due"] == utc_today() + timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)
+
+
+def test_published_counts_list_only_the_days_a_scan_finished(redis, index_path):
+ count_urls_step(redis, index_path, time_budget_seconds=60)
+
+ assert get_published_counts(redis, num_days=30) == [{"date": utc_today(), "urls": 4, "domains": 3, "results": 5}]