An open source, non-profit web search engine
Harden the index count admin page
Claudee5726cc5a4ec
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.pyindex 3ec5efd..3a7471b 100644--- a/mwmbl/admin_views.py+++ b/mwmbl/admin_views.py@@ -18,7 +18,6 @@ SRANDMEMBER so looking at it cannot consume it.import osfrom datetime import timedeltafrom logging import getLogger-from pathlib import Pathfrom background_task.models import CompletedTask, Taskfrom django.conf import settings@@ -28,7 +27,7 @@ from django.db.models import F, OuterRef, Q, Subqueryfrom django.shortcuts import renderfrom redis import RedisError-from mwmbl import count_urls, pricing, quota+from mwmbl import count_urls, pricing, quota, search_setupfrom mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEYfrom mwmbl.curated_domains import get_curated_domainsfrom mwmbl.indexer import blacklist_snapshot, purge_queue@@ -52,7 +51,8 @@ INDEX_COUNT_TASK_NAME = "mwmbl.background.count_index_urls"QUEUE_SAMPLE_SIZE = 50REMOVED_COUNT_DAYS = 14-PUBLISHED_INDEX_COUNT_DAYS = 60+# As far back as published counts are kept.+PUBLISHED_INDEX_COUNT_DAYS = count_urls.LONG_EXPIRE_SECONDS // (60 * 60 * 24)def _snapshot_status() -> dict:@@ -186,12 +186,12 @@ def blacklist_status_view(request):@staff_member_requireddef index_count_view(request):context = {"title": "Index count"}- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME# Rendered with Redis down, like the blacklist page: the scan stalling because Redis# is unreachable is one of the things this page is for.try:redis = count_urls.get_redis()- context["scan"] = count_urls.get_scan_status(redis, index_path)+ # The index this process already has open, rather than opening it again per request.+ context["scan"] = count_urls.get_scan_status(redis, search_setup.tiny_index.num_pages)context["published"] = count_urls.get_published_counts(redis, PUBLISHED_INDEX_COUNT_DAYS)except RedisError as e:logger.exception("Could not read the index count from Redis")diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.pyindex c3d18d8..e432161 100644--- a/mwmbl/count_urls.py+++ b/mwmbl/count_urls.py@@ -174,22 +174,20 @@ def _finish_scan(redis: Redis):f"Counted {url_count} unique URLs, {domain_count} unique domains and {num_results} results in the index.")+ # One transaction, so the counts are published together or not at all.today = utc_today()- _set_count(INDEX_URL_COUNT_KEY, redis, today, url_count)- _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, domain_count)- _set_count(INDEX_RESULT_COUNT_KEY, redis, today, num_results)-pipeline = redis.pipeline()+ for key, count in (+ (INDEX_URL_COUNT_KEY, url_count),+ (INDEX_DOMAIN_COUNT_KEY, domain_count),+ (INDEX_RESULT_COUNT_KEY, num_results),+ ):+ pipeline.set(key.format(date=today), count, ex=LONG_EXPIRE_SECONDS)pipeline.set(INDEX_SCAN_LAST_FINISHED_KEY, str(today))pipeline.delete(INDEX_SCAN_KEY, INDEX_SCAN_URL_HLL_KEY, INDEX_SCAN_DOMAIN_HLL_KEY)pipeline.execute()-def _set_count(key, redis, today, count):- redis.set(key.format(date=today), count)- redis.expire(key.format(date=today), LONG_EXPIRE_SECONDS)--def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:"""The index counts for each of the last 30 days.@@ -204,14 +202,23 @@ def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:}-def get_scan_status(redis: Redis, index_path: Path) -> dict:+def get_scan_status(redis: Redis, num_pages: int) -> dict:"""How far the scan in progress has got, and when the last one finished.- For the admin status page. The counts so far are PFCOUNTs of the scan's HyperLogLogs,- which is cheap, so looking at the scan does not slow it down.+ For the admin status page. num_pages is the size of the index being scanned, taken+ from the index the caller already has open. The counts so far are PFCOUNTs of the+ scan's HyperLogLogs, which is cheap, so looking at the scan does not slow it down."""- scan = redis.hgetall(INDEX_SCAN_KEY)- last_finished = redis.get(INDEX_SCAN_LAST_FINISHED_KEY)+ # Read in one transaction, so a scan finishing or expiring part way through is not+ # shown as a mix of its old and new state.+ pipeline = redis.pipeline(transaction=True)+ pipeline.hgetall(INDEX_SCAN_KEY)+ pipeline.get(INDEX_SCAN_LAST_FINISHED_KEY)+ pipeline.pfcount(INDEX_SCAN_URL_HLL_KEY)+ pipeline.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY)+ pipeline.ttl(INDEX_SCAN_KEY)+ scan, last_finished, urls_so_far, domains_so_far, expires_in_seconds = pipeline.execute()+last_finished_date = None if last_finished is None else date.fromisoformat(last_finished)interval = timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)status = {@@ -223,16 +230,15 @@ def get_scan_status(redis: Redis, index_path: Path) -> dict:"seconds_per_run": settings.INDEX_COUNT_SECONDS_PER_RUN,}if scan:- num_pages = TinyIndex(item_factory=Document, index_path=index_path).num_pagesnext_page = int(scan["next_page"])status |= {"next_page": next_page,"num_pages": num_pages,- "percent_done": 100 * next_page / num_pages,+ "percent_done": 100 * next_page / num_pages if num_pages else None,"num_results": int(scan["num_results"]),- "urls_so_far": redis.pfcount(INDEX_SCAN_URL_HLL_KEY),- "domains_so_far": redis.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY),- "expires_in_seconds": redis.ttl(INDEX_SCAN_KEY),+ "urls_so_far": urls_so_far,+ "domains_so_far": domains_so_far,+ "expires_in_seconds": expires_in_seconds,}return status@@ -245,9 +251,11 @@ def get_published_counts(redis: Redis, num_days: int) -> list[dict]:counts = redis.mget([key.format(date=day) for day in days for key in keys])published = []for i, day in enumerate(days):- urls, domains, results = counts[i * len(keys) : (i + 1) * len(keys)]- if urls is not None:- published.append({"date": day, "urls": int(urls), "domains": int(domains), "results": int(results)})+ day_counts = counts[i * len(keys) : (i + 1) * len(keys)]+ # A day missing any of its counts was not published whole, so it is left out.+ if None not in day_counts:+ urls, domains, results = map(int, day_counts)+ published.append({"date": day, "urls": urls, "domains": domains, "results": results})return publisheddiff --git a/mwmbl/templates/admin/_task_status.html b/mwmbl/templates/admin/_task_status.htmlnew file mode 100644index 0000000..2a04ff9--- /dev/null+++ b/mwmbl/templates/admin/_task_status.html@@ -0,0 +1,30 @@+{% comment %} The rows from admin_views._task_status. {% endcomment %}+<table>+ <thead>+ <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>+ </thead>+ <tbody>+ {% for task in tasks %}+ <tr>+ <td><code>{{ task.name }}</code></td>+ <td>+ {% if task.task %}+ {{ task.task.run_at }}+ {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}+ {% else %}+ <span class="bad">not scheduled</span>+ {% endif %}+ </td>+ <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>+ <td>+ {% if task.last_completed %}+ {{ task.last_completed.run_at }}+ {% else %}+ <span class="bad">never</span>+ {% endif %}+ </td>+ <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>+ </tr>+ {% endfor %}+ </tbody>+</table>diff --git a/mwmbl/templates/admin/blacklist_status.html b/mwmbl/templates/admin/blacklist_status.htmlindex 86bc837..90d79bf 100644--- a/mwmbl/templates/admin/blacklist_status.html+++ b/mwmbl/templates/admin/blacklist_status.html@@ -204,35 +204,7 @@<div class="module"><h2>Background tasks</h2>- <table>- <thead>- <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>- </thead>- <tbody>- {% for task in tasks %}- <tr>- <td><code>{{ task.name }}</code></td>- <td>- {% if task.task %}- {{ task.task.run_at }}- {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}- {% else %}- <span class="bad">not scheduled</span>- {% endif %}- </td>- <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>- <td>- {% if task.last_completed %}- {{ task.last_completed.run_at }}- {% else %}- <span class="bad">never</span>- {% endif %}- </td>- <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>- </tr>- {% endfor %}- </tbody>- </table>+ {% include "admin/_task_status.html" %}<p class="note">Both tasks only run under a <code>manage.py process_tasks</code> worker. A workerrunning an image that predates these tasks skips them silently by name, which showsdiff --git a/mwmbl/templates/admin/index_count.html b/mwmbl/templates/admin/index_count.htmlindex 447e103..747d635 100644--- a/mwmbl/templates/admin/index_count.html+++ b/mwmbl/templates/admin/index_count.html@@ -37,7 +37,7 @@<td class="key">Progress</td><td>page {{ scan.next_page|intcomma }} of {{ scan.num_pages|intcomma }}- ({{ scan.percent_done|floatformat:1 }}%)+ {% if scan.percent_done is not None %}({{ scan.percent_done|floatformat:1 }}%){% endif %}</td></tr><tr><td class="key">Results so far</td><td>{{ scan.num_results|intcomma }}</td></tr>@@ -58,10 +58,12 @@<td class="key">Last scan finished</td><td>{% if scan.last_finished %}{{ scan.last_finished }}{% else %}<span class="bad">never</span>{% endif %}</td></tr>- <tr>- <td class="key">Next scan due</td>- <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>- </tr>+ {% if not scan.in_progress %}+ <tr>+ <td class="key">Next scan due</td>+ <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>+ </tr>+ {% endif %}</table><p class="note">A scan reads every page of the index into HyperLogLogs, about 1% accurate. It runs@@ -96,35 +98,7 @@<div class="module"><h2>Background task</h2>- <table>- <thead>- <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>- </thead>- <tbody>- {% for task in tasks %}- <tr>- <td><code>{{ task.name }}</code></td>- <td>- {% if task.task %}- {{ task.task.run_at }}- {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}- {% else %}- <span class="bad">not scheduled</span>- {% endif %}- </td>- <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>- <td>- {% if task.last_completed %}- {{ task.last_completed.run_at }}- {% else %}- <span class="bad">never</span>- {% endif %}- </td>- <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>- </tr>- {% endfor %}- </tbody>- </table>+ {% include "admin/_task_status.html" %}<p class="note">The task logs and swallows its own errors so that it keeps being rescheduled, so afailing scan shows here as runs completing without the progress above moving. Lookdiff --git a/test/test_admin_index_count.py b/test/test_admin_index_count.pyindex 10876e3..dfe2898 100644--- a/test/test_admin_index_count.py+++ b/test/test_admin_index_count.py@@ -5,7 +5,7 @@ import pytestfrom django.contrib.auth import get_user_modelfrom redis import ConnectionError as RedisConnectionError-from mwmbl import count_urls+from mwmbl import count_urls, search_setupfrom mwmbl.count_urls import count_urls_stepfrom mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex@@ -23,12 +23,13 @@ def redis(monkeypatch):@pytest.fixture-def index_path(tmp_path, settings):- settings.DATA_PATH = str(tmp_path)- path = tmp_path / settings.INDEX_NAME+def index_path(tmp_path, monkeypatch):+ path = tmp_path / "index.tinysearch"TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=NUM_PAGES, page_size=PAGE_SIZE)with TinyIndex(Document, str(path), "w") as index:index.store_in_page(0, [Document(title="A", url="https://a.test/", extract="x", score=1.0, term="a")])+ # The page reads the size of the index the web process has open.+ monkeypatch.setattr(search_setup, "tiny_index", TinyIndex(Document, str(path)))return path@@ -56,6 +57,7 @@ def test_shows_a_scan_in_progress(staff_client, redis, index_path, monkeypatch):assert response.context["scan"]["in_progress"]assert response.context["scan"]["next_page"] == 1assert b"page 1 of 4" in response.content+ assert b"Next scan due" not in response.contentdef test_shows_published_counts(staff_client, redis, index_path):diff --git a/test/test_count_urls.py b/test/test_count_urls.pyindex 8aad31a..0758009 100644--- a/test/test_count_urls.py+++ b/test/test_count_urls.py@@ -194,7 +194,7 @@ def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_patwith patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):count_urls_step(redis, index_path, time_budget_seconds=0)- status = get_scan_status(redis, index_path)+ status = get_scan_status(redis, NUM_PAGES)assert status["in_progress"]assert status["next_page"] == 4@@ -210,7 +210,7 @@ def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_patdef test_scan_status_after_a_scan_has_finished(redis, index_path, settings):count_urls_step(redis, index_path, time_budget_seconds=60)- status = get_scan_status(redis, index_path)+ status = get_scan_status(redis, NUM_PAGES)assert not status["in_progress"]assert status["last_finished"] == utc_today()@@ -221,3 +221,16 @@ def test_published_counts_list_only_the_days_a_scan_finished(redis, index_path):count_urls_step(redis, index_path, time_budget_seconds=60)assert get_published_counts(redis, num_days=30) == [{"date": utc_today(), "urls": 4, "domains": 3, "results": 5}]+++def test_scan_status_of_an_empty_index_has_no_percent_done(redis, index_path):+ with patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):+ count_urls_step(redis, index_path, time_budget_seconds=0)++ assert get_scan_status(redis, num_pages=0)["percent_done"] is None+++def test_published_counts_skip_a_day_missing_some_of_its_counts(redis):+ redis.set(INDEX_URL_COUNT_KEY.format(date=utc_today()), 10)++ assert get_published_counts(redis, num_days=30) == []