Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Harden the index count admin page

Claudee5726cc5a4ec
diff --git a/mwmbl/admin_views.py b/mwmbl/admin_views.py
index 3ec5efd..3a7471b 100644
--- a/mwmbl/admin_views.py
+++ b/mwmbl/admin_views.py
@@ -18,7 +18,6 @@ SRANDMEMBER so looking at it cannot consume it.
import os
from datetime import timedelta
from logging import getLogger
-from pathlib import Path
from background_task.models import CompletedTask, Task
from django.conf import settings
@@ -28,7 +27,7 @@ from django.db.models import F, OuterRef, Q, Subquery
from django.shortcuts import render
from redis import RedisError
-from mwmbl import count_urls, pricing, quota
+from mwmbl import count_urls, pricing, quota, search_setup
from mwmbl.crawler.stats import BLACKLISTED_REMOVED_COUNT_KEY
from mwmbl.curated_domains import get_curated_domains
from mwmbl.indexer import blacklist_snapshot, purge_queue
@@ -52,7 +51,8 @@ INDEX_COUNT_TASK_NAME = "mwmbl.background.count_index_urls"
QUEUE_SAMPLE_SIZE = 50
REMOVED_COUNT_DAYS = 14
-PUBLISHED_INDEX_COUNT_DAYS = 60
+# As far back as published counts are kept.
+PUBLISHED_INDEX_COUNT_DAYS = count_urls.LONG_EXPIRE_SECONDS // (60 * 60 * 24)
def _snapshot_status() -> dict:
@@ -186,12 +186,12 @@ def blacklist_status_view(request):
@staff_member_required
def index_count_view(request):
context = {"title": "Index count"}
- index_path = Path(settings.DATA_PATH) / settings.INDEX_NAME
# Rendered with Redis down, like the blacklist page: the scan stalling because Redis
# is unreachable is one of the things this page is for.
try:
redis = count_urls.get_redis()
- context["scan"] = count_urls.get_scan_status(redis, index_path)
+ # The index this process already has open, rather than opening it again per request.
+ context["scan"] = count_urls.get_scan_status(redis, search_setup.tiny_index.num_pages)
context["published"] = count_urls.get_published_counts(redis, PUBLISHED_INDEX_COUNT_DAYS)
except RedisError as e:
logger.exception("Could not read the index count from Redis")
diff --git a/mwmbl/count_urls.py b/mwmbl/count_urls.py
index c3d18d8..e432161 100644
--- a/mwmbl/count_urls.py
+++ b/mwmbl/count_urls.py
@@ -174,22 +174,20 @@ def _finish_scan(redis: Redis):
f"Counted {url_count} unique URLs, {domain_count} unique domains and {num_results} results in the index."
)
+ # One transaction, so the counts are published together or not at all.
today = utc_today()
- _set_count(INDEX_URL_COUNT_KEY, redis, today, url_count)
- _set_count(INDEX_DOMAIN_COUNT_KEY, redis, today, domain_count)
- _set_count(INDEX_RESULT_COUNT_KEY, redis, today, num_results)
-
pipeline = redis.pipeline()
+ for key, count in (
+ (INDEX_URL_COUNT_KEY, url_count),
+ (INDEX_DOMAIN_COUNT_KEY, domain_count),
+ (INDEX_RESULT_COUNT_KEY, num_results),
+ ):
+ pipeline.set(key.format(date=today), count, ex=LONG_EXPIRE_SECONDS)
pipeline.set(INDEX_SCAN_LAST_FINISHED_KEY, str(today))
pipeline.delete(INDEX_SCAN_KEY, INDEX_SCAN_URL_HLL_KEY, INDEX_SCAN_DOMAIN_HLL_KEY)
pipeline.execute()
-def _set_count(key, redis, today, count):
- redis.set(key.format(date=today), count)
- redis.expire(key.format(date=today), LONG_EXPIRE_SECONDS)
-
-
def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:
"""The index counts for each of the last 30 days.
@@ -204,14 +202,23 @@ def get_counts(redis: Redis | None = None) -> dict[str, dict[str, int]]:
}
-def get_scan_status(redis: Redis, index_path: Path) -> dict:
+def get_scan_status(redis: Redis, num_pages: int) -> dict:
"""How far the scan in progress has got, and when the last one finished.
- For the admin status page. The counts so far are PFCOUNTs of the scan's HyperLogLogs,
- which is cheap, so looking at the scan does not slow it down.
+ For the admin status page. num_pages is the size of the index being scanned, taken
+ from the index the caller already has open. The counts so far are PFCOUNTs of the
+ scan's HyperLogLogs, which is cheap, so looking at the scan does not slow it down.
"""
- scan = redis.hgetall(INDEX_SCAN_KEY)
- last_finished = redis.get(INDEX_SCAN_LAST_FINISHED_KEY)
+ # Read in one transaction, so a scan finishing or expiring part way through is not
+ # shown as a mix of its old and new state.
+ pipeline = redis.pipeline(transaction=True)
+ pipeline.hgetall(INDEX_SCAN_KEY)
+ pipeline.get(INDEX_SCAN_LAST_FINISHED_KEY)
+ pipeline.pfcount(INDEX_SCAN_URL_HLL_KEY)
+ pipeline.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY)
+ pipeline.ttl(INDEX_SCAN_KEY)
+ scan, last_finished, urls_so_far, domains_so_far, expires_in_seconds = pipeline.execute()
+
last_finished_date = None if last_finished is None else date.fromisoformat(last_finished)
interval = timedelta(days=settings.INDEX_COUNT_INTERVAL_DAYS)
status = {
@@ -223,16 +230,15 @@ def get_scan_status(redis: Redis, index_path: Path) -> dict:
"seconds_per_run": settings.INDEX_COUNT_SECONDS_PER_RUN,
}
if scan:
- num_pages = TinyIndex(item_factory=Document, index_path=index_path).num_pages
next_page = int(scan["next_page"])
status |= {
"next_page": next_page,
"num_pages": num_pages,
- "percent_done": 100 * next_page / num_pages,
+ "percent_done": 100 * next_page / num_pages if num_pages else None,
"num_results": int(scan["num_results"]),
- "urls_so_far": redis.pfcount(INDEX_SCAN_URL_HLL_KEY),
- "domains_so_far": redis.pfcount(INDEX_SCAN_DOMAIN_HLL_KEY),
- "expires_in_seconds": redis.ttl(INDEX_SCAN_KEY),
+ "urls_so_far": urls_so_far,
+ "domains_so_far": domains_so_far,
+ "expires_in_seconds": expires_in_seconds,
}
return status
@@ -245,9 +251,11 @@ def get_published_counts(redis: Redis, num_days: int) -> list[dict]:
counts = redis.mget([key.format(date=day) for day in days for key in keys])
published = []
for i, day in enumerate(days):
- urls, domains, results = counts[i * len(keys) : (i + 1) * len(keys)]
- if urls is not None:
- published.append({"date": day, "urls": int(urls), "domains": int(domains), "results": int(results)})
+ day_counts = counts[i * len(keys) : (i + 1) * len(keys)]
+ # A day missing any of its counts was not published whole, so it is left out.
+ if None not in day_counts:
+ urls, domains, results = map(int, day_counts)
+ published.append({"date": day, "urls": urls, "domains": domains, "results": results})
return published
diff --git a/mwmbl/templates/admin/_task_status.html b/mwmbl/templates/admin/_task_status.html
new file mode 100644
index 0000000..2a04ff9
--- /dev/null
+++ b/mwmbl/templates/admin/_task_status.html
@@ -0,0 +1,30 @@
+{% comment %} The rows from admin_views._task_status. {% endcomment %}
+<table>
+ <thead>
+ <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>
+ </thead>
+ <tbody>
+ {% for task in tasks %}
+ <tr>
+ <td><code>{{ task.name }}</code></td>
+ <td>
+ {% if task.task %}
+ {{ task.task.run_at }}
+ {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}
+ {% else %}
+ <span class="bad">not scheduled</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>
+ <td>
+ {% if task.last_completed %}
+ {{ task.last_completed.run_at }}
+ {% else %}
+ <span class="bad">never</span>
+ {% endif %}
+ </td>
+ <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>
+ </tr>
+ {% endfor %}
+ </tbody>
+</table>
diff --git a/mwmbl/templates/admin/blacklist_status.html b/mwmbl/templates/admin/blacklist_status.html
index 86bc837..90d79bf 100644
--- a/mwmbl/templates/admin/blacklist_status.html
+++ b/mwmbl/templates/admin/blacklist_status.html
@@ -204,35 +204,7 @@
<div class="module">
<h2>Background tasks</h2>
- <table>
- <thead>
- <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>
- </thead>
- <tbody>
- {% for task in tasks %}
- <tr>
- <td><code>{{ task.name }}</code></td>
- <td>
- {% if task.task %}
- {{ task.task.run_at }}
- {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}
- {% else %}
- <span class="bad">not scheduled</span>
- {% endif %}
- </td>
- <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>
- <td>
- {% if task.last_completed %}
- {{ task.last_completed.run_at }}
- {% else %}
- <span class="bad">never</span>
- {% endif %}
- </td>
- <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>
- </tr>
- {% endfor %}
- </tbody>
- </table>
+ {% include "admin/_task_status.html" %}
<p class="note">
Both tasks only run under a <code>manage.py process_tasks</code> worker. A worker
running an image that predates these tasks skips them silently by name, which shows
diff --git a/mwmbl/templates/admin/index_count.html b/mwmbl/templates/admin/index_count.html
index 447e103..747d635 100644
--- a/mwmbl/templates/admin/index_count.html
+++ b/mwmbl/templates/admin/index_count.html
@@ -37,7 +37,7 @@
<td class="key">Progress</td>
<td>
page {{ scan.next_page|intcomma }} of {{ scan.num_pages|intcomma }}
- ({{ scan.percent_done|floatformat:1 }}%)
+ {% if scan.percent_done is not None %}({{ scan.percent_done|floatformat:1 }}%){% endif %}
</td>
</tr>
<tr><td class="key">Results so far</td><td>{{ scan.num_results|intcomma }}</td></tr>
@@ -58,10 +58,12 @@
<td class="key">Last scan finished</td>
<td>{% if scan.last_finished %}{{ scan.last_finished }}{% else %}<span class="bad">never</span>{% endif %}</td>
</tr>
- <tr>
- <td class="key">Next scan due</td>
- <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>
- </tr>
+ {% if not scan.in_progress %}
+ <tr>
+ <td class="key">Next scan due</td>
+ <td>{% if scan.next_scan_due %}{{ scan.next_scan_due }}{% else %}now{% endif %}</td>
+ </tr>
+ {% endif %}
</table>
<p class="note">
A scan reads every page of the index into HyperLogLogs, about 1% accurate. It runs
@@ -96,35 +98,7 @@
<div class="module">
<h2>Background task</h2>
- <table>
- <thead>
- <tr><th>Task</th><th>Next run</th><th>Attempts</th><th>Last completed</th><th>Last error</th></tr>
- </thead>
- <tbody>
- {% for task in tasks %}
- <tr>
- <td><code>{{ task.name }}</code></td>
- <td>
- {% if task.task %}
- {{ task.task.run_at }}
- {% if task.task.locked_by %}<br><span class="note">running on {{ task.task.locked_by }}</span>{% endif %}
- {% else %}
- <span class="bad">not scheduled</span>
- {% endif %}
- </td>
- <td>{% if task.task %}{{ task.task.attempts }}{% endif %}</td>
- <td>
- {% if task.last_completed %}
- {{ task.last_completed.run_at }}
- {% else %}
- <span class="bad">never</span>
- {% endif %}
- </td>
- <td>{% if task.task.last_error %}<span class="bad">{{ task.task.last_error|truncatechars:200 }}</span>{% endif %}</td>
- </tr>
- {% endfor %}
- </tbody>
- </table>
+ {% include "admin/_task_status.html" %}
<p class="note">
The task logs and swallows its own errors so that it keeps being rescheduled, so a
failing scan shows here as runs completing without the progress above moving. Look
diff --git a/test/test_admin_index_count.py b/test/test_admin_index_count.py
index 10876e3..dfe2898 100644
--- a/test/test_admin_index_count.py
+++ b/test/test_admin_index_count.py
@@ -5,7 +5,7 @@ import pytest
from django.contrib.auth import get_user_model
from redis import ConnectionError as RedisConnectionError
-from mwmbl import count_urls
+from mwmbl import count_urls, search_setup
from mwmbl.count_urls import count_urls_step
from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, TinyIndex
@@ -23,12 +23,13 @@ def redis(monkeypatch):
@pytest.fixture
-def index_path(tmp_path, settings):
- settings.DATA_PATH = str(tmp_path)
- path = tmp_path / settings.INDEX_NAME
+def index_path(tmp_path, monkeypatch):
+ path = tmp_path / "index.tinysearch"
TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=NUM_PAGES, page_size=PAGE_SIZE)
with TinyIndex(Document, str(path), "w") as index:
index.store_in_page(0, [Document(title="A", url="https://a.test/", extract="x", score=1.0, term="a")])
+ # The page reads the size of the index the web process has open.
+ monkeypatch.setattr(search_setup, "tiny_index", TinyIndex(Document, str(path)))
return path
@@ -56,6 +57,7 @@ def test_shows_a_scan_in_progress(staff_client, redis, index_path, monkeypatch):
assert response.context["scan"]["in_progress"]
assert response.context["scan"]["next_page"] == 1
assert b"page 1 of 4" in response.content
+ assert b"Next scan due" not in response.content
def test_shows_published_counts(staff_client, redis, index_path):
diff --git a/test/test_count_urls.py b/test/test_count_urls.py
index 8aad31a..0758009 100644
--- a/test/test_count_urls.py
+++ b/test/test_count_urls.py
@@ -194,7 +194,7 @@ def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_pat
with patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):
count_urls_step(redis, index_path, time_budget_seconds=0)
- status = get_scan_status(redis, index_path)
+ status = get_scan_status(redis, NUM_PAGES)
assert status["in_progress"]
assert status["next_page"] == 4
@@ -210,7 +210,7 @@ def test_scan_status_reports_the_progress_of_a_scan_in_progress(redis, index_pat
def test_scan_status_after_a_scan_has_finished(redis, index_path, settings):
count_urls_step(redis, index_path, time_budget_seconds=60)
- status = get_scan_status(redis, index_path)
+ status = get_scan_status(redis, NUM_PAGES)
assert not status["in_progress"]
assert status["last_finished"] == utc_today()
@@ -221,3 +221,16 @@ def test_published_counts_list_only_the_days_a_scan_finished(redis, index_path):
count_urls_step(redis, index_path, time_budget_seconds=60)
assert get_published_counts(redis, num_days=30) == [{"date": utc_today(), "urls": 4, "domains": 3, "results": 5}]
+
+
+def test_scan_status_of_an_empty_index_has_no_percent_done(redis, index_path):
+ with patch.object(count_urls, "NUM_PAGES_IN_BATCH", 4):
+ count_urls_step(redis, index_path, time_budget_seconds=0)
+
+ assert get_scan_status(redis, num_pages=0)["percent_done"] is None
+
+
+def test_published_counts_skip_a_day_missing_some_of_its_counts(redis):
+ redis.set(INDEX_URL_COUNT_KEY.format(date=utc_today()), 10)
+
+ assert get_published_counts(redis, num_days=30) == []