Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Remove the rest of the code the old crawler stats fed

Daoud Clarke5686f352c013
diff --git a/mwmbl/api.py b/mwmbl/api.py
index ee33d78..54f3f07 100644
--- a/mwmbl/api.py
+++ b/mwmbl/api.py
@@ -131,12 +131,12 @@ def v2_invalid_request_handler(request, exc: InvalidRequest):
# Routers are imported after the api instances are created to avoid circular imports,
# and after init functions have been called from urls.py.
-def register_routers(ranker, batch_cache, queued_batches):
+def register_routers(ranker, queued_batches):
"""
Initialise and register all sub-routers on the v1 and v2 APIs.
This is called from urls.py after Django app setup is complete, so that
- dependencies (ranker, batch_cache, queued_batches) are available.
+ dependencies (ranker, queued_batches) are available.
"""
import mwmbl.crawler.app as crawler_module
import mwmbl.tinysearchengine.search as search_module
@@ -147,7 +147,7 @@ def register_routers(ranker, batch_cache, queued_batches):
search_module.init_router(ranker)
search_module.init_v2_router(ranker)
super_search_module.init_router()
- crawler_module.init_router(batch_cache, queued_batches)
+ crawler_module.init_router(queued_batches)
api.add_router("/search/", search_module.router, tags=["Search"])
api.add_router("/crawler/", crawler_module.router, tags=["Crawler"])
diff --git a/mwmbl/crawl.py b/mwmbl/crawl.py
index a65691a..f4c9d3d 100644
--- a/mwmbl/crawl.py
+++ b/mwmbl/crawl.py
@@ -1,6 +1,7 @@
import argparse
import logging
import os
+import platform
import random
import time
from datetime import datetime, timedelta, timezone
@@ -289,7 +290,12 @@ class Crawler:
if CRAWL_SUBMIT_MODE == SUBMIT_MODE_OFF:
logger.info(f"Submission is off, not posting {len(result_items)} results")
else:
- results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)
+ results = Results(
+ api_key=MWMBL_API_KEY,
+ results=result_items,
+ crawler_version=CRAWLER_VERSION,
+ device_name=platform.uname().node,
+ )
results_url = f"{REMOTE_SERVER}/api/v1/crawler/results"
if CRAWL_SUBMIT_MODE == SUBMIT_MODE_DRY_RUN:
results_url += "?dry_run=true"
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index d8c84f8..538bcca 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -18,14 +18,12 @@ from redis import Redis
from mwmbl.crawler.batch import (
DatasetRequest,
Error,
- HashedBatch,
HashedDataset,
NewBatchRequest,
PostResultsResponse,
Results,
)
from mwmbl.crawler.stats import MwmblStats, StatsManager
-from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.index_batches import index_documents
from mwmbl.models import ApiKey, Device
from mwmbl.redis_url_queue import RedisURLQueue
@@ -68,9 +66,6 @@ def upload(data: bytes, name: str):
return result
-last_batch = None
-
-
def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):
seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
@@ -87,7 +82,7 @@ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object
return filename
-def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batches: RedisURLQueue):
+def _register_routes(r: Router | NinjaAPI, queued_batches: RedisURLQueue):
"""Register all crawler routes on the given router or API instance."""
@r.post(
@@ -156,14 +151,14 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
@r.get(
"/latest-batch",
- summary="Get the latest batch",
+ summary="Get the latest batch (removed)",
description=(
- "Return the most recently submitted crawl batch held in memory. "
- "Returns an empty list if no batch has been submitted since the server started."
+ "Removed - this returned the most recent submission to the now-removed "
+ "`POST /batches/` endpoint, so it always returns 410 Gone."
),
)
- def get_latest_batch(request) -> list[HashedBatch]:
- return [] if last_batch is None else [last_batch]
+ def get_latest_batch(request):
+ raise HttpError(410, "This endpoint has been removed along with POST /batches/.")
@r.get(
"/batches/{date_str}/users",
@@ -300,7 +295,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
)
def post_dataset(request, dataset: DatasetRequest):
if len(dataset.user_id) != USER_ID_LENGTH:
- return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
+ raise HttpError(400, f"Incorrect user ID length, should be {USER_ID_LENGTH}")
user_id_hash = _get_user_id_hash(dataset)
@@ -327,15 +322,15 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
}
-def init_router(batch_cache: BatchCache, queued_batches: RedisURLQueue):
+def init_router(queued_batches: RedisURLQueue):
"""Initialise the module-level router (called from urls.py for the unified v1 API)."""
- _register_routes(router, batch_cache, queued_batches)
+ _register_routes(router, queued_batches)
-def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:
+def create_router(queued_batches: RedisURLQueue, version: str) -> NinjaAPI:
"""Create a standalone NinjaAPI for a specific version (used for legacy routes)."""
api = NinjaAPI(urls_namespace=f"crawler-{version}")
- _register_routes(api, batch_cache, queued_batches)
+ _register_routes(api, queued_batches)
return api
diff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.py
index 7b1bf46..6041fdd 100644
--- a/mwmbl/crawler/retrieve.py
+++ b/mwmbl/crawler/retrieve.py
@@ -33,9 +33,6 @@ ALLOWED_EXCEPTIONS = (
SSLCertVerificationError,
)
-POST_BATCH_URL = "/api/v1/crawler/batches/"
-POST_NEW_BATCH_URL = "/api/v1/crawler/batches/new"
-
TIMEOUT_SECONDS = 3
MAX_REDIRECTS = 5
MAX_FETCH_SIZE = 1024 * 1024
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 5ecc77d..2a6c637 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -11,10 +11,6 @@ from mwmbl.utils import utc_today
logger = getLogger(__name__)
USERS_KEY = "users-{date}"
-HOST_COUNT_KEY = "host-count-{date}"
-HOST_COUNT_ALL_KEY = "host-count-all-{date}"
-HOST_COUNT_LINK_KEY = "host-count-link-{date}"
-HOST_COUNT_LINK_NEW_KEY = "host-count-link-new-{date}"
RESULTS_COUNT_KEY = "results-count-{date}"
USER_RESULTS_COUNT_KEY = "user-results-count-{date}"
DATASET_QUERIES_COUNT_KEY = "dataset-queries-count-{date}"
@@ -27,10 +23,6 @@ LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30
class DomainStats(BaseModel):
domain_name: str
- num_crawled: int
- num_successful: int
- num_links: int
- num_links_new: int
num_index_results: int
@@ -159,43 +151,8 @@ class StatsManager:
"results_indexed_today": results_indexed_daily[str(date)],
}
- def get_domain_stats(self) -> list[DomainStats]:
- today = utc_today()
- host_all_key = HOST_COUNT_ALL_KEY.format(date=today)
- host_counts_all = self.redis.zrevrange(host_all_key, 0, 1000, withscores=True)
- all_domain_stats = []
- for host, count in host_counts_all:
- num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=today), host)
- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=today), host)
- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=today), host)
- num_index_results = get_domain_result_count(host)
- domain_stats = DomainStats(
- domain_name=host,
- num_crawled=count,
- num_successful=num_successful or 0,
- num_links=num_links or 0,
- num_links_new=num_links_new or 0,
- num_index_results=num_index_results,
- )
- all_domain_stats.append(domain_stats)
- return all_domain_stats
-
def get_stats_for_domain(self, host: str) -> DomainStats:
- today = utc_today()
- num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=today), host)
- num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=today), host)
- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=today), host)
- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=today), host)
- num_index_results = get_domain_result_count(host)
- domain_stats = DomainStats(
- domain_name=host,
- num_crawled=num_crawled or 0,
- num_successful=num_successful or 0,
- num_links=num_links or 0,
- num_links_new=num_links_new or 0,
- num_index_results=num_index_results,
- )
- return domain_stats
+ return DomainStats(domain_name=host, num_index_results=get_domain_result_count(host))
def record_results(self, results: Results, username: str) -> None:
result_count_key = RESULTS_COUNT_KEY.format(date=utc_today())
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 9b977dc..6d62721 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -5,7 +5,6 @@ from django.conf import settings
from redis import Redis
from mwmbl.curated_domains import get_curated_domains
-from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.tinysearchengine.completer import Completer
@@ -41,5 +40,3 @@ ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
# passed 3 here while mwmbl.rankeval.ltr.dataset trained on 5, so the model was ranking a
# pool a size it had never been fitted to.
ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True))
-
-batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/templates/mwmbl/domain.html b/mwmbl/templates/mwmbl/domain.html
index 12a641c..98d614a 100644
--- a/mwmbl/templates/mwmbl/domain.html
+++ b/mwmbl/templates/mwmbl/domain.html
@@ -4,10 +4,6 @@
{% block content %}
<h1>Stats for <a href="https://{{ domain_stats.domain_name }}">{{ domain_stats.domain_name }}</a></h1>
<ul>
-<li>Number of URLs crawled today: {{ domain_stats.num_crawled }}</li>
-<li>Number successfully retrieved: {{ domain_stats.num_successful }}</li>
-<li>Number of links extracted from this domain: {{ domain_stats.num_links }}</li>
-<li>Number of links which are new: {{ domain_stats.num_links_new }}</li>
<li>Number of results for this domain in the index: {{ domain_stats.num_index_results }}</li>
</ul>
diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.html
deleted file mode 100644
index 4f0dcfc..0000000
--- a/mwmbl/templates/mwmbl/domains.html
+++ /dev/null
@@ -1,24 +0,0 @@
-{% extends "base.html" %}
-
-{% block content %}
-
- <table>
- <thead>
- <tr>
- <th>Domain name</th><th>Crawled</th><th>Successful</th><th>Links</th><th>New links</th><th>Index results</th>
- </tr>
- </thead>
- {% for domain in domain_stats %}
- <tr>
- <td><a href="{% url "domain" domain.domain_name %}">{{ domain.domain_name }}</a></td>
- <td>{{ domain.num_crawled }}</td>
- <td>{{ domain.num_successful }}</td>
- <td>{{ domain.num_links }}</td>
- <td>{{ domain.num_links_new }}</td>
- <td>{{ domain.num_index_results }}</td>
- </tr>
- {% endfor %}
- </table>
-
-
-{% endblock %}
\ No newline at end of file
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 6aa043d..c8c8bf0 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -24,7 +24,7 @@ import mwmbl.crawler.app as crawler
from mwmbl.admin_views import blacklist_status_view
from mwmbl.api import api as v1_api
from mwmbl.api import register_routers, v2_api
-from mwmbl.search_setup import batch_cache, queued_batches, ranker
+from mwmbl.search_setup import queued_batches, ranker
from mwmbl.tinysearchengine import search
from mwmbl.views import (
CurationDetailView,
@@ -34,7 +34,6 @@ from mwmbl.views import (
add_url,
approve,
domain_view,
- domains_view,
flag_curation,
flag_curation_update,
home_fragment,
@@ -46,7 +45,7 @@ from mwmbl.views import (
# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.
# This must be called before urlpatterns is evaluated.
-register_routers(ranker=ranker, batch_cache=batch_cache, queued_batches=queued_batches)
+register_routers(ranker=ranker, queued_batches=queued_batches)
def trigger_error(request):
@@ -71,11 +70,10 @@ urlpatterns = [
path("app/curations/<int:curation_id>/flag/", flag_curation, name="flag_curation"),
path("app/flags/curation", CurationFlagListView.as_view(), name="flag_curation_list"),
path("app/flags/curation/<int:flag_curation_id>/update", flag_curation_update, name="flag_curation_update"),
- path("app/domains/", domains_view, name="domains"),
path("app/domains/<str:domain>/", domain_view, name="domain"),
# TODO: these are the old APIs, deprecated and to be removed once all clients have moved over
path("search/", search.create_router(ranker, "0.1").urls),
- path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),
+ path("crawler/", crawler.create_router(queued_batches=queued_batches, version="0.1").urls),
# Unified v1 API — search returns plain list; single docs page at /api/v1/docs
path("api/v1/", v1_api.urls),
# v2 API — search returns object with quota metadata; docs at /api/v2/docs
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 12ba58f..1dad637 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -558,11 +558,6 @@ class CurationFlagListView(LoginRequiredMixin, ListView):
return FlagCuration.objects.filter(status="PENDING").order_by("-timestamp")
-def domains_view(request):
- domain_stats = stats_manager.get_domain_stats()
- return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})
-
-
def domain_view(request, domain):
if request.method == "POST":
if request.user.has_perm("mwmbl.change_domain_submission_status"):
diff --git a/test/test_batches_endpoint_removed.py b/test/test_batches_endpoint_removed.py
index f2356e5..12f783c 100644
--- a/test/test_batches_endpoint_removed.py
+++ b/test/test_batches_endpoint_removed.py
@@ -128,3 +128,29 @@ def test_post_results_is_idempotent_for_a_repeated_device(api_client, crawl_api_
assert response.status_code == 200
assert Device.objects.filter(user=crawl_user, hostname="my-device").count() == 1
+
+
+def test_latest_batch_returns_410(api_client):
+ """It only ever returned what POST /batches/ stored in memory."""
+ response = api_client.get("/api/v1/crawler/latest-batch")
+ assert response.status_code == 410
+
+
+def test_post_dataset_bad_user_id_returns_400_on_the_router_mounted_api(api_client):
+ """This path used to call r.create_response, which a Router does not have, so a
+ wrong-length user ID raised AttributeError and 500 instead of returning 400."""
+ response = api_client.post(
+ "/api/v1/crawler/dataset",
+ content_type="application/json",
+ data={
+ "user_id": "too-short",
+ "date": "2026-01-01",
+ "timestamp": 1704672000000,
+ "extensionVersion": "0.6.1",
+ "queryDataset": [],
+ "searchResults": [],
+ },
+ )
+ assert response.status_code == 400