An open source, non-profit web search engine
Remove the rest of the code the old crawler stats fed
Daoud Clarke5686f352c013
diff --git a/mwmbl/api.py b/mwmbl/api.pyindex ee33d78..54f3f07 100644--- a/mwmbl/api.py+++ b/mwmbl/api.py@@ -131,12 +131,12 @@ def v2_invalid_request_handler(request, exc: InvalidRequest):# Routers are imported after the api instances are created to avoid circular imports,# and after init functions have been called from urls.py.-def register_routers(ranker, batch_cache, queued_batches):+def register_routers(ranker, queued_batches):"""Initialise and register all sub-routers on the v1 and v2 APIs.This is called from urls.py after Django app setup is complete, so that- dependencies (ranker, batch_cache, queued_batches) are available.+ dependencies (ranker, queued_batches) are available."""import mwmbl.crawler.app as crawler_moduleimport mwmbl.tinysearchengine.search as search_module@@ -147,7 +147,7 @@ def register_routers(ranker, batch_cache, queued_batches):search_module.init_router(ranker)search_module.init_v2_router(ranker)super_search_module.init_router()- crawler_module.init_router(batch_cache, queued_batches)+ crawler_module.init_router(queued_batches)api.add_router("/search/", search_module.router, tags=["Search"])api.add_router("/crawler/", crawler_module.router, tags=["Crawler"])diff --git a/mwmbl/crawl.py b/mwmbl/crawl.pyindex a65691a..f4c9d3d 100644--- a/mwmbl/crawl.py+++ b/mwmbl/crawl.py@@ -1,6 +1,7 @@import argparseimport loggingimport os+import platformimport randomimport timefrom datetime import datetime, timedelta, timezone@@ -289,7 +290,12 @@ class Crawler:if CRAWL_SUBMIT_MODE == SUBMIT_MODE_OFF:logger.info(f"Submission is off, not posting {len(result_items)} results")else:- results = Results(api_key=MWMBL_API_KEY, results=result_items, crawler_version=CRAWLER_VERSION)+ results = Results(+ api_key=MWMBL_API_KEY,+ results=result_items,+ crawler_version=CRAWLER_VERSION,+ device_name=platform.uname().node,+ )results_url = f"{REMOTE_SERVER}/api/v1/crawler/results"if CRAWL_SUBMIT_MODE == SUBMIT_MODE_DRY_RUN:results_url += "?dry_run=true"diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex d8c84f8..538bcca 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -18,14 +18,12 @@ from redis import Redisfrom mwmbl.crawler.batch import (DatasetRequest,Error,- HashedBatch,HashedDataset,NewBatchRequest,PostResultsResponse,Results,)from mwmbl.crawler.stats import MwmblStats, StatsManager-from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index_batches import index_documentsfrom mwmbl.models import ApiKey, Devicefrom mwmbl.redis_url_queue import RedisURLQueue@@ -68,9 +66,6 @@ def upload(data: bytes, name: str):return result-last_batch = None--def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds@@ -87,7 +82,7 @@ def upload_object(model_object: Schema, now: datetime, user_id_hash: str, objectreturn filename-def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batches: RedisURLQueue):+def _register_routes(r: Router | NinjaAPI, queued_batches: RedisURLQueue):"""Register all crawler routes on the given router or API instance."""@r.post(@@ -156,14 +151,14 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch@r.get("/latest-batch",- summary="Get the latest batch",+ summary="Get the latest batch (removed)",description=(- "Return the most recently submitted crawl batch held in memory. "- "Returns an empty list if no batch has been submitted since the server started."+ "Removed - this returned the most recent submission to the now-removed "+ "`POST /batches/` endpoint, so it always returns 410 Gone."),)- def get_latest_batch(request) -> list[HashedBatch]:- return [] if last_batch is None else [last_batch]+ def get_latest_batch(request):+ raise HttpError(410, "This endpoint has been removed along with POST /batches/.")@r.get("/batches/{date_str}/users",@@ -300,7 +295,7 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch)def post_dataset(request, dataset: DatasetRequest):if len(dataset.user_id) != USER_ID_LENGTH:- return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)+ raise HttpError(400, f"Incorrect user ID length, should be {USER_ID_LENGTH}")user_id_hash = _get_user_id_hash(dataset)@@ -327,15 +322,15 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch}-def init_router(batch_cache: BatchCache, queued_batches: RedisURLQueue):+def init_router(queued_batches: RedisURLQueue):"""Initialise the module-level router (called from urls.py for the unified v1 API)."""- _register_routes(router, batch_cache, queued_batches)+ _register_routes(router, queued_batches)-def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:+def create_router(queued_batches: RedisURLQueue, version: str) -> NinjaAPI:"""Create a standalone NinjaAPI for a specific version (used for legacy routes)."""api = NinjaAPI(urls_namespace=f"crawler-{version}")- _register_routes(api, batch_cache, queued_batches)+ _register_routes(api, queued_batches)return apidiff --git a/mwmbl/crawler/retrieve.py b/mwmbl/crawler/retrieve.pyindex 7b1bf46..6041fdd 100644--- a/mwmbl/crawler/retrieve.py+++ b/mwmbl/crawler/retrieve.py@@ -33,9 +33,6 @@ ALLOWED_EXCEPTIONS = (SSLCertVerificationError,)-POST_BATCH_URL = "/api/v1/crawler/batches/"-POST_NEW_BATCH_URL = "/api/v1/crawler/batches/new"-TIMEOUT_SECONDS = 3MAX_REDIRECTS = 5MAX_FETCH_SIZE = 1024 * 1024diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 5ecc77d..2a6c637 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -11,10 +11,6 @@ from mwmbl.utils import utc_todaylogger = getLogger(__name__)USERS_KEY = "users-{date}"-HOST_COUNT_KEY = "host-count-{date}"-HOST_COUNT_ALL_KEY = "host-count-all-{date}"-HOST_COUNT_LINK_KEY = "host-count-link-{date}"-HOST_COUNT_LINK_NEW_KEY = "host-count-link-new-{date}"RESULTS_COUNT_KEY = "results-count-{date}"USER_RESULTS_COUNT_KEY = "user-results-count-{date}"DATASET_QUERIES_COUNT_KEY = "dataset-queries-count-{date}"@@ -27,10 +23,6 @@ LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30class DomainStats(BaseModel):domain_name: str- num_crawled: int- num_successful: int- num_links: int- num_links_new: intnum_index_results: int@@ -159,43 +151,8 @@ class StatsManager:"results_indexed_today": results_indexed_daily[str(date)],}- def get_domain_stats(self) -> list[DomainStats]:- today = utc_today()- host_all_key = HOST_COUNT_ALL_KEY.format(date=today)- host_counts_all = self.redis.zrevrange(host_all_key, 0, 1000, withscores=True)- all_domain_stats = []- for host, count in host_counts_all:- num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=today), host)- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=today), host)- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=today), host)- num_index_results = get_domain_result_count(host)- domain_stats = DomainStats(- domain_name=host,- num_crawled=count,- num_successful=num_successful or 0,- num_links=num_links or 0,- num_links_new=num_links_new or 0,- num_index_results=num_index_results,- )- all_domain_stats.append(domain_stats)- return all_domain_stats-def get_stats_for_domain(self, host: str) -> DomainStats:- today = utc_today()- num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=today), host)- num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=today), host)- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=today), host)- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=today), host)- num_index_results = get_domain_result_count(host)- domain_stats = DomainStats(- domain_name=host,- num_crawled=num_crawled or 0,- num_successful=num_successful or 0,- num_links=num_links or 0,- num_links_new=num_links_new or 0,- num_index_results=num_index_results,- )- return domain_stats+ return DomainStats(domain_name=host, num_index_results=get_domain_result_count(host))def record_results(self, results: Results, username: str) -> None:result_count_key = RESULTS_COUNT_KEY.format(date=utc_today())diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 9b977dc..6d62721 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -5,7 +5,6 @@ from django.conf import settingsfrom redis import Redisfrom mwmbl.curated_domains import get_curated_domains-from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklistfrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.tinysearchengine.completer import Completer@@ -41,5 +40,3 @@ ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))# passed 3 here while mwmbl.rankeval.ltr.dataset trained on 5, so the model was ranking a# pool a size it had never been fitted to.ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True))--batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/templates/mwmbl/domain.html b/mwmbl/templates/mwmbl/domain.htmlindex 12a641c..98d614a 100644--- a/mwmbl/templates/mwmbl/domain.html+++ b/mwmbl/templates/mwmbl/domain.html@@ -4,10 +4,6 @@{% block content %}<h1>Stats for <a href="https://{{ domain_stats.domain_name }}">{{ domain_stats.domain_name }}</a></h1><ul>-<li>Number of URLs crawled today: {{ domain_stats.num_crawled }}</li>-<li>Number successfully retrieved: {{ domain_stats.num_successful }}</li>-<li>Number of links extracted from this domain: {{ domain_stats.num_links }}</li>-<li>Number of links which are new: {{ domain_stats.num_links_new }}</li><li>Number of results for this domain in the index: {{ domain_stats.num_index_results }}</li></ul>diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.htmldeleted file mode 100644index 4f0dcfc..0000000--- a/mwmbl/templates/mwmbl/domains.html+++ /dev/null@@ -1,24 +0,0 @@-{% extends "base.html" %}--{% block content %}-- <table>- <thead>- <tr>- <th>Domain name</th><th>Crawled</th><th>Successful</th><th>Links</th><th>New links</th><th>Index results</th>- </tr>- </thead>- {% for domain in domain_stats %}- <tr>- <td><a href="{% url "domain" domain.domain_name %}">{{ domain.domain_name }}</a></td>- <td>{{ domain.num_crawled }}</td>- <td>{{ domain.num_successful }}</td>- <td>{{ domain.num_links }}</td>- <td>{{ domain.num_links_new }}</td>- <td>{{ domain.num_index_results }}</td>- </tr>- {% endfor %}- </table>---{% endblock %}\ No newline at end of filediff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 6aa043d..c8c8bf0 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -24,7 +24,7 @@ import mwmbl.crawler.app as crawlerfrom mwmbl.admin_views import blacklist_status_viewfrom mwmbl.api import api as v1_apifrom mwmbl.api import register_routers, v2_api-from mwmbl.search_setup import batch_cache, queued_batches, ranker+from mwmbl.search_setup import queued_batches, rankerfrom mwmbl.tinysearchengine import searchfrom mwmbl.views import (CurationDetailView,@@ -34,7 +34,6 @@ from mwmbl.views import (add_url,approve,domain_view,- domains_view,flag_curation,flag_curation_update,home_fragment,@@ -46,7 +45,7 @@ from mwmbl.views import (# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.# This must be called before urlpatterns is evaluated.-register_routers(ranker=ranker, batch_cache=batch_cache, queued_batches=queued_batches)+register_routers(ranker=ranker, queued_batches=queued_batches)def trigger_error(request):@@ -71,11 +70,10 @@ urlpatterns = [path("app/curations/<int:curation_id>/flag/", flag_curation, name="flag_curation"),path("app/flags/curation", CurationFlagListView.as_view(), name="flag_curation_list"),path("app/flags/curation/<int:flag_curation_id>/update", flag_curation_update, name="flag_curation_update"),- path("app/domains/", domains_view, name="domains"),path("app/domains/<str:domain>/", domain_view, name="domain"),# TODO: these are the old APIs, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),- path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),+ path("crawler/", crawler.create_router(queued_batches=queued_batches, version="0.1").urls),# Unified v1 API — search returns plain list; single docs page at /api/v1/docspath("api/v1/", v1_api.urls),# v2 API — search returns object with quota metadata; docs at /api/v2/docsdiff --git a/mwmbl/views.py b/mwmbl/views.pyindex 12ba58f..1dad637 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -558,11 +558,6 @@ class CurationFlagListView(LoginRequiredMixin, ListView):return FlagCuration.objects.filter(status="PENDING").order_by("-timestamp")-def domains_view(request):- domain_stats = stats_manager.get_domain_stats()- return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})--def domain_view(request, domain):if request.method == "POST":if request.user.has_perm("mwmbl.change_domain_submission_status"):diff --git a/test/test_batches_endpoint_removed.py b/test/test_batches_endpoint_removed.pyindex f2356e5..12f783c 100644--- a/test/test_batches_endpoint_removed.py+++ b/test/test_batches_endpoint_removed.py@@ -128,3 +128,29 @@ def test_post_results_is_idempotent_for_a_repeated_device(api_client, crawl_api_assert response.status_code == 200assert Device.objects.filter(user=crawl_user, hostname="my-device").count() == 1+++def test_latest_batch_returns_410(api_client):+ """It only ever returned what POST /batches/ stored in memory."""+ response = api_client.get("/api/v1/crawler/latest-batch")+ assert response.status_code == 410+++def test_post_dataset_bad_user_id_returns_400_on_the_router_mounted_api(api_client):+ """This path used to call r.create_response, which a Router does not have, so a+ wrong-length user ID raised AttributeError and 500 instead of returning 400."""+ response = api_client.post(+ "/api/v1/crawler/dataset",+ content_type="application/json",+ data={+ "user_id": "too-short",+ "date": "2026-01-01",+ "timestamp": 1704672000000,+ "extensionVersion": "0.6.1",+ "queryDataset": [],+ "searchResults": [],+ },+ )+ assert response.status_code == 400