An open source, non-profit web search engine
Fix lint and test failures from the record_batch removal
Daoud Clarke988e4f20b460
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 458a944..6eee5f7 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -28,7 +28,6 @@ from mwmbl.crawler.batch import (from mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index_batches import index_documents-from mwmbl.indexer.indexdb import BatchInfo, BatchStatusfrom mwmbl.models import ApiKey, Devicefrom mwmbl.redis_url_queue import RedisURLQueuefrom mwmbl.settings import (@@ -151,9 +150,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchbatch_url = f"{PUBLIC_URL_PREFIX}{filename}"batch_cache.store(hashed_batch, batch_url)- # Record the batch as being local so that we don't retrieve it again when the server restarts- infos = [BatchInfo(batch_url, user_id_hash, BatchStatus.LOCAL, device_name=batch.device_name)]-# Update or create Device records for this batchif batch.device_name:Device.objects.update_or_create(user=user, hostname=batch.device_name, defaults={})diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex f7883f3..b71ec2e 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -85,7 +85,6 @@ class StatsManager:def __init__(self, redis: Redis):self.redis = redis-def get_stats(self) -> MwmblStats:date_time = datetime.now(timezone.utc)date = date_time.date()diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.pyindex 35273a1..104f6a6 100644--- a/mwmbl/platform/schemas.py+++ b/mwmbl/platform/schemas.py@@ -23,14 +23,6 @@ class UserStatsResponse(Schema):results_indexed_daily: dict[str, int]-class UserStatsResponse(Schema):- """Per-user contribution stats for the logged-in user."""-- username: str- results_indexed_today: int- results_indexed_daily: dict[str, int]--class SubscriptionResponse(Schema):status: strmax_monthly_spend_cents: intdiff --git a/test/test_stats_hourly_counts.py b/test/test_stats_hourly_counts.pydeleted file mode 100644index 5117910..0000000--- a/test/test_stats_hourly_counts.py+++ /dev/null@@ -1,52 +0,0 @@-"""-Tests for the hourly crawl counter behind the "URLs crawled" chart.--The key is a formatted datetime, so it is the one daily counter whose Redis key changes-if the datetime it is built from becomes timezone-aware: `str()` of an aware datetime-carries a `+00:00` suffix. Writer and reader are in different methods, so getting this-wrong on one side alone empties the chart silently.-"""--from datetime import datetime, timezone-from unittest.mock import patch--import fakeredis--from mwmbl.crawler.batch import Result, Results-from mwmbl.crawler.stats import StatsManager, hour_count_key--NO_INDEX_COUNTS = {- "urls_in_index_daily": {},- "domains_in_index_daily": {},- "results_in_index_daily": {},-}---def make_results(crawled_at: datetime) -> Results:- content = Result(- url="https://fake.url",- title="Example",- extract="Hello from fake.url!",- last_crawled=int(crawled_at.timestamp()),- )- return Results(results=[content])---def test_the_hour_key_has_no_timezone_offset_in_it():- aware = datetime(2026, 9, 8, 13, 45, tzinfo=timezone.utc)- assert hour_count_key(aware) == "url-count-hour-2026-09-08 13:00:00"---def test_the_stats_read_back_the_hour_key_the_batch_wrote():- """The chart is built by get_stats reading keys record_results wrote, so the two have to- agree on the format - not just each be self-consistent."""- redis = fakeredis.FakeRedis(decode_responses=True)- now = datetime.now(timezone.utc)-- with patch("mwmbl.crawler.urls.URLDatabase"):- stats_manager = StatsManager(redis)- stats_manager.record_results(make_results(now), "fake user name")- with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):- stats = stats_manager.get_stats()-- assert stats.urls_crawled_hourly[now.hour] == 1