Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fix lint and test failures from the record_batch removal

Daoud Clarke988e4f20b460
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 458a944..6eee5f7 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -28,7 +28,6 @@ from mwmbl.crawler.batch import (
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.index_batches import index_documents
-from mwmbl.indexer.indexdb import BatchInfo, BatchStatus
from mwmbl.models import ApiKey, Device
from mwmbl.redis_url_queue import RedisURLQueue
from mwmbl.settings import (
@@ -151,9 +150,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
batch_url = f"{PUBLIC_URL_PREFIX}{filename}"
batch_cache.store(hashed_batch, batch_url)
- # Record the batch as being local so that we don't retrieve it again when the server restarts
- infos = [BatchInfo(batch_url, user_id_hash, BatchStatus.LOCAL, device_name=batch.device_name)]
-
# Update or create Device records for this batch
if batch.device_name:
Device.objects.update_or_create(user=user, hostname=batch.device_name, defaults={})
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index f7883f3..b71ec2e 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -85,7 +85,6 @@ class StatsManager:
def __init__(self, redis: Redis):
self.redis = redis
-
def get_stats(self) -> MwmblStats:
date_time = datetime.now(timezone.utc)
date = date_time.date()
diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.py
index 35273a1..104f6a6 100644
--- a/mwmbl/platform/schemas.py
+++ b/mwmbl/platform/schemas.py
@@ -23,14 +23,6 @@ class UserStatsResponse(Schema):
results_indexed_daily: dict[str, int]
-class UserStatsResponse(Schema):
- """Per-user contribution stats for the logged-in user."""
-
- username: str
- results_indexed_today: int
- results_indexed_daily: dict[str, int]
-
-
class SubscriptionResponse(Schema):
status: str
max_monthly_spend_cents: int
diff --git a/test/test_stats_hourly_counts.py b/test/test_stats_hourly_counts.py
deleted file mode 100644
index 5117910..0000000
--- a/test/test_stats_hourly_counts.py
+++ /dev/null
@@ -1,52 +0,0 @@
-"""
-Tests for the hourly crawl counter behind the "URLs crawled" chart.
-
-The key is a formatted datetime, so it is the one daily counter whose Redis key changes
-if the datetime it is built from becomes timezone-aware: `str()` of an aware datetime
-carries a `+00:00` suffix. Writer and reader are in different methods, so getting this
-wrong on one side alone empties the chart silently.
-"""
-
-from datetime import datetime, timezone
-from unittest.mock import patch
-
-import fakeredis
-
-from mwmbl.crawler.batch import Result, Results
-from mwmbl.crawler.stats import StatsManager, hour_count_key
-
-NO_INDEX_COUNTS = {
- "urls_in_index_daily": {},
- "domains_in_index_daily": {},
- "results_in_index_daily": {},
-}
-
-
-def make_results(crawled_at: datetime) -> Results:
- content = Result(
- url="https://fake.url",
- title="Example",
- extract="Hello from fake.url!",
- last_crawled=int(crawled_at.timestamp()),
- )
- return Results(results=[content])
-
-
-def test_the_hour_key_has_no_timezone_offset_in_it():
- aware = datetime(2026, 9, 8, 13, 45, tzinfo=timezone.utc)
- assert hour_count_key(aware) == "url-count-hour-2026-09-08 13:00:00"
-
-
-def test_the_stats_read_back_the_hour_key_the_batch_wrote():
- """The chart is built by get_stats reading keys record_results wrote, so the two have to
- agree on the format - not just each be self-consistent."""
- redis = fakeredis.FakeRedis(decode_responses=True)
- now = datetime.now(timezone.utc)
-
- with patch("mwmbl.crawler.urls.URLDatabase"):
- stats_manager = StatsManager(redis)
- stats_manager.record_results(make_results(now), "fake user name")
- with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):
- stats = stats_manager.get_stats()
-
- assert stats.urls_crawled_hourly[now.hour] == 1