An open source, non-profit web search engine
Remove all references to the extinct `record_batch` stats method.
blueheron7861fb80f90b56c
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex f087385..6d8c7bc 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -144,8 +144,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchepoch_time = (now - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()hashed_batch = HashedBatch(user_id_hash=user_id_hash, timestamp=epoch_time, items=batch.items)- stats_manager.record_batch(hashed_batch)-filename = upload_object(hashed_batch, now, user_id_hash, "batch")global last_batch@@ -157,10 +155,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch# Record the batch as being local so that we don't retrieve it again when the server restartsinfos = [BatchInfo(batch_url, user_id_hash, BatchStatus.LOCAL, device_name=batch.device_name)]- with Database() as db:- index_db = IndexDatabase(db.connection)- index_db.record_batches(infos)-# Update or create Device records for this batchif batch.device_name:Device.objects.update_or_create(user=user, hostname=batch.device_name, defaults={})diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 4f04264..5f57a2a 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -37,8 +37,7 @@ def hour_count_key(date_time: datetime) -> str:Formatted rather than built from a truncated datetime so that the key does not dependon whether ``date_time`` is aware: ``str()`` of an aware datetime carries a ``+00:00``- suffix, which would split the writer in :meth:`StatsManager.record_batch` from the- reader in :meth:`StatsManager.get_stats` and zero the hourly chart.+ suffix, which would split the reader in :meth:`StatsManager.get_stats` and zero the hourly chart."""return URL_HOUR_COUNT_KEY.format(hour=date_time.strftime("%Y-%m-%d %H:00:00"))@@ -240,9 +239,6 @@ class StatsManager:self.redis.zincrby(user_result_count_key, num_results, username)self.redis.expire(user_result_count_key, SHORT_EXPIRE_SECONDS)- # The new crawler submits via the API-key-authenticated results endpoint rather than- # record_batch, so this is the path that sees real crawlers. Add the account so that- # users_crawled_daily counts distinct crawlers that contacted us today.users_key = USERS_KEY.format(date=utc_today())self.redis.sadd(users_key, username)self.redis.expire(users_key, LONG_EXPIRE_SECONDS)@@ -283,31 +279,3 @@ class StatsManager:dataset_results_count_key = DATASET_RESULTS_COUNT_KEY.format(date=dataset_date)self.redis.incrby(dataset_results_count_key, num_successful_results)self.redis.expire(dataset_results_count_key, LONG_EXPIRE_SECONDS)---def get_test_batches():- for path in glob("./devdata/batches/**/*.json.gz", recursive=True):- print("Processing path", path)- with gzip.open(path) as gzip_file:- yield HashedBatch.parse_raw(gzip_file.read())---if __name__ == "__main__":- django.setup()- redis = Redis(host="localhost", port=6379, decode_responses=True)- stats = StatsManager(redis)- batches = get_test_batches()- start = datetime.now(timezone.utc)- processed = 0- import logging-- logging.basicConfig(level=logging.INFO)- for batch in islice(batches, 10000):- if len(batch.items) <= 2:- continue- stats.record_batch(batch)- processed += 1- total_time = (datetime.now(timezone.utc) - start).total_seconds()- print("Processed", processed)- print("Total time", total_time)- print("Time per batch", total_time / processed)diff --git a/test/test_stats_hourly_counts.py b/test/test_stats_hourly_counts.pyindex 8bb1ff4..8b501b7 100644--- a/test/test_stats_hourly_counts.py+++ b/test/test_stats_hourly_counts.py@@ -33,26 +33,14 @@ def test_the_hour_key_has_no_timezone_offset_in_it():assert hour_count_key(aware) == "url-count-hour-2026-09-08 13:00:00"-def test_a_recorded_batch_is_counted_against_the_utc_hour_it_was_crawled_in():- redis = fakeredis.FakeRedis(decode_responses=True)- crawled_at = datetime(2026, 9, 8, 13, 45, tzinfo=timezone.utc)-- with patch("mwmbl.crawler.stats.URLDatabase"):- StatsManager(redis).record_batch(make_batch(crawled_at))-- assert redis.get(URL_HOUR_COUNT_KEY.format(hour="2026-09-08 13:00:00")) == "1"--def test_the_stats_read_back_the_hour_key_the_batch_wrote():- """The chart is built by get_stats reading keys record_batch wrote, so the two have to+ """The chart is built by get_stats reading keys record_results wrote, so the two have toagree on the format - not just each be self-consistent."""redis = fakeredis.FakeRedis(decode_responses=True)now = datetime.now(timezone.utc)- with patch("mwmbl.crawler.stats.URLDatabase"):+ with patch("mwmbl.crawler.urls.URLDatabase"):stats_manager = StatsManager(redis)- stats_manager.record_batch(make_batch(now))-with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):stats = stats_manager.get_stats()diff --git a/test/test_stats_users_crawled.py b/test/test_stats_users_crawled.pyindex e2268e0..ee76471 100644--- a/test/test_stats_users_crawled.py+++ b/test/test_stats_users_crawled.py@@ -1,13 +1,11 @@"""Tests for "users crawled today" — the count of distinct crawlers that contacted us.-Both submission paths feed the same users set that `get_stats` reads back with `scard`+The submission path feeds the same users set that `get_stats` reads back with `scard`to build `users_crawled_daily`:-- `record_batch` (the deprecated batch flow) adds the crawler's self-claimed- `user_id_hash`.-- `record_results` (the modern API-key-authenticated results endpoint) adds the- account's `username`.+`record_results` (the modern API-key-authenticated results endpoint) adds the+account's `username`.Because writer and reader are in different methods, a change to one side that splitsthe key format would silently zero the chart, so these tests pin the two writers to the@@ -59,19 +57,3 @@ def test_record_results_counts_each_crawler_once_per_day():today = str(utc_today())assert stats.users_crawled_daily[today] == 2---def test_both_submission_paths_feed_the_same_users_set():- redis = fakeredis.FakeRedis(decode_responses=True)- stats_manager = StatsManager(redis)-- # The old batch flow and the new results endpoint each contribute a distinct crawler.- with patch("mwmbl.crawler.stats.URLDatabase"):- stats_manager.record_batch(make_batch(user_id_hash="a" * 64))- stats_manager.record_results(make_results("https://example.com/"), "alice")-- with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):- stats = stats_manager.get_stats()-- today = str(utc_today())- assert stats.users_crawled_daily[today] == 2