Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Remove all references to the extinct `record_batch` stats method.

blueheron7861fb80f90b56c
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index f087385..6d8c7bc 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -144,8 +144,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
epoch_time = (now - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds()
hashed_batch = HashedBatch(user_id_hash=user_id_hash, timestamp=epoch_time, items=batch.items)
- stats_manager.record_batch(hashed_batch)
-
filename = upload_object(hashed_batch, now, user_id_hash, "batch")
global last_batch
@@ -157,10 +155,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
# Record the batch as being local so that we don't retrieve it again when the server restarts
infos = [BatchInfo(batch_url, user_id_hash, BatchStatus.LOCAL, device_name=batch.device_name)]
- with Database() as db:
- index_db = IndexDatabase(db.connection)
- index_db.record_batches(infos)
-
# Update or create Device records for this batch
if batch.device_name:
Device.objects.update_or_create(user=user, hostname=batch.device_name, defaults={})
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 4f04264..5f57a2a 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -37,8 +37,7 @@ def hour_count_key(date_time: datetime) -> str:
Formatted rather than built from a truncated datetime so that the key does not depend
on whether ``date_time`` is aware: ``str()`` of an aware datetime carries a ``+00:00``
- suffix, which would split the writer in :meth:`StatsManager.record_batch` from the
- reader in :meth:`StatsManager.get_stats` and zero the hourly chart.
+ suffix, which would split the reader in :meth:`StatsManager.get_stats` and zero the hourly chart.
"""
return URL_HOUR_COUNT_KEY.format(hour=date_time.strftime("%Y-%m-%d %H:00:00"))
@@ -240,9 +239,6 @@ class StatsManager:
self.redis.zincrby(user_result_count_key, num_results, username)
self.redis.expire(user_result_count_key, SHORT_EXPIRE_SECONDS)
- # The new crawler submits via the API-key-authenticated results endpoint rather than
- # record_batch, so this is the path that sees real crawlers. Add the account so that
- # users_crawled_daily counts distinct crawlers that contacted us today.
users_key = USERS_KEY.format(date=utc_today())
self.redis.sadd(users_key, username)
self.redis.expire(users_key, LONG_EXPIRE_SECONDS)
@@ -283,31 +279,3 @@ class StatsManager:
dataset_results_count_key = DATASET_RESULTS_COUNT_KEY.format(date=dataset_date)
self.redis.incrby(dataset_results_count_key, num_successful_results)
self.redis.expire(dataset_results_count_key, LONG_EXPIRE_SECONDS)
-
-
-def get_test_batches():
- for path in glob("./devdata/batches/**/*.json.gz", recursive=True):
- print("Processing path", path)
- with gzip.open(path) as gzip_file:
- yield HashedBatch.parse_raw(gzip_file.read())
-
-
-if __name__ == "__main__":
- django.setup()
- redis = Redis(host="localhost", port=6379, decode_responses=True)
- stats = StatsManager(redis)
- batches = get_test_batches()
- start = datetime.now(timezone.utc)
- processed = 0
- import logging
-
- logging.basicConfig(level=logging.INFO)
- for batch in islice(batches, 10000):
- if len(batch.items) <= 2:
- continue
- stats.record_batch(batch)
- processed += 1
- total_time = (datetime.now(timezone.utc) - start).total_seconds()
- print("Processed", processed)
- print("Total time", total_time)
- print("Time per batch", total_time / processed)
diff --git a/test/test_stats_hourly_counts.py b/test/test_stats_hourly_counts.py
index 8bb1ff4..8b501b7 100644
--- a/test/test_stats_hourly_counts.py
+++ b/test/test_stats_hourly_counts.py
@@ -33,26 +33,14 @@ def test_the_hour_key_has_no_timezone_offset_in_it():
assert hour_count_key(aware) == "url-count-hour-2026-09-08 13:00:00"
-def test_a_recorded_batch_is_counted_against_the_utc_hour_it_was_crawled_in():
- redis = fakeredis.FakeRedis(decode_responses=True)
- crawled_at = datetime(2026, 9, 8, 13, 45, tzinfo=timezone.utc)
-
- with patch("mwmbl.crawler.stats.URLDatabase"):
- StatsManager(redis).record_batch(make_batch(crawled_at))
-
- assert redis.get(URL_HOUR_COUNT_KEY.format(hour="2026-09-08 13:00:00")) == "1"
-
-
def test_the_stats_read_back_the_hour_key_the_batch_wrote():
- """The chart is built by get_stats reading keys record_batch wrote, so the two have to
+ """The chart is built by get_stats reading keys record_results wrote, so the two have to
agree on the format - not just each be self-consistent."""
redis = fakeredis.FakeRedis(decode_responses=True)
now = datetime.now(timezone.utc)
- with patch("mwmbl.crawler.stats.URLDatabase"):
+ with patch("mwmbl.crawler.urls.URLDatabase"):
stats_manager = StatsManager(redis)
- stats_manager.record_batch(make_batch(now))
-
with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):
stats = stats_manager.get_stats()
diff --git a/test/test_stats_users_crawled.py b/test/test_stats_users_crawled.py
index e2268e0..ee76471 100644
--- a/test/test_stats_users_crawled.py
+++ b/test/test_stats_users_crawled.py
@@ -1,13 +1,11 @@
"""
Tests for "users crawled today" — the count of distinct crawlers that contacted us.
-Both submission paths feed the same users set that `get_stats` reads back with `scard`
+The submission path feeds the same users set that `get_stats` reads back with `scard`
to build `users_crawled_daily`:
-- `record_batch` (the deprecated batch flow) adds the crawler's self-claimed
- `user_id_hash`.
-- `record_results` (the modern API-key-authenticated results endpoint) adds the
- account's `username`.
+`record_results` (the modern API-key-authenticated results endpoint) adds the
+account's `username`.
Because writer and reader are in different methods, a change to one side that splits
the key format would silently zero the chart, so these tests pin the two writers to the
@@ -59,19 +57,3 @@ def test_record_results_counts_each_crawler_once_per_day():
today = str(utc_today())
assert stats.users_crawled_daily[today] == 2
-
-
-def test_both_submission_paths_feed_the_same_users_set():
- redis = fakeredis.FakeRedis(decode_responses=True)
- stats_manager = StatsManager(redis)
-
- # The old batch flow and the new results endpoint each contribute a distinct crawler.
- with patch("mwmbl.crawler.stats.URLDatabase"):
- stats_manager.record_batch(make_batch(user_id_hash="a" * 64))
- stats_manager.record_results(make_results("https://example.com/"), "alice")
-
- with patch("mwmbl.crawler.stats.get_counts", return_value=NO_INDEX_COUNTS):
- stats = stats_manager.get_stats()
-
- today = str(utc_today())
- assert stats.users_crawled_daily[today] == 2