An open source, non-profit web search engine
Record the crawl time as a naive UTC datetime
Daoud Clarkefbb6c874e507
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex 7b5b02b..5638d22 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -2,7 +2,7 @@Database storing info on URLs"""from dataclasses import dataclass-from datetime import datetime, timedelta+from datetime import datetime, timedelta, timezonefrom enum import Enumfrom logging import getLoggerfrom pathlib import Path@@ -108,8 +108,11 @@ class URLDatabase:# before this batch), so queue_urls treats it as never-crawled and hands# it out again. The bloom filters only record the month, which is too# coarse to use here: on the 31st, the start of the month is already 30- # days ago, which is exactly the re-queue threshold.- found_date = url.timestamp+ # days ago, which is exactly the re-queue threshold. Crawl timestamps+ # are timezone-aware, while last_crawled is naive UTC everywhere else+ # (the bloom filter dates, and the comparison in queue_urls), so convert+ # it here.+ found_date = url.timestamp.astimezone(timezone.utc).replace(tzinfo=None)new_urls.append(FoundURL(url.url, url.user_id_hash, url.status, url.timestamp, found_date))return new_urlsdiff --git a/test/test_url_database.py b/test/test_url_database.pyindex 41df870..d7373ce 100644--- a/test/test_url_database.py+++ b/test/test_url_database.py@@ -10,13 +10,14 @@ as never-crawled."""import globimport os-from datetime import datetime+from datetime import datetime, timezoneimport fakeredisimport pytestfrom mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatusfrom mwmbl.indexer.blacklist_providers import StaticBlacklistProvider+from mwmbl.indexer.update_urls import get_datetime_from_timestampfrom mwmbl.redis_url_queue import RedisURLQueue@@ -112,6 +113,30 @@ def test_crawl_date_is_the_crawl_time_not_the_start_of_the_month(clean_bloom_filassert new_urls[0].last_crawled == crawled_at+def test_crawl_date_from_a_real_batch_timestamp_is_naive(clean_bloom_filters):+ """Batch timestamps are timezone-aware, but last_crawled is compared against a naive+ datetime.utcnow() in queue_urls, which raises if the two are mixed."""+ crawled_at = get_datetime_from_timestamp(datetime.now(timezone.utc).timestamp())+ found = FoundURL(+ url="https://aware.example/page",+ user_id_hash="user",+ status=URLStatus.CRAWLED,+ timestamp=crawled_at,+ last_crawled=None,+ )+ with URLDatabase() as url_db:+ new_urls = url_db.update_found_urls([found])++ assert new_urls[0].last_crawled.tzinfo is None+ assert new_urls[0].last_crawled == crawled_at.replace(tzinfo=None)++ redis = fakeredis.FakeRedis(decode_responses=True)+ url_queue = RedisURLQueue(redis, lambda: set(), StaticBlacklistProvider(set()))+ url_queue.queue_urls(new_urls)++ assert url_queue.get_domain_count("aware.example") == 0++def test_url_queue_gives_its_curated_domains_to_the_default_blacklist():"""The standalone crawler has no database and fetches approved domains over HTTP, sothe queue's own source of curated domains is what has to reach the blacklist."""