Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Record the crawl time as a naive UTC datetime

Daoud Clarkefbb6c874e507
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index 7b5b02b..5638d22 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -2,7 +2,7 @@
Database storing info on URLs
"""
from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
from enum import Enum
from logging import getLogger
from pathlib import Path
@@ -108,8 +108,11 @@ class URLDatabase:
# before this batch), so queue_urls treats it as never-crawled and hands
# it out again. The bloom filters only record the month, which is too
# coarse to use here: on the 31st, the start of the month is already 30
- # days ago, which is exactly the re-queue threshold.
- found_date = url.timestamp
+ # days ago, which is exactly the re-queue threshold. Crawl timestamps
+ # are timezone-aware, while last_crawled is naive UTC everywhere else
+ # (the bloom filter dates, and the comparison in queue_urls), so convert
+ # it here.
+ found_date = url.timestamp.astimezone(timezone.utc).replace(tzinfo=None)
new_urls.append(FoundURL(url.url, url.user_id_hash, url.status, url.timestamp, found_date))
return new_urls
diff --git a/test/test_url_database.py b/test/test_url_database.py
index 41df870..d7373ce 100644
--- a/test/test_url_database.py
+++ b/test/test_url_database.py
@@ -10,13 +10,14 @@ as never-crawled.
"""
import glob
import os
-from datetime import datetime
+from datetime import datetime, timezone
import fakeredis
import pytest
from mwmbl.crawler.urls import URLDatabase, FoundURL, URLStatus
from mwmbl.indexer.blacklist_providers import StaticBlacklistProvider
+from mwmbl.indexer.update_urls import get_datetime_from_timestamp
from mwmbl.redis_url_queue import RedisURLQueue
@@ -112,6 +113,30 @@ def test_crawl_date_is_the_crawl_time_not_the_start_of_the_month(clean_bloom_fil
assert new_urls[0].last_crawled == crawled_at
+def test_crawl_date_from_a_real_batch_timestamp_is_naive(clean_bloom_filters):
+ """Batch timestamps are timezone-aware, but last_crawled is compared against a naive
+ datetime.utcnow() in queue_urls, which raises if the two are mixed."""
+ crawled_at = get_datetime_from_timestamp(datetime.now(timezone.utc).timestamp())
+ found = FoundURL(
+ url="https://aware.example/page",
+ user_id_hash="user",
+ status=URLStatus.CRAWLED,
+ timestamp=crawled_at,
+ last_crawled=None,
+ )
+ with URLDatabase() as url_db:
+ new_urls = url_db.update_found_urls([found])
+
+ assert new_urls[0].last_crawled.tzinfo is None
+ assert new_urls[0].last_crawled == crawled_at.replace(tzinfo=None)
+
+ redis = fakeredis.FakeRedis(decode_responses=True)
+ url_queue = RedisURLQueue(redis, lambda: set(), StaticBlacklistProvider(set()))
+ url_queue.queue_urls(new_urls)
+
+ assert url_queue.get_domain_count("aware.example") == 0
+
+
def test_url_queue_gives_its_curated_domains_to_the_default_blacklist():
"""The standalone crawler has no database and fetches approved domains over HTTP, so
the queue's own source of curated domains is what has to reach the blacklist."""