An open source, non-profit web search engine
Estimate the last time we crawled a URL using multiple Bloom filters
Daoud Clarkea7574eaac801
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex 5dfe3ed..8342259 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -2,10 +2,11 @@Database storing info on URLs"""from dataclasses import dataclass-from datetime import datetime+from datetime import datetime, timedeltafrom enum import Enumfrom logging import getLoggerfrom pathlib import Path+from typing import Optionalfrom django.conf import settingsfrom psycopg2.extras import execute_values@@ -47,38 +48,55 @@ class FoundURL:user_id_hash: strstatus: URLStatustimestamp: datetime+ last_crawled: Optional[datetime] = Noneclass URLDatabase:def __init__(self):- self.urls = None+ self.urls = {}def __enter__(self):- try:- self.urls = BloomFilter.open(settings.URLS_BLOOM_FILTER_PATH)- except FileNotFoundError:- logger.info("No existing bloom filter found, creating a new one")- self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)- return self+ month_date = datetime.utcnow()+ for i in range(3):+ # Start from current month and go back two months+ month_date = datetime(month_date.year, month_date.month, 1)+ urls_path = settings.URLS_BLOOM_FILTER_PATH.format(month=month_date.month, year=month_date.year)+ try:+ self.urls[month_date] = BloomFilter.open(urls_path)+ except FileNotFoundError:+ if i == 0:+ logger.info("No existing bloom filter found, creating a new one")+ self.urls[month_date] = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, urls_path, perm=0o666)+ else:+ logger.info("No existing bloom filter found, using fallback")+ month_date -= timedelta(days=1)+ return self++ self.urls[datetime(2024, 1, 1)] = BloomFilter.open(settings.URLS_BLOOM_FILTER_FALLBACK_PATH)+ logger.info(f"Initialised URL crawled DB with dates {self.urls.keys()}")def __exit__(self, exc_type, exc_val, exc_tb):- self.urls.close()+ for month_date, bloom_filter in self.urls.items():+ bloom_filter.close()def update_found_urls(self, found_urls: list[FoundURL]):"""- Update URL that have been crawled, and return any that have not yet been crawled+ Update URLs with the most recent crawl date, if they've been crawled before, or None otherwise.+ Update the most recent URL status in the database."""+ most_recent_urls = next(iter(self.urls.values()))new_urls = []- num_crawled_urls = 0for url in found_urls:- if url.url not in self.urls:- if url.status in CRAWLED_STATUSES:- self.urls.add(url.url)- num_crawled_urls += 1- else:- new_urls.append(url)+ found_date = None+ for date, bloom_filter in self.urls.items():+ if url.url in bloom_filter:+ found_date = date+ break++ if url.status in CRAWLED_STATUSES:+ most_recent_urls.add(url.url)- logger.info(f"Found {num_crawled_urls} crawled URLs and {len(new_urls)} new URLs")+ new_urls.append(FoundURL(url.url, url.user_id_hash, url.status, url.timestamp, found_date))return new_urlsdef __contains__(self, url):diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex bc0290d..5db50b0 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -29,7 +29,6 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):while True:try:run(batch_cache, new_item_queue)- add_hn_links(new_item_queue)except Exception:logger.exception("Error updating URLs")sleep(10)diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex a1db1db..52bd0f3 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -1,3 +1,6 @@+import math+from datetime import datetime, timedelta+from logging import getLoggerfrom random import Randomfrom urllib.parse import urlparse@@ -11,6 +14,8 @@ from mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklistedfrom mwmbl.settings import CORE_DOMAINS+MAX_TIME_DELTA = timedelta(days=100000)+random = Random(1)logger = getLogger(__name__)@@ -33,6 +38,9 @@ NUM_OTHER_URLS_TO_INCLUDE = 100BATCH_SIZE = 100+# Discount URLs crawled recently - this is the scale - currently 10 months+SCORE_TIME_CONSTANT = 60 * 60 * 24 * 30 * 10+def get_domain_max_urls(domain: str):if domain in CORE_DOMAINS:@@ -51,8 +59,21 @@ class RedisURLQueue:def queue_urls(self, found_urls: list[FoundURL]):with DomainLinkDatabase() as link_db:for url in found_urls:+ time_since_crawled = (datetime.utcnow() - url.last_crawled+ if url.last_crawled is not None else MAX_TIME_DELTA)++ # Skip URLs crawled in the last month+ if time_since_crawled < timedelta(days=30):+ continue+domain = urlparse(url.url).netlocurl_score = 1/len(url.url)++ # Discount URLs that were crawled recently+ score_multiplier = 1 - math.exp(-time_since_crawled.total_seconds() / SCORE_TIME_CONSTANT)+ url_score *= score_multiplier+ logger.info(f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}")+domain_score = link_db.get_domain_score(domain) + url_scoremax_urls = get_domain_max_urls(domain)self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex a795754..1301e18 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -30,7 +30,8 @@ INDEX_NAME = 'index-v2.tinysearch'NUM_PAGES = 2560-URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")+URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")+URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 100_000DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex 7a82d23..445e43b 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -35,7 +35,8 @@ INDEX_NAME = 'index-v2-400G.tinysearch'# 400GB indexNUM_PAGES = 102400000-URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")+URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")+URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 200_000_000DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")