Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Estimate the last time we crawled a URL using multiple Bloom filters

Daoud Clarkea7574eaac801
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index 5dfe3ed..8342259 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -2,10 +2,11 @@
Database storing info on URLs
"""
from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
from enum import Enum
from logging import getLogger
from pathlib import Path
+from typing import Optional
from django.conf import settings
from psycopg2.extras import execute_values
@@ -47,38 +48,55 @@ class FoundURL:
user_id_hash: str
status: URLStatus
timestamp: datetime
+ last_crawled: Optional[datetime] = None
class URLDatabase:
def __init__(self):
- self.urls = None
+ self.urls = {}
def __enter__(self):
- try:
- self.urls = BloomFilter.open(settings.URLS_BLOOM_FILTER_PATH)
- except FileNotFoundError:
- logger.info("No existing bloom filter found, creating a new one")
- self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)
- return self
+ month_date = datetime.utcnow()
+ for i in range(3):
+ # Start from current month and go back two months
+ month_date = datetime(month_date.year, month_date.month, 1)
+ urls_path = settings.URLS_BLOOM_FILTER_PATH.format(month=month_date.month, year=month_date.year)
+ try:
+ self.urls[month_date] = BloomFilter.open(urls_path)
+ except FileNotFoundError:
+ if i == 0:
+ logger.info("No existing bloom filter found, creating a new one")
+ self.urls[month_date] = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, urls_path, perm=0o666)
+ else:
+ logger.info("No existing bloom filter found, using fallback")
+ month_date -= timedelta(days=1)
+ return self
+
+ self.urls[datetime(2024, 1, 1)] = BloomFilter.open(settings.URLS_BLOOM_FILTER_FALLBACK_PATH)
+ logger.info(f"Initialised URL crawled DB with dates {self.urls.keys()}")
def __exit__(self, exc_type, exc_val, exc_tb):
- self.urls.close()
+ for month_date, bloom_filter in self.urls.items():
+ bloom_filter.close()
def update_found_urls(self, found_urls: list[FoundURL]):
"""
- Update URL that have been crawled, and return any that have not yet been crawled
+ Update URLs with the most recent crawl date, if they've been crawled before, or None otherwise.
+ Update the most recent URL status in the database.
"""
+ most_recent_urls = next(iter(self.urls.values()))
new_urls = []
- num_crawled_urls = 0
for url in found_urls:
- if url.url not in self.urls:
- if url.status in CRAWLED_STATUSES:
- self.urls.add(url.url)
- num_crawled_urls += 1
- else:
- new_urls.append(url)
+ found_date = None
+ for date, bloom_filter in self.urls.items():
+ if url.url in bloom_filter:
+ found_date = date
+ break
+
+ if url.status in CRAWLED_STATUSES:
+ most_recent_urls.add(url.url)
- logger.info(f"Found {num_crawled_urls} crawled URLs and {len(new_urls)} new URLs")
+ new_urls.append(FoundURL(url.url, url.user_id_hash, url.status, url.timestamp, found_date))
return new_urls
def __contains__(self, url):
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index bc0290d..5db50b0 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -29,7 +29,6 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):
while True:
try:
run(batch_cache, new_item_queue)
- add_hn_links(new_item_queue)
except Exception:
logger.exception("Error updating URLs")
sleep(10)
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.py
index a1db1db..52bd0f3 100644
--- a/mwmbl/redis_url_queue.py
+++ b/mwmbl/redis_url_queue.py
@@ -1,3 +1,6 @@
+import math
+from datetime import datetime, timedelta
+
from logging import getLogger
from random import Random
from urllib.parse import urlparse
@@ -11,6 +14,8 @@ from mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklisted
from mwmbl.settings import CORE_DOMAINS
+MAX_TIME_DELTA = timedelta(days=100000)
+
random = Random(1)
logger = getLogger(__name__)
@@ -33,6 +38,9 @@ NUM_OTHER_URLS_TO_INCLUDE = 100
BATCH_SIZE = 100
+# Discount URLs crawled recently - this is the scale - currently 10 months
+SCORE_TIME_CONSTANT = 60 * 60 * 24 * 30 * 10
+
def get_domain_max_urls(domain: str):
if domain in CORE_DOMAINS:
@@ -51,8 +59,21 @@ class RedisURLQueue:
def queue_urls(self, found_urls: list[FoundURL]):
with DomainLinkDatabase() as link_db:
for url in found_urls:
+ time_since_crawled = (datetime.utcnow() - url.last_crawled
+ if url.last_crawled is not None else MAX_TIME_DELTA)
+
+ # Skip URLs crawled in the last month
+ if time_since_crawled < timedelta(days=30):
+ continue
+
domain = urlparse(url.url).netloc
url_score = 1/len(url.url)
+
+ # Discount URLs that were crawled recently
+ score_multiplier = 1 - math.exp(-time_since_crawled.total_seconds() / SCORE_TIME_CONSTANT)
+ url_score *= score_multiplier
+ logger.info(f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}")
+
domain_score = link_db.get_domain_score(domain) + url_score
max_urls = get_domain_max_urls(domain)
self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index a795754..1301e18 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -30,7 +30,8 @@ INDEX_NAME = 'index-v2.tinysearch'
NUM_PAGES = 2560
-URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
+URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")
+URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 100_000
DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")
diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.py
index 7a82d23..445e43b 100644
--- a/mwmbl/settings_prod.py
+++ b/mwmbl/settings_prod.py
@@ -35,7 +35,8 @@ INDEX_NAME = 'index-v2-400G.tinysearch'
# 400GB index
NUM_PAGES = 102400000
-URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
+URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")
+URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 200_000_000
DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")