An open source, non-profit web search engine
Only allow the user to crawl URLs that have been assigned to them
Daoud Clarke94fd4df9bec9
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 11b7006..438cac7 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -68,6 +68,11 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiouser_id_hash = _get_user_id_hash(batch)+ urls = [item.url for item in batch.items]+ invalid_urls = queued_batches.check_user_crawled_urls(user_id_hash, urls)+ if invalid_urls:+ raise HTTPException(400, f"The following URLs were not assigned to the user: {invalid_urls}")+now = datetime.now(timezone.utc)seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds@@ -111,7 +116,7 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiodef request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:user_id_hash = _get_user_id_hash(batch_request)try:- urls = queued_batches.get_batch()+ urls = queued_batches.get_batch(user_id_hash)except Empty:return []# TODO: track which URLs are currently being crawleddiff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex 3f6d3cd..485bced 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -1,3 +1,4 @@+import jsonimport mathfrom datetime import datetime, timedelta@@ -23,6 +24,9 @@ logger = getLogger(__name__)DOMAIN_URLS_KEY = "domain-urls-{domain}"DOMAIN_SCORE_KEY = "domain-scores"+USER_URLS_KEY = "user-urls-{user_id}"++USER_EXPIRY_SECONDS = 60 * 60 * 24 * 7MAX_URLS_PER_CORE_DOMAIN = 1000@@ -87,7 +91,7 @@ class RedisURLQueue:logger.info(f"Queued {len(found_urls)} URLs, number of domains: {self.redis.zcard(DOMAIN_SCORE_KEY)}")- def get_batch(self) -> list[str]:+ def get_batch(self, user_id: str) -> list[str]:top_scoring_domains = set(self.redis.zrange(DOMAIN_SCORE_KEY, 0, 2000, desc=True))top_other_domains = top_scoring_domains - DOMAINS.keys()curated_domains = self.get_curated_domains_function()@@ -131,7 +135,21 @@ class RedisURLQueue:breaklogger.info(f"Returning URLs: {urls}")++ # Assign the URLs to this user+ user_url_str = json.dumps(urls)+ self.redis.set(USER_URLS_KEY.format(user_id=user_id), user_url_str)+ self.redis.expire(USER_URLS_KEY.format(user_id=user_id), USER_EXPIRY_SECONDS)+return urls+ def check_user_crawled_urls(self, user_id: str, urls: list[str]):+ user_assigned_urls = self.redis.get(USER_URLS_KEY.format(user_id=user_id))+ if user_assigned_urls is None:+ return urls++ user_assigned_url_set = set(json.loads(user_assigned_urls))+ return [url for url in urls if url not in user_assigned_url_set]+def get_domain_count(self, domain: str):return self.redis.zcard(DOMAIN_URLS_KEY.format(domain=domain))