Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Only allow the user to crawl URLs that have been assigned to them

Daoud Clarke94fd4df9bec9
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 11b7006..438cac7 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -68,6 +68,11 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
user_id_hash = _get_user_id_hash(batch)
+ urls = [item.url for item in batch.items]
+ invalid_urls = queued_batches.check_user_crawled_urls(user_id_hash, urls)
+ if invalid_urls:
+ raise HTTPException(400, f"The following URLs were not assigned to the user: {invalid_urls}")
+
now = datetime.now(timezone.utc)
seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
@@ -111,7 +116,7 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:
user_id_hash = _get_user_id_hash(batch_request)
try:
- urls = queued_batches.get_batch()
+ urls = queued_batches.get_batch(user_id_hash)
except Empty:
return []
# TODO: track which URLs are currently being crawled
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.py
index 3f6d3cd..485bced 100644
--- a/mwmbl/redis_url_queue.py
+++ b/mwmbl/redis_url_queue.py
@@ -1,3 +1,4 @@
+import json
import math
from datetime import datetime, timedelta
@@ -23,6 +24,9 @@ logger = getLogger(__name__)
DOMAIN_URLS_KEY = "domain-urls-{domain}"
DOMAIN_SCORE_KEY = "domain-scores"
+USER_URLS_KEY = "user-urls-{user_id}"
+
+USER_EXPIRY_SECONDS = 60 * 60 * 24 * 7
MAX_URLS_PER_CORE_DOMAIN = 1000
@@ -87,7 +91,7 @@ class RedisURLQueue:
logger.info(f"Queued {len(found_urls)} URLs, number of domains: {self.redis.zcard(DOMAIN_SCORE_KEY)}")
- def get_batch(self) -> list[str]:
+ def get_batch(self, user_id: str) -> list[str]:
top_scoring_domains = set(self.redis.zrange(DOMAIN_SCORE_KEY, 0, 2000, desc=True))
top_other_domains = top_scoring_domains - DOMAINS.keys()
curated_domains = self.get_curated_domains_function()
@@ -131,7 +135,21 @@ class RedisURLQueue:
break
logger.info(f"Returning URLs: {urls}")
+
+ # Assign the URLs to this user
+ user_url_str = json.dumps(urls)
+ self.redis.set(USER_URLS_KEY.format(user_id=user_id), user_url_str)
+ self.redis.expire(USER_URLS_KEY.format(user_id=user_id), USER_EXPIRY_SECONDS)
+
return urls
+ def check_user_crawled_urls(self, user_id: str, urls: list[str]):
+ user_assigned_urls = self.redis.get(USER_URLS_KEY.format(user_id=user_id))
+ if user_assigned_urls is None:
+ return urls
+
+ user_assigned_url_set = set(json.loads(user_assigned_urls))
+ return [url for url in urls if url not in user_assigned_url_set]
+
def get_domain_count(self, domain: str):
return self.redis.zcard(DOMAIN_URLS_KEY.format(domain=domain))