An open source, non-profit web search engine
Don't try and sample more domains than we have
Daoud Clarke9646f4b7bb69
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex fcd47da..a06588c 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -9,6 +9,7 @@ from mwmbl.crawler.urls import FoundURLfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.settings import CORE_DOMAINS+random = Random(1)logger = getLogger(__name__)@@ -26,6 +27,9 @@ MAX_BATCH_URLS_PER_CORE_DOMAIN = 100MAX_BATCH_URLS_PER_TOP_DOMAIN = 10MAX_BATCH_URLS_PER_OTHER_DOMAIN = 1+NUM_TOP_DOMAIN_URLS_TO_INCLUDE = 50+NUM_OTHER_URLS_TO_INCLUDE = 100+BATCH_SIZE = 100@@ -64,9 +68,17 @@ class RedisURLQueue:top_scoring_domains = set(self.redis.zrange(DOMAIN_SCORE_KEY, 0, 2000, desc=True))top_other_domains = top_scoring_domains - DOMAINS.keys()- domains = (list(CORE_DOMAINS)- + random.sample(DOMAINS.keys(), 50)- + random.sample(top_other_domains, 100))+ domains = list(CORE_DOMAINS)++ if len(DOMAINS) > NUM_TOP_DOMAIN_URLS_TO_INCLUDE:+ domains += random.sample(DOMAINS.keys(), NUM_TOP_DOMAIN_URLS_TO_INCLUDE)+ else:+ domains += list(DOMAINS.keys())++ if len(top_other_domains) > NUM_OTHER_URLS_TO_INCLUDE:+ domains += random.sample(top_other_domains, NUM_OTHER_URLS_TO_INCLUDE)+ else:+ domains += list(top_other_domains)logger.info(f"Getting batch from domains {domains}")