An open source, non-profit web search engine
Add curated domains to the crawl queue
Daoud Clarkea26fa6225fa4
diff --git a/mwmbl/main.py b/mwmbl/main.pyindex aefe80a..a8ba2e7 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -5,16 +5,15 @@ import uvicornfrom django.core.management import call_commandfrom redis import Redis-from mwmbl.count_urls import count_urls_continuously-from mwmbl.indexer.update_urls import update_urls_continuously-from mwmbl.redis_url_queue import RedisURLQueue-def run():django.setup()from django.conf import settingsfrom mwmbl import background+ from mwmbl.redis_url_queue import RedisURLQueue+ from mwmbl.count_urls import count_urls_continuously+ from mwmbl.indexer.update_urls import update_urls_continuouslyif settings.STATIC_ROOT:call_command("collectstatic", "--clear", "--noinput")diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex 52bd0f3..7519183 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -11,6 +11,7 @@ from mwmbl.crawler.domains import DomainLinkDatabase, TOP_DOMAINSfrom mwmbl.crawler.urls import FoundURLfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklisted+from mwmbl.models import DomainSubmissionfrom mwmbl.settings import CORE_DOMAINS@@ -29,10 +30,6 @@ MAX_URLS_PER_TOP_DOMAIN = 100MAX_URLS_PER_OTHER_DOMAIN = 5MAX_OTHER_DOMAINS = 10000-MAX_BATCH_URLS_PER_CORE_DOMAIN = 100-MAX_BATCH_URLS_PER_TOP_DOMAIN = 10-MAX_BATCH_URLS_PER_OTHER_DOMAIN = 1-NUM_TOP_DOMAIN_URLS_TO_INCLUDE = 50NUM_OTHER_URLS_TO_INCLUDE = 100@@ -42,8 +39,8 @@ BATCH_SIZE = 100SCORE_TIME_CONSTANT = 60 * 60 * 24 * 30 * 10-def get_domain_max_urls(domain: str):- if domain in CORE_DOMAINS:+def get_domain_max_urls(domain: str, curated_domains: set[str]):+ if domain in CORE_DOMAINS | curated_domains:return MAX_URLS_PER_CORE_DOMAINelif domain in TOP_DOMAINS:return MAX_URLS_PER_TOP_DOMAIN@@ -51,12 +48,19 @@ def get_domain_max_urls(domain: str):return MAX_URLS_PER_OTHER_DOMAIN+def _get_curated_domains():+ curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))+ return curated_domains++class RedisURLQueue:def __init__(self, redis: Redis):self.redis = redisself.black_listed_domains = get_blacklist_domains()def queue_urls(self, found_urls: list[FoundURL]):+ curated_domains = _get_curated_domains()+ logger.info(f"Got {len(found_urls)} URLs, {len(curated_domains)} curated domains")with DomainLinkDatabase() as link_db:for url in found_urls:time_since_crawled = (datetime.utcnow() - url.last_crawled@@ -75,7 +79,7 @@ class RedisURLQueue:logger.info(f"URL score: {url_score}, score multiplier: {score_multiplier} for domain {domain} and age {time_since_crawled}")domain_score = link_db.get_domain_score(domain) + url_score- max_urls = get_domain_max_urls(domain)+ max_urls = get_domain_max_urls(domain, curated_domains)self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -(max_urls + 1))self.redis.zadd(DOMAIN_SCORE_KEY, {domain: domain_score}, gt=True)@@ -90,9 +94,10 @@ class RedisURLQueue:def get_batch(self) -> list[str]:top_scoring_domains = set(self.redis.zrange(DOMAIN_SCORE_KEY, 0, 2000, desc=True))top_other_domains = top_scoring_domains - DOMAINS.keys()+ curated_domains = _get_curated_domains()domains = list(CORE_DOMAINS)- top_curated_domains = DOMAINS.keys() & top_scoring_domains+ top_curated_domains = (DOMAINS.keys() & top_scoring_domains) | curated_domainsif len(top_curated_domains) > NUM_TOP_DOMAIN_URLS_TO_INCLUDE:domains += random.sample(top_curated_domains, NUM_TOP_DOMAIN_URLS_TO_INCLUDE)else:@@ -107,7 +112,7 @@ class RedisURLQueue:logger.info(f"Getting batch from domains {domains}")# Add a random url as the root domain of one of DOMAINS- random_domain = random.choice(list(DOMAINS.keys()))+ random_domain = random.choice(list(DOMAINS.keys() | curated_domains))urls = [f"https://{random_domain}/"]# Pop the highest scoring URL from each domain