An open source, non-profit web search engine
Add HN links to URL queue
Daoud Clarke8dce03ee3dbf
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.pyindex 1579261..7b08b2d 100644--- a/mwmbl/crawler/domains.py+++ b/mwmbl/crawler/domains.py@@ -23,8 +23,8 @@ DOMAIN_GROUPS = [('other', 1),]-TOP_DOMAINS = set(islice(DOMAINS, 1000))-OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))+TOP_DOMAINS = set(islice(DOMAINS, 4000))+OTHER_DOMAINS = set(islice(DOMAINS, 4000, 10000))def get_bloom_filter(domain_group: str) -> BloomFilter:@@ -45,7 +45,7 @@ class DomainLinkDatabase:return selfdef __exit__(self, exc_type, exc_val, exc_tb):- for bloom_filter in self.links.values():+ for bloom_filter, _ in self.links.values():bloom_filter.close()def update_domain_links(self, source: str, target: set[str]):diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex e98ea4b..bc0290d 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -6,6 +6,8 @@ from time import sleepfrom typing import Collectionfrom urllib.parse import urlparse+import requests+from mwmbl.crawler.batch import HashedBatchfrom mwmbl.crawler.domains import DomainLinkDatabasefrom mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL@@ -14,7 +16,7 @@ from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklistedfrom mwmbl.indexer.index_batches import get_url_error_statusfrom mwmbl.indexer.indexdb import BatchStatus-from mwmbl.redis_url_queue import RedisURLQueue+from mwmbl.redis_url_queue import RedisURLQueue, get_domain_max_urlsfrom mwmbl.utils import get_domainfrom django.conf import settings@@ -27,6 +29,7 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):while True:try:run(batch_cache, new_item_queue)+ add_hn_links(new_item_queue)except Exception:logger.exception("Error updating URLs")sleep(10)@@ -76,14 +79,34 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Relogger.info(f"Found URLs, {len(found_urls)}")+ _add_found_urls_to_db_and_queue(found_urls, new_item_queue)++ with DomainLinkDatabase() as domain_link_db:+ for source_domain, target_domains in domain_links.items():+ domain_link_db.update_domain_links(source_domain, target_domains)+++def _add_found_urls_to_db_and_queue(found_urls, new_item_queue):with URLDatabase() as url_db:new_urls = url_db.update_found_urls(found_urls)new_item_queue.queue_urls(new_urls)logger.info(f"Put {len(new_urls)} new items in the URL queue")- with DomainLinkDatabase() as domain_link_db:- for source_domain, target_domains in domain_links.items():- domain_link_db.update_domain_links(source_domain, target_domains)++def add_hn_links(new_item_queue: RedisURLQueue):+ hn_count = new_item_queue.get_domain_count('news.ycombinator.com')+ max_count = get_domain_max_urls('news.ycombinator.com')++ max_item = requests.get('https://hacker-news.firebaseio.com/v0/maxitem.json').json()+ logger.info(f"Adding HN links, current count: {hn_count}, max count: {max_count}, max item ID: {max_item}")+ num_items_to_add = 100+ while hn_count < max_count:+ urls = [f'https://news.ycombinator.com/item?id={item_id}' for item_id in range(max_item, max_item - num_items_to_add, -1)]+ found_urls = [FoundURL(url, 'hn', URLStatus.NEW, datetime.now()) for url in urls]+ _add_found_urls_to_db_and_queue(found_urls, new_item_queue)+ hn_count = new_item_queue.get_domain_count('news.ycombinator.com')+ max_item -= num_items_to_add+ logger.info(f"Added {len(urls)} HN links, {hn_count} total, max item ID: {max_item}")def process_link(user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users, blacklist_domains,diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.pyindex 4c6bc68..a1db1db 100644--- a/mwmbl/redis_url_queue.py+++ b/mwmbl/redis_url_queue.py@@ -56,7 +56,7 @@ class RedisURLQueue:domain_score = link_db.get_domain_score(domain) + url_scoremax_urls = get_domain_max_urls(domain)self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})- self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -max_urls)+ self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -(max_urls + 1))self.redis.zadd(DOMAIN_SCORE_KEY, {domain: domain_score}, gt=True)# Remove the lowest scoring domains@@ -110,3 +110,6 @@ class RedisURLQueue:logger.info(f"Returning URLs: {urls}")return urls++ def get_domain_count(self, domain: str):+ return self.redis.zcard(DOMAIN_URLS_KEY.format(domain=domain))diff --git a/mwmbl/settings.py b/mwmbl/settings.pyindex 5fc8cfa..3be9052 100644--- a/mwmbl/settings.py+++ b/mwmbl/settings.py@@ -42,6 +42,7 @@ CORE_DOMAINS = {'developer.mozilla.org','arxiv.org','www.python.org',+ 'news.ycombinator.com',}BLACKLIST_DOMAINS_URL = "https://get.domainsblacklists.com/blacklist.txt"