Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add HN links to URL queue

Daoud Clarke8dce03ee3dbf
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.py
index 1579261..7b08b2d 100644
--- a/mwmbl/crawler/domains.py
+++ b/mwmbl/crawler/domains.py
@@ -23,8 +23,8 @@ DOMAIN_GROUPS = [
('other', 1),
]
-TOP_DOMAINS = set(islice(DOMAINS, 1000))
-OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))
+TOP_DOMAINS = set(islice(DOMAINS, 4000))
+OTHER_DOMAINS = set(islice(DOMAINS, 4000, 10000))
def get_bloom_filter(domain_group: str) -> BloomFilter:
@@ -45,7 +45,7 @@ class DomainLinkDatabase:
return self
def __exit__(self, exc_type, exc_val, exc_tb):
- for bloom_filter in self.links.values():
+ for bloom_filter, _ in self.links.values():
bloom_filter.close()
def update_domain_links(self, source: str, target: set[str]):
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index e98ea4b..bc0290d 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -6,6 +6,8 @@ from time import sleep
from typing import Collection
from urllib.parse import urlparse
+import requests
+
from mwmbl.crawler.batch import HashedBatch
from mwmbl.crawler.domains import DomainLinkDatabase
from mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL
@@ -14,7 +16,7 @@ from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.blacklist import get_blacklist_domains, is_domain_blacklisted
from mwmbl.indexer.index_batches import get_url_error_status
from mwmbl.indexer.indexdb import BatchStatus
-from mwmbl.redis_url_queue import RedisURLQueue
+from mwmbl.redis_url_queue import RedisURLQueue, get_domain_max_urls
from mwmbl.utils import get_domain
from django.conf import settings
@@ -27,6 +29,7 @@ def update_urls_continuously(data_path: str, new_item_queue: RedisURLQueue):
while True:
try:
run(batch_cache, new_item_queue)
+ add_hn_links(new_item_queue)
except Exception:
logger.exception("Error updating URLs")
sleep(10)
@@ -76,14 +79,34 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Re
logger.info(f"Found URLs, {len(found_urls)}")
+ _add_found_urls_to_db_and_queue(found_urls, new_item_queue)
+
+ with DomainLinkDatabase() as domain_link_db:
+ for source_domain, target_domains in domain_links.items():
+ domain_link_db.update_domain_links(source_domain, target_domains)
+
+
+def _add_found_urls_to_db_and_queue(found_urls, new_item_queue):
with URLDatabase() as url_db:
new_urls = url_db.update_found_urls(found_urls)
new_item_queue.queue_urls(new_urls)
logger.info(f"Put {len(new_urls)} new items in the URL queue")
- with DomainLinkDatabase() as domain_link_db:
- for source_domain, target_domains in domain_links.items():
- domain_link_db.update_domain_links(source_domain, target_domains)
+
+def add_hn_links(new_item_queue: RedisURLQueue):
+ hn_count = new_item_queue.get_domain_count('news.ycombinator.com')
+ max_count = get_domain_max_urls('news.ycombinator.com')
+
+ max_item = requests.get('https://hacker-news.firebaseio.com/v0/maxitem.json').json()
+ logger.info(f"Adding HN links, current count: {hn_count}, max count: {max_count}, max item ID: {max_item}")
+ num_items_to_add = 100
+ while hn_count < max_count:
+ urls = [f'https://news.ycombinator.com/item?id={item_id}' for item_id in range(max_item, max_item - num_items_to_add, -1)]
+ found_urls = [FoundURL(url, 'hn', URLStatus.NEW, datetime.now()) for url in urls]
+ _add_found_urls_to_db_and_queue(found_urls, new_item_queue)
+ hn_count = new_item_queue.get_domain_count('news.ycombinator.com')
+ max_item -= num_items_to_add
+ logger.info(f"Added {len(urls)} HN links, {hn_count} total, max item ID: {max_item}")
def process_link(user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users, blacklist_domains,
diff --git a/mwmbl/redis_url_queue.py b/mwmbl/redis_url_queue.py
index 4c6bc68..a1db1db 100644
--- a/mwmbl/redis_url_queue.py
+++ b/mwmbl/redis_url_queue.py
@@ -56,7 +56,7 @@ class RedisURLQueue:
domain_score = link_db.get_domain_score(domain) + url_score
max_urls = get_domain_max_urls(domain)
self.redis.zadd(DOMAIN_URLS_KEY.format(domain=domain), {url.url: url_score})
- self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -max_urls)
+ self.redis.zremrangebyrank(DOMAIN_URLS_KEY.format(domain=domain), 0, -(max_urls + 1))
self.redis.zadd(DOMAIN_SCORE_KEY, {domain: domain_score}, gt=True)
# Remove the lowest scoring domains
@@ -110,3 +110,6 @@ class RedisURLQueue:
logger.info(f"Returning URLs: {urls}")
return urls
+
+ def get_domain_count(self, domain: str):
+ return self.redis.zcard(DOMAIN_URLS_KEY.format(domain=domain))
diff --git a/mwmbl/settings.py b/mwmbl/settings.py
index 5fc8cfa..3be9052 100644
--- a/mwmbl/settings.py
+++ b/mwmbl/settings.py
@@ -42,6 +42,7 @@ CORE_DOMAINS = {
'developer.mozilla.org',
'arxiv.org',
'www.python.org',
+ 'news.ycombinator.com',
}
BLACKLIST_DOMAINS_URL = "https://get.domainsblacklists.com/blacklist.txt"