An open source, non-profit web search engine
Update domain links
Daoud Clarkeb3db857eafa3
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.pyindex 45af3f5..3cbdfe3 100644--- a/mwmbl/crawler/domains.py+++ b/mwmbl/crawler/domains.py@@ -9,7 +9,7 @@ from pybloomfilter import BloomFilterfrom mwmbl.hn_top_domains_filtered import DOMAINS-URL_GROUPS = [+DOMAIN_GROUPS = ['github.com','en.wikipedia.org','news.ycombinator.com',@@ -23,12 +23,12 @@ TOP_DOMAINS = set(islice(DOMAINS, 1000))OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))-def get_bloom_filter(url_group: str) -> BloomFilter:+def get_bloom_filter(domain_group: str) -> BloomFilter:try:- bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))+ bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group))except FileNotFoundError:bloom_filter = BloomFilter(settings.NUM_DOMAINS_IN_BLOOM_FILTER, 1e-6,- settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group), perm=0o666)return bloom_filter@@ -37,18 +37,19 @@ class DomainLinkDatabase:self.links = {}def __enter__(self):- self.links = {url_group: get_bloom_filter(url_group) for url_group in URL_GROUPS}+ self.links = {domain_group: get_bloom_filter(domain_group) for domain_group in DOMAIN_GROUPS}+ return selfdef __exit__(self, exc_type, exc_val, exc_tb):for bloom_filter in self.links.values():bloom_filter.close()- def update_domain_links(self, source: str, target: list[str]):+ def update_domain_links(self, source: str, target: set[str]):if source in TOP_DOMAINS:url_group = 'top'elif source in OTHER_DOMAINS:url_group = 'other'- elif source in URL_GROUPS:+ elif source in DOMAIN_GROUPS:url_group = sourceelse:# This is a URL that we don't care aboutdiff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex 4bf6b48..c0fa0ba 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -59,6 +59,7 @@ class URLDatabase:except FileNotFoundError:logger.info("No existing bloom filter found, creating a new one")self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)+ return selfdef __exit__(self, exc_type, exc_val, exc_tb):self.urls.close()diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex 2ac8f59..94d76cb 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -8,6 +8,7 @@ from typing import Collectionfrom urllib.parse import urlparsefrom mwmbl.crawler.batch import HashedBatch+from mwmbl.crawler.domains import DomainLinkDatabasefrom mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURLfrom mwmbl.database import Databasefrom mwmbl.hn_top_domains_filtered import DOMAINS@@ -45,10 +46,10 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qulogger.info(f"Recording URLs in database for {len(batches)} batches, with {len(blacklist_domains)} blacklist "f"domains, retrieved in {blacklist_retrieval_time.total_seconds()} seconds")- url_scores = defaultdict(float)url_users = {}url_timestamps = {}url_statuses = defaultdict(lambda: URLStatus.NEW)+ domain_links = defaultdict(set)for batch in batches:for item in batch.items:timestamp = get_datetime_from_timestamp(item.timestamp / 1000.0)@@ -63,18 +64,17 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Quexcept ValueError:logger.info(f"Couldn't parse URL {item.url}")continue- score_multiplier = 1 if crawled_page_domain in DOMAINS else UNKNOWN_DOMAIN_MULTIPLIERfor link in item.content.links:- process_link(batch.user_id_hash, crawled_page_domain, link, score_multiplier, timestamp, url_scores,- url_timestamps, url_users, False, blacklist_domains)+ process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,+ blacklist_domains, domain_links)if item.content.extra_links:for link in item.content.extra_links:- process_link(batch.user_id_hash, crawled_page_domain, link, score_multiplier, timestamp, url_scores,- url_timestamps, url_users, True, blacklist_domains)+ process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,+ blacklist_domains, domain_links)found_urls = [FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])- for url in url_scores.keys() | url_statuses.keys()]+ for url in url_statuses.keys() | url_users.keys()]logger.info(f"Found URLs, {len(found_urls)}")@@ -83,8 +83,13 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qunew_item_queue.put(new_urls)logger.info(f"Put {len(new_urls)} new items in the URL queue")+ with DomainLinkDatabase() as domain_link_db:+ for source_domain, target_domains in domain_links.items():+ domain_link_db.update_domain_links(source_domain, target_domains)-def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multiplier, timestamp, url_scores, url_timestamps, url_users, is_extra: bool, blacklist_domains):++def process_link(user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users, blacklist_domains,+ domain_links):try:parsed_link = urlparse(link)except ValueError:@@ -95,15 +100,12 @@ def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multipllogger.debug(f"Excluding link for blacklisted domain: {parsed_link}")return- extra_multiplier = EXTRA_LINK_MULTIPLIER if is_extra else 1.0- score = SCORE_FOR_SAME_DOMAIN if parsed_link.netloc == crawled_page_domain else SCORE_FOR_DIFFERENT_DOMAIN- url_scores[link] += score * unknown_domain_multiplier * extra_multiplierurl_users[link] = user_id_hashurl_timestamps[link] = timestampdomain = f'{parsed_link.scheme}://{parsed_link.netloc}/'- url_scores[domain] += SCORE_FOR_ROOT_PATH * unknown_domain_multiplierurl_users[domain] = user_id_hashurl_timestamps[domain] = timestamp+ domain_links[crawled_page_domain].add(domain)def get_datetime_from_timestamp(timestamp: float) -> datetime:diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex 2f8875c..843a970 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -32,5 +32,5 @@ NUM_PAGES = 2560URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 100_000-DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")+DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")NUM_DOMAINS_IN_BLOOM_FILTER = 100_000diff --git a/test/test_update_urls.py b/test/test_update_urls.pyindex 8f205f8..387fbdf 100644--- a/test/test_update_urls.py+++ b/test/test_update_urls.py@@ -10,12 +10,10 @@ def test_process_link_normal():user_id_hash="abc123",crawled_page_domain="somewhere.com",link="https://somesite.com/something.html",- unknown_domain_multiplier=1,timestamp=1234,- url_scores=url_scores,url_timestamps=url_timestamps,url_users=url_users,- is_extra=False, blacklist_domains=[]+ blacklist_domains=[])assert url_scores["https://somesite.com/something.html"] > 0.0@@ -30,12 +28,10 @@ def test_process_link_excludes_porn():user_id_hash="abc123",crawled_page_domain="somewhere.com",link="https://somepornsite.com/something.html",- unknown_domain_multiplier=1,timestamp=1234,- url_scores=url_scores,url_timestamps=url_timestamps,url_users=url_users,- is_extra=False, blacklist_domains=[]+ blacklist_domains=[])assert url_scores == {}