Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Update domain links

Daoud Clarkeb3db857eafa3
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.py
index 45af3f5..3cbdfe3 100644
--- a/mwmbl/crawler/domains.py
+++ b/mwmbl/crawler/domains.py
@@ -9,7 +9,7 @@ from pybloomfilter import BloomFilter
from mwmbl.hn_top_domains_filtered import DOMAINS
-URL_GROUPS = [
+DOMAIN_GROUPS = [
'github.com',
'en.wikipedia.org',
'news.ycombinator.com',
@@ -23,12 +23,12 @@ TOP_DOMAINS = set(islice(DOMAINS, 1000))
OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))
-def get_bloom_filter(url_group: str) -> BloomFilter:
+def get_bloom_filter(domain_group: str) -> BloomFilter:
try:
- bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))
+ bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group))
except FileNotFoundError:
bloom_filter = BloomFilter(settings.NUM_DOMAINS_IN_BLOOM_FILTER, 1e-6,
- settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))
+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(domain_group=domain_group), perm=0o666)
return bloom_filter
@@ -37,18 +37,19 @@ class DomainLinkDatabase:
self.links = {}
def __enter__(self):
- self.links = {url_group: get_bloom_filter(url_group) for url_group in URL_GROUPS}
+ self.links = {domain_group: get_bloom_filter(domain_group) for domain_group in DOMAIN_GROUPS}
+ return self
def __exit__(self, exc_type, exc_val, exc_tb):
for bloom_filter in self.links.values():
bloom_filter.close()
- def update_domain_links(self, source: str, target: list[str]):
+ def update_domain_links(self, source: str, target: set[str]):
if source in TOP_DOMAINS:
url_group = 'top'
elif source in OTHER_DOMAINS:
url_group = 'other'
- elif source in URL_GROUPS:
+ elif source in DOMAIN_GROUPS:
url_group = source
else:
# This is a URL that we don't care about
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index 4bf6b48..c0fa0ba 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -59,6 +59,7 @@ class URLDatabase:
except FileNotFoundError:
logger.info("No existing bloom filter found, creating a new one")
self.urls = BloomFilter(settings.NUM_URLS_IN_BLOOM_FILTER, 1e-6, settings.URLS_BLOOM_FILTER_PATH, perm=0o666)
+ return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.urls.close()
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index 2ac8f59..94d76cb 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -8,6 +8,7 @@ from typing import Collection
from urllib.parse import urlparse
from mwmbl.crawler.batch import HashedBatch
+from mwmbl.crawler.domains import DomainLinkDatabase
from mwmbl.crawler.urls import URLDatabase, URLStatus, FoundURL
from mwmbl.database import Database
from mwmbl.hn_top_domains_filtered import DOMAINS
@@ -45,10 +46,10 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qu
logger.info(f"Recording URLs in database for {len(batches)} batches, with {len(blacklist_domains)} blacklist "
f"domains, retrieved in {blacklist_retrieval_time.total_seconds()} seconds")
- url_scores = defaultdict(float)
url_users = {}
url_timestamps = {}
url_statuses = defaultdict(lambda: URLStatus.NEW)
+ domain_links = defaultdict(set)
for batch in batches:
for item in batch.items:
timestamp = get_datetime_from_timestamp(item.timestamp / 1000.0)
@@ -63,18 +64,17 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qu
except ValueError:
logger.info(f"Couldn't parse URL {item.url}")
continue
- score_multiplier = 1 if crawled_page_domain in DOMAINS else UNKNOWN_DOMAIN_MULTIPLIER
for link in item.content.links:
- process_link(batch.user_id_hash, crawled_page_domain, link, score_multiplier, timestamp, url_scores,
- url_timestamps, url_users, False, blacklist_domains)
+ process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,
+ blacklist_domains, domain_links)
if item.content.extra_links:
for link in item.content.extra_links:
- process_link(batch.user_id_hash, crawled_page_domain, link, score_multiplier, timestamp, url_scores,
- url_timestamps, url_users, True, blacklist_domains)
+ process_link(batch.user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users,
+ blacklist_domains, domain_links)
found_urls = [FoundURL(url, url_users[url], url_statuses[url], url_timestamps[url])
- for url in url_scores.keys() | url_statuses.keys()]
+ for url in url_statuses.keys() | url_users.keys()]
logger.info(f"Found URLs, {len(found_urls)}")
@@ -83,8 +83,13 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qu
new_item_queue.put(new_urls)
logger.info(f"Put {len(new_urls)} new items in the URL queue")
+ with DomainLinkDatabase() as domain_link_db:
+ for source_domain, target_domains in domain_links.items():
+ domain_link_db.update_domain_links(source_domain, target_domains)
-def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multiplier, timestamp, url_scores, url_timestamps, url_users, is_extra: bool, blacklist_domains):
+
+def process_link(user_id_hash, crawled_page_domain, link, timestamp, url_timestamps, url_users, blacklist_domains,
+ domain_links):
try:
parsed_link = urlparse(link)
except ValueError:
@@ -95,15 +100,12 @@ def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multipl
logger.debug(f"Excluding link for blacklisted domain: {parsed_link}")
return
- extra_multiplier = EXTRA_LINK_MULTIPLIER if is_extra else 1.0
- score = SCORE_FOR_SAME_DOMAIN if parsed_link.netloc == crawled_page_domain else SCORE_FOR_DIFFERENT_DOMAIN
- url_scores[link] += score * unknown_domain_multiplier * extra_multiplier
url_users[link] = user_id_hash
url_timestamps[link] = timestamp
domain = f'{parsed_link.scheme}://{parsed_link.netloc}/'
- url_scores[domain] += SCORE_FOR_ROOT_PATH * unknown_domain_multiplier
url_users[domain] = user_id_hash
url_timestamps[domain] = timestamp
+ domain_links[crawled_page_domain].add(domain)
def get_datetime_from_timestamp(timestamp: float) -> datetime:
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index 2f8875c..843a970 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -32,5 +32,5 @@ NUM_PAGES = 2560
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 100_000
-DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")
+DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")
NUM_DOMAINS_IN_BLOOM_FILTER = 100_000
diff --git a/test/test_update_urls.py b/test/test_update_urls.py
index 8f205f8..387fbdf 100644
--- a/test/test_update_urls.py
+++ b/test/test_update_urls.py
@@ -10,12 +10,10 @@ def test_process_link_normal():
user_id_hash="abc123",
crawled_page_domain="somewhere.com",
link="https://somesite.com/something.html",
- unknown_domain_multiplier=1,
timestamp=1234,
- url_scores=url_scores,
url_timestamps=url_timestamps,
url_users=url_users,
- is_extra=False, blacklist_domains=[]
+ blacklist_domains=[]
)
assert url_scores["https://somesite.com/something.html"] > 0.0
@@ -30,12 +28,10 @@ def test_process_link_excludes_porn():
user_id_hash="abc123",
crawled_page_domain="somewhere.com",
link="https://somepornsite.com/something.html",
- unknown_domain_multiplier=1,
timestamp=1234,
- url_scores=url_scores,
url_timestamps=url_timestamps,
url_users=url_users,
- is_extra=False, blacklist_domains=[]
+ blacklist_domains=[]
)
assert url_scores == {}