An open source, non-profit web search engine
WIP: track links from top domains
Daoud Clarkeea31d5204b37
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.pynew file mode 100644index 0000000..54b79b9--- /dev/null+++ b/mwmbl/crawler/domains.py@@ -0,0 +1,62 @@+"""+Record which source domains link to destination domains. Each source domain is a bloom filter containing sets of+destination domains.+"""+from itertools import islice++from django.conf import settings+from pybloomfilter import BloomFilter++from mwmbl.hn_top_domains_filtered import DOMAINS++URL_GROUPS = [+ 'github.com',+ 'en.wikipedia.org',+ 'news.ycombinator.com',+ 'lemmy.ml',+ 'mastodon.social',+ 'top',+ 'other',+]++TOP_DOMAINS = set(islice(DOMAINS, 1000))+OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))+++def get_bloom_filter(url_group: str) -> BloomFilter:+ try:+ bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))+ except FileNotFoundError:+ bloom_filter = BloomFilter(settings.NUM_DOMAINS_IN_BLOOM_FILTER, 1e-6,+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))+ return bloom_filter+++class DomainLinkDatabase:+ def __init__(self):+ self.links = {}++ def __enter__(self):+ self.links = {url_group: get_bloom_filter(url_group) for url_group in URL_GROUPS}++ def __exit__(self, exc_type, exc_val, exc_tb):+ for bloom_filter in self.links.values():+ bloom_filter.close()++ def update_domain_links(self, source: str, target: list[str]):+ if source in TOP_DOMAINS:+ url_group = 'top'+ elif source in OTHER_DOMAINS:+ url_group = 'other'+ elif source in URL_GROUPS:+ url_group = source+ else:+ # This is a URL that we don't care about+ return++ bloom_filter = self.links[url_group]+ bloom_filter.update(target)++++diff --git a/mwmbl/indexer/domains.py b/mwmbl/indexer/domains.pydeleted file mode 100644index 242df76..0000000--- a/mwmbl/indexer/domains.py+++ /dev/null@@ -1,34 +0,0 @@-"""-Extract top domains from BigQuery result.-"""-import json-import os-import sys-from pathlib import Path--import numpy as np-import pandas as pd--def get_top_domains():- data = pd.read_csv(ALL_DOMAINS_PATH, index_col='domain')- data = data[data.index.notnull()]-- frequent = data[data['total'] >= MIN_COUNT]- scores = frequent['mean_score'] * np.log(frequent['total']) ** 2- median_score = np.median(scores)- print("Median score", median_score)- probabilities = scores / (scores + median_score)-- top_probabilities = probabilities[probabilities > PROBABILITY_THRESHOLD]- top_probabilities.sort_values(ascending=False, inplace=True)- with open(TOP_DOMAINS_PATH, 'w') as output_file:- probabilities_str = str(top_probabilities.to_dict()).replace(', ', ',\n')- output_file.write("DOMAINS = " + probabilities_str + '\n\n')- # json.dump(probabilities.to_dict(), output_file, indent=2)-- # for row in probabilities.iterrows():- # output_file.write(json.dumps(row.to_dict()) + '\n')---if __name__ == '__main__':- get_top_domains()diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.pyindex 6163974..daeac08 100644--- a/mwmbl/indexer/update_urls.py+++ b/mwmbl/indexer/update_urls.py@@ -78,9 +78,9 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qufor url in url_scores.keys() | url_statuses.keys()]logger.info(f"Found URLs, {len(found_urls)}")- urls = url_db.update_found_urls(found_urls)- new_item_queue.put(urls)- logger.info(f"Put {len(urls)} new items in the URL queue")+ new_urls = url_db.update_found_urls(found_urls)+ new_item_queue.put(new_urls)+ logger.info(f"Put {len(new_urls)} new items in the URL queue")def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multiplier, timestamp, url_scores, url_timestamps, url_users, is_extra: bool, blacklist_domains):diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex c8dc860..2f8875c 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -31,3 +31,6 @@ NUM_PAGES = 2560URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 100_000++DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")+NUM_DOMAINS_IN_BLOOM_FILTER = 100_000diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex bfbbbf8..30ca98d 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -35,3 +35,6 @@ NUM_PAGES = 10240000URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")NUM_URLS_IN_BLOOM_FILTER = 200_000_000++DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")+NUM_DOMAINS_IN_BLOOM_FILTER = 100_000_000diff --git a/requirements.txt b/requirements.txtnew file mode 100644index 0000000..ac919e5--- /dev/null+++ b/requirements.txt@@ -0,0 +1,16 @@+zstandard~=0.16.0+pandas~=1.5.3+django~=4.2.6+boto3~=1.28.62+requests~=2.31.0+fastapi~=0.70.1+redis~=5.0.1+pydantic~=1.8.2+uvicorn~=0.16.0+urllib3~=2.0.6+justext~=3.0.0+scikit-learn~=1.3.1+mmh3~=3.1.0+numpy~=1.26.0+scipy~=1.11.3+pybloomfilter3~=0.5.7