Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

WIP: track links from top domains

Daoud Clarkeea31d5204b37
diff --git a/mwmbl/crawler/domains.py b/mwmbl/crawler/domains.py
new file mode 100644
index 0000000..54b79b9
--- /dev/null
+++ b/mwmbl/crawler/domains.py
@@ -0,0 +1,62 @@
+"""
+Record which source domains link to destination domains. Each source domain is a bloom filter containing sets of
+destination domains.
+"""
+from itertools import islice
+
+from django.conf import settings
+from pybloomfilter import BloomFilter
+
+from mwmbl.hn_top_domains_filtered import DOMAINS
+
+URL_GROUPS = [
+ 'github.com',
+ 'en.wikipedia.org',
+ 'news.ycombinator.com',
+ 'lemmy.ml',
+ 'mastodon.social',
+ 'top',
+ 'other',
+]
+
+TOP_DOMAINS = set(islice(DOMAINS, 1000))
+OTHER_DOMAINS = set(islice(DOMAINS, 1000, 10000))
+
+
+def get_bloom_filter(url_group: str) -> BloomFilter:
+ try:
+ bloom_filter = BloomFilter.open(settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))
+ except FileNotFoundError:
+ bloom_filter = BloomFilter(settings.NUM_DOMAINS_IN_BLOOM_FILTER, 1e-6,
+ settings.DOMAIN_LINKS_BLOOM_FILTER_PATH.format(url_group=url_group))
+ return bloom_filter
+
+
+class DomainLinkDatabase:
+ def __init__(self):
+ self.links = {}
+
+ def __enter__(self):
+ self.links = {url_group: get_bloom_filter(url_group) for url_group in URL_GROUPS}
+
+ def __exit__(self, exc_type, exc_val, exc_tb):
+ for bloom_filter in self.links.values():
+ bloom_filter.close()
+
+ def update_domain_links(self, source: str, target: list[str]):
+ if source in TOP_DOMAINS:
+ url_group = 'top'
+ elif source in OTHER_DOMAINS:
+ url_group = 'other'
+ elif source in URL_GROUPS:
+ url_group = source
+ else:
+ # This is a URL that we don't care about
+ return
+
+ bloom_filter = self.links[url_group]
+ bloom_filter.update(target)
+
+
+
+
diff --git a/mwmbl/indexer/domains.py b/mwmbl/indexer/domains.py
deleted file mode 100644
index 242df76..0000000
--- a/mwmbl/indexer/domains.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""
-Extract top domains from BigQuery result.
-"""
-import json
-import os
-import sys
-from pathlib import Path
-
-import numpy as np
-import pandas as pd
-
-def get_top_domains():
- data = pd.read_csv(ALL_DOMAINS_PATH, index_col='domain')
- data = data[data.index.notnull()]
-
- frequent = data[data['total'] >= MIN_COUNT]
- scores = frequent['mean_score'] * np.log(frequent['total']) ** 2
- median_score = np.median(scores)
- print("Median score", median_score)
- probabilities = scores / (scores + median_score)
-
- top_probabilities = probabilities[probabilities > PROBABILITY_THRESHOLD]
- top_probabilities.sort_values(ascending=False, inplace=True)
- with open(TOP_DOMAINS_PATH, 'w') as output_file:
- probabilities_str = str(top_probabilities.to_dict()).replace(', ', ',\n')
- output_file.write("DOMAINS = " + probabilities_str + '\n\n')
- # json.dump(probabilities.to_dict(), output_file, indent=2)
-
- # for row in probabilities.iterrows():
- # output_file.write(json.dumps(row.to_dict()) + '\n')
-
-
-if __name__ == '__main__':
- get_top_domains()
diff --git a/mwmbl/indexer/update_urls.py b/mwmbl/indexer/update_urls.py
index 6163974..daeac08 100644
--- a/mwmbl/indexer/update_urls.py
+++ b/mwmbl/indexer/update_urls.py
@@ -78,9 +78,9 @@ def record_urls_in_database(batches: Collection[HashedBatch], new_item_queue: Qu
for url in url_scores.keys() | url_statuses.keys()]
logger.info(f"Found URLs, {len(found_urls)}")
- urls = url_db.update_found_urls(found_urls)
- new_item_queue.put(urls)
- logger.info(f"Put {len(urls)} new items in the URL queue")
+ new_urls = url_db.update_found_urls(found_urls)
+ new_item_queue.put(new_urls)
+ logger.info(f"Put {len(new_urls)} new items in the URL queue")
def process_link(user_id_hash, crawled_page_domain, link, unknown_domain_multiplier, timestamp, url_scores, url_timestamps, url_users, is_extra: bool, blacklist_domains):
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index c8dc860..2f8875c 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -31,3 +31,6 @@ NUM_PAGES = 2560
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 100_000
+
+DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")
+NUM_DOMAINS_IN_BLOOM_FILTER = 100_000
diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.py
index bfbbbf8..30ca98d 100644
--- a/mwmbl/settings_prod.py
+++ b/mwmbl/settings_prod.py
@@ -35,3 +35,6 @@ NUM_PAGES = 10240000
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls.bloom")
NUM_URLS_IN_BLOOM_FILTER = 200_000_000
+
+DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain}.bloom")
+NUM_DOMAINS_IN_BLOOM_FILTER = 100_000_000
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..ac919e5
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,16 @@
+zstandard~=0.16.0
+pandas~=1.5.3
+django~=4.2.6
+boto3~=1.28.62
+requests~=2.31.0
+fastapi~=0.70.1
+redis~=5.0.1
+pydantic~=1.8.2
+uvicorn~=0.16.0
+urllib3~=2.0.6
+justext~=3.0.0
+scikit-learn~=1.3.1
+mmh3~=3.1.0
+numpy~=1.26.0
+scipy~=1.11.3
+pybloomfilter3~=0.5.7