An open source, non-profit web search engine
Count unique domains instead of links
Daoud Clarke2fc999b40279
diff --git a/analyse/top_links.py b/analyse/top_links.pyindex 466d4a9..012c44f 100644--- a/analyse/top_links.py+++ b/analyse/top_links.py@@ -5,6 +5,7 @@ import globimport gzipimport jsonfrom collections import defaultdict+from urllib.parse import urlparsefrom analyse.analyse_crawled_domains import CRAWL_GLOB@@ -14,8 +15,9 @@ def get_urls():data = json.load(gzip.open(path))for item in data['items']:url = item['url']+ domain = urlparse(url).hostnamefor link in item['links']:- yield url, link+ yield domain, linkdef collect_links(urls):@@ -28,7 +30,7 @@ def collect_links(urls):def run():url_links = get_urls()collected = collect_links(url_links)- top = sorted(collected.items(), key=lambda x: len(x[1]), reverse=True)[:100]+ top = sorted(collected.items(), key=lambda x: len(x[1]), reverse=True)[:1000]for url, items in top:print("URL", url, len(items))