An open source, non-profit web search engine
Add domain stats
Daoud Clarkebce0c47f8a0a
diff --git a/analyse/analyse_crawled_links.py b/analyse/analyse_crawled_links.pyindex 775aba8..2de43e2 100644--- a/analyse/analyse_crawled_links.py+++ b/analyse/analyse_crawled_links.py@@ -1,6 +1,8 @@import globimport gzipimport json+from collections import Counter+from urllib.parse import urlparseimport djangofrom django.conf import settings@@ -10,27 +12,44 @@ django.setup()from mwmbl.crawler.batch import HashedBatch-CRAWL_GLOB = "./devdata/batches/**/*.json.gz"+CRAWL_GLOB = "./devdata/batches/**/2024-05-25/**/*.json.gz"def get_urls():for path in glob.glob(CRAWL_GLOB, recursive=True):+ print("Getting data from path", path)data = json.load(gzip.open(path))batch = HashedBatch.parse_obj(data)user = batch.user_id_hashfor item in batch.items:if item.content is not None:- for url in item.content.links:- yield user, url+ yield item.url, item.content.links, item.content.extra_linksdef run():- for user, url in get_urls():- print(user, url)--+ link_counts = Counter()+ source_counts = Counter()+ extra_counts = Counter()+ for source, urls, extra_urls in get_urls():+ source_domain = urlparse(source).netloc+ source_counts.update([source_domain])++ for url in urls:+ link_domain = urlparse(url).netloc+ link_counts.update([link_domain])++ for url in extra_urls:+ extra_domain = urlparse(url).netloc+ extra_counts.update([extra_domain])++ if len(link_counts) > 1000:+ break++ print("Source counts", source_counts.most_common())+ print("Link counts", link_counts.most_common())+ print("Extra counts", extra_counts.most_common())if __name__ == "__main__":run()diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 07c5ac0..11b7006 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -227,5 +227,6 @@ def get_batches_for_date(date_str):# Don't cache data from today since it may changedata = gzip.compress(json.dumps(result).encode('utf8'))upload(data, cache_filename)- print(f"Cached batches for {date_str} in {PUBLIC_URL_PREFIX}{cache_filename}")+ print(f"Cached batches for {date_str} in {PUBLIC_URL_PREFIX}{cache_filename}")+ print(f"Returning {len(result['batch_urls'])} batches for {date_str}")return resultdiff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 1e2dbf0..709dfda 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -3,13 +3,18 @@ from datetime import datetime, timedeltafrom glob import globfrom itertools import islicefrom logging import getLogger+from pathlib import Pathfrom urllib.parse import urlparse+import django+from django.conf import settingsfrom pydantic import BaseModelfrom redis import Redisfrom mwmbl.count_urls import get_countsfrom mwmbl.crawler.batch import HashedBatch+from mwmbl.crawler.urls import URLDatabase+from mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.update_urls import get_datetime_from_timestamplogger = getLogger(__name__)@@ -19,10 +24,21 @@ URL_HOUR_COUNT_KEY = "url-count-hour-{hour}"USERS_KEY = "users-{date}"USER_COUNT_KEY = "user-count-{date}"HOST_COUNT_KEY = "host-count-{date}"+HOST_COUNT_ALL_KEY = "host-count-all-{date}"+HOST_COUNT_LINK_KEY = "host-count-link-{date}"+HOST_COUNT_LINK_NEW_KEY = "host-count-link-new-{date}"SHORT_EXPIRE_SECONDS = 60 * 60 * 24LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30+class DomainStats(BaseModel):+ domain_name: str+ num_crawled: int+ num_successful: int+ num_links: int+ num_links_new: int++class MwmblStats(BaseModel):urls_crawled_today: inturls_crawled_daily: dict[str, int]@@ -30,9 +46,24 @@ class MwmblStats(BaseModel):users_crawled_daily: dict[str, int]top_users: dict[str, int]top_domains: dict[str, int]- urls_in_index_daily: dict[str, int]- domains_in_index_daily: dict[str, int]- results_in_index_daily: dict[str, int]+ domain_stats: list[DomainStats]+++# New stats we want per domain:+# - Number of results in index+# - Number of links to this domain in URL queue+# - Best score of links for this domain in URL queue+# - Number of URLs crawled for this domain today:+# - Total - done+# - Number of successes - done+# - Number of timeouts+# - Number of 404s+# - Number excluded by robots.txt+# - Number of other errors+# - Number of internal and external links to this domain crawled today+# - Number of links excluded because they've already been crawled+# - Number of external links extracted from this domain today+# -class StatsManager:@@ -44,7 +75,8 @@ class StatsManager:num_crawled_urls = sum(1 for item in hashed_batch.items if item.content is not None)- url_count_key = URL_DATE_COUNT_KEY.format(date=date_time.date())+ date = date_time.date()+ url_count_key = URL_DATE_COUNT_KEY.format(date=date)self.redis.incrby(url_count_key, num_crawled_urls)self.redis.expire(url_count_key, LONG_EXPIRE_SECONDS)@@ -54,25 +86,37 @@ class StatsManager:self.redis.incrby(hour_key, num_crawled_urls)self.redis.expire(hour_key, SHORT_EXPIRE_SECONDS)- users_key = USERS_KEY.format(date=date_time.date())+ users_key = USERS_KEY.format(date=date)self.redis.sadd(users_key, hashed_batch.user_id_hash)self.redis.expire(users_key, LONG_EXPIRE_SECONDS)- user_count_key = USER_COUNT_KEY.format(date=date_time.date())+ user_count_key = USER_COUNT_KEY.format(date=date)self.redis.zincrby(user_count_key, num_crawled_urls, hashed_batch.user_id_hash)self.redis.expire(user_count_key, SHORT_EXPIRE_SECONDS)- host_key = HOST_COUNT_KEY.format(date=date_time.date())- for item in hashed_batch.items:- if item.content is None:- continue+ host_key = HOST_COUNT_KEY.format(date=date)+ host_all_key = HOST_COUNT_ALL_KEY.format(date=date)+ with URLDatabase() as url_db:+ for item in hashed_batch.items:+ host = urlparse(item.url).netloc+ self.redis.zincrby(host_all_key, 1, host)++ if item.content is None:+ continue++ self.redis.zincrby(host_key, 1, host)++ for link in item.content.links:+ link_host = urlparse(link).netloc+ self.redis.zincrby(HOST_COUNT_LINK_KEY.format(date=date), 1, link_host)++ if link not in url_db:+ self.redis.zincrby(HOST_COUNT_LINK_NEW_KEY.format(date=date), 1, link_host)- host = urlparse(item.url).netloc- self.redis.zincrby(host_key, 1, host)self.redis.expire(host_key, SHORT_EXPIRE_SECONDS)def get_stats(self) -> MwmblStats:- date_time = datetime.now()+ date_time = datetime.utcnow()date = date_time.date()urls_crawled_daily = {}@@ -104,6 +148,20 @@ class StatsManager:host_key = HOST_COUNT_KEY.format(date=date_time.date())host_counts = self.redis.zrevrange(host_key, 0, 100, withscores=True)+ all_domain_stats = []+ for host, count in host_counts:+ num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)+ domain_stats = DomainStats(+ domain_name=host,+ num_crawled=num_crawled,+ num_successful=count,+ num_links=num_links,+ num_links_new=num_links_new,+ )+ all_domain_stats.append(domain_stats)+urls_crawled_today = list(urls_crawled_daily.values())[-1]index_stats = get_counts()return MwmblStats(@@ -113,6 +171,7 @@ class StatsManager:users_crawled_daily=users_crawled_daily,top_users=user_counts,top_domains=host_counts,+ domain_stats=all_domain_stats,**index_stats,)@@ -125,6 +184,7 @@ def get_test_batches():if __name__ == '__main__':+ django.setup()redis = Redis(host='localhost', port=6379, decode_responses=True)stats = StatsManager(redis)batches = get_test_batches()diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.pyindex c0fa0ba..5dfe3ed 100644--- a/mwmbl/crawler/urls.py+++ b/mwmbl/crawler/urls.py@@ -80,3 +80,6 @@ class URLDatabase:logger.info(f"Found {num_crawled_urls} crawled URLs and {len(new_urls)} new URLs")return new_urls++ def __contains__(self, url):+ return url in self.urls