Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add domain stats

Daoud Clarkebce0c47f8a0a
diff --git a/analyse/analyse_crawled_links.py b/analyse/analyse_crawled_links.py
index 775aba8..2de43e2 100644
--- a/analyse/analyse_crawled_links.py
+++ b/analyse/analyse_crawled_links.py
@@ -1,6 +1,8 @@
import glob
import gzip
import json
+from collections import Counter
+from urllib.parse import urlparse
import django
from django.conf import settings
@@ -10,27 +12,44 @@ django.setup()
from mwmbl.crawler.batch import HashedBatch
-CRAWL_GLOB = "./devdata/batches/**/*.json.gz"
+CRAWL_GLOB = "./devdata/batches/**/2024-05-25/**/*.json.gz"
def get_urls():
for path in glob.glob(CRAWL_GLOB, recursive=True):
+ print("Getting data from path", path)
data = json.load(gzip.open(path))
batch = HashedBatch.parse_obj(data)
user = batch.user_id_hash
for item in batch.items:
if item.content is not None:
- for url in item.content.links:
- yield user, url
+ yield item.url, item.content.links, item.content.extra_links
def run():
- for user, url in get_urls():
- print(user, url)
-
-
+ link_counts = Counter()
+ source_counts = Counter()
+ extra_counts = Counter()
+ for source, urls, extra_urls in get_urls():
+ source_domain = urlparse(source).netloc
+ source_counts.update([source_domain])
+
+ for url in urls:
+ link_domain = urlparse(url).netloc
+ link_counts.update([link_domain])
+
+ for url in extra_urls:
+ extra_domain = urlparse(url).netloc
+ extra_counts.update([extra_domain])
+
+ if len(link_counts) > 1000:
+ break
+
+ print("Source counts", source_counts.most_common())
+ print("Link counts", link_counts.most_common())
+ print("Extra counts", extra_counts.most_common())
if __name__ == "__main__":
run()
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 07c5ac0..11b7006 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -227,5 +227,6 @@ def get_batches_for_date(date_str):
# Don't cache data from today since it may change
data = gzip.compress(json.dumps(result).encode('utf8'))
upload(data, cache_filename)
- print(f"Cached batches for {date_str} in {PUBLIC_URL_PREFIX}{cache_filename}")
+ print(f"Cached batches for {date_str} in {PUBLIC_URL_PREFIX}{cache_filename}")
+ print(f"Returning {len(result['batch_urls'])} batches for {date_str}")
return result
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 1e2dbf0..709dfda 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -3,13 +3,18 @@ from datetime import datetime, timedelta
from glob import glob
from itertools import islice
from logging import getLogger
+from pathlib import Path
from urllib.parse import urlparse
+import django
+from django.conf import settings
from pydantic import BaseModel
from redis import Redis
from mwmbl.count_urls import get_counts
from mwmbl.crawler.batch import HashedBatch
+from mwmbl.crawler.urls import URLDatabase
+from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.update_urls import get_datetime_from_timestamp
logger = getLogger(__name__)
@@ -19,10 +24,21 @@ URL_HOUR_COUNT_KEY = "url-count-hour-{hour}"
USERS_KEY = "users-{date}"
USER_COUNT_KEY = "user-count-{date}"
HOST_COUNT_KEY = "host-count-{date}"
+HOST_COUNT_ALL_KEY = "host-count-all-{date}"
+HOST_COUNT_LINK_KEY = "host-count-link-{date}"
+HOST_COUNT_LINK_NEW_KEY = "host-count-link-new-{date}"
SHORT_EXPIRE_SECONDS = 60 * 60 * 24
LONG_EXPIRE_SECONDS = 60 * 60 * 24 * 30
+class DomainStats(BaseModel):
+ domain_name: str
+ num_crawled: int
+ num_successful: int
+ num_links: int
+ num_links_new: int
+
+
class MwmblStats(BaseModel):
urls_crawled_today: int
urls_crawled_daily: dict[str, int]
@@ -30,9 +46,24 @@ class MwmblStats(BaseModel):
users_crawled_daily: dict[str, int]
top_users: dict[str, int]
top_domains: dict[str, int]
- urls_in_index_daily: dict[str, int]
- domains_in_index_daily: dict[str, int]
- results_in_index_daily: dict[str, int]
+ domain_stats: list[DomainStats]
+
+
+# New stats we want per domain:
+# - Number of results in index
+# - Number of links to this domain in URL queue
+# - Best score of links for this domain in URL queue
+# - Number of URLs crawled for this domain today:
+# - Total - done
+# - Number of successes - done
+# - Number of timeouts
+# - Number of 404s
+# - Number excluded by robots.txt
+# - Number of other errors
+# - Number of internal and external links to this domain crawled today
+# - Number of links excluded because they've already been crawled
+# - Number of external links extracted from this domain today
+# -
class StatsManager:
@@ -44,7 +75,8 @@ class StatsManager:
num_crawled_urls = sum(1 for item in hashed_batch.items if item.content is not None)
- url_count_key = URL_DATE_COUNT_KEY.format(date=date_time.date())
+ date = date_time.date()
+ url_count_key = URL_DATE_COUNT_KEY.format(date=date)
self.redis.incrby(url_count_key, num_crawled_urls)
self.redis.expire(url_count_key, LONG_EXPIRE_SECONDS)
@@ -54,25 +86,37 @@ class StatsManager:
self.redis.incrby(hour_key, num_crawled_urls)
self.redis.expire(hour_key, SHORT_EXPIRE_SECONDS)
- users_key = USERS_KEY.format(date=date_time.date())
+ users_key = USERS_KEY.format(date=date)
self.redis.sadd(users_key, hashed_batch.user_id_hash)
self.redis.expire(users_key, LONG_EXPIRE_SECONDS)
- user_count_key = USER_COUNT_KEY.format(date=date_time.date())
+ user_count_key = USER_COUNT_KEY.format(date=date)
self.redis.zincrby(user_count_key, num_crawled_urls, hashed_batch.user_id_hash)
self.redis.expire(user_count_key, SHORT_EXPIRE_SECONDS)
- host_key = HOST_COUNT_KEY.format(date=date_time.date())
- for item in hashed_batch.items:
- if item.content is None:
- continue
+ host_key = HOST_COUNT_KEY.format(date=date)
+ host_all_key = HOST_COUNT_ALL_KEY.format(date=date)
+ with URLDatabase() as url_db:
+ for item in hashed_batch.items:
+ host = urlparse(item.url).netloc
+ self.redis.zincrby(host_all_key, 1, host)
+
+ if item.content is None:
+ continue
+
+ self.redis.zincrby(host_key, 1, host)
+
+ for link in item.content.links:
+ link_host = urlparse(link).netloc
+ self.redis.zincrby(HOST_COUNT_LINK_KEY.format(date=date), 1, link_host)
+
+ if link not in url_db:
+ self.redis.zincrby(HOST_COUNT_LINK_NEW_KEY.format(date=date), 1, link_host)
- host = urlparse(item.url).netloc
- self.redis.zincrby(host_key, 1, host)
self.redis.expire(host_key, SHORT_EXPIRE_SECONDS)
def get_stats(self) -> MwmblStats:
- date_time = datetime.now()
+ date_time = datetime.utcnow()
date = date_time.date()
urls_crawled_daily = {}
@@ -104,6 +148,20 @@ class StatsManager:
host_key = HOST_COUNT_KEY.format(date=date_time.date())
host_counts = self.redis.zrevrange(host_key, 0, 100, withscores=True)
+ all_domain_stats = []
+ for host, count in host_counts:
+ num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)
+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)
+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)
+ domain_stats = DomainStats(
+ domain_name=host,
+ num_crawled=num_crawled,
+ num_successful=count,
+ num_links=num_links,
+ num_links_new=num_links_new,
+ )
+ all_domain_stats.append(domain_stats)
+
urls_crawled_today = list(urls_crawled_daily.values())[-1]
index_stats = get_counts()
return MwmblStats(
@@ -113,6 +171,7 @@ class StatsManager:
users_crawled_daily=users_crawled_daily,
top_users=user_counts,
top_domains=host_counts,
+ domain_stats=all_domain_stats,
**index_stats,
)
@@ -125,6 +184,7 @@ def get_test_batches():
if __name__ == '__main__':
+ django.setup()
redis = Redis(host='localhost', port=6379, decode_responses=True)
stats = StatsManager(redis)
batches = get_test_batches()
diff --git a/mwmbl/crawler/urls.py b/mwmbl/crawler/urls.py
index c0fa0ba..5dfe3ed 100644
--- a/mwmbl/crawler/urls.py
+++ b/mwmbl/crawler/urls.py
@@ -80,3 +80,6 @@ class URLDatabase:
logger.info(f"Found {num_crawled_urls} crawled URLs and {len(new_urls)} new URLs")
return new_urls
+
+ def __contains__(self, url):
+ return url in self.urls