An open source, non-profit web search engine
Add domain stats page
Daoud Clarke504efc6c3ecf
diff --git a/analyse/analyse_crawled_domains.py b/analyse/analyse_crawled_domains.pyindex 5e87abb..28410bf 100644--- a/analyse/analyse_crawled_domains.py+++ b/analyse/analyse_crawled_domains.py@@ -11,10 +11,6 @@ from mwmbl.crawler import HashedBatchfrom mwmbl.indexer import CRAWL_GLOB, MWMBL_DATA_DIR-# TODO: remove this line - temporary override-CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/2022-06-23/*/*/*.json.gz"--def get_urls():for path in glob.glob(CRAWL_GLOB):data = json.load(gzip.open(path))diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 709dfda..7a8bf86 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -46,7 +46,6 @@ class MwmblStats(BaseModel):users_crawled_daily: dict[str, int]top_users: dict[str, int]top_domains: dict[str, int]- domain_stats: list[DomainStats]# New stats we want per domain:@@ -114,6 +113,7 @@ class StatsManager:self.redis.zincrby(HOST_COUNT_LINK_NEW_KEY.format(date=date), 1, link_host)self.redis.expire(host_key, SHORT_EXPIRE_SECONDS)+ self.redis.expire(host_all_key, SHORT_EXPIRE_SECONDS)def get_stats(self) -> MwmblStats:date_time = datetime.utcnow()@@ -148,20 +148,6 @@ class StatsManager:host_key = HOST_COUNT_KEY.format(date=date_time.date())host_counts = self.redis.zrevrange(host_key, 0, 100, withscores=True)- all_domain_stats = []- for host, count in host_counts:- num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)- domain_stats = DomainStats(- domain_name=host,- num_crawled=num_crawled,- num_successful=count,- num_links=num_links,- num_links_new=num_links_new,- )- all_domain_stats.append(domain_stats)-urls_crawled_today = list(urls_crawled_daily.values())[-1]index_stats = get_counts()return MwmblStats(@@ -171,10 +157,28 @@ class StatsManager:users_crawled_daily=users_crawled_daily,top_users=user_counts,top_domains=host_counts,- domain_stats=all_domain_stats,**index_stats,)+ def get_domain_stats(self) -> list[DomainStats]:+ date_time = datetime.utcnow()+ host_all_key = HOST_COUNT_ALL_KEY.format(date=date_time.date())+ host_counts_all = self.redis.zrevrange(host_all_key, 0, 1000, withscores=True)+ all_domain_stats = []+ for host, count in host_counts_all:+ num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=date_time.date()), host)+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)+ domain_stats = DomainStats(+ domain_name=host,+ num_crawled=count,+ num_successful=num_successful or 0,+ num_links=num_links or 0,+ num_links_new=num_links_new or 0,+ )+ all_domain_stats.append(domain_stats)+ return all_domain_stats+def get_test_batches():for path in glob("./devdata/batches/**/*.json.gz", recursive=True):diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.htmlnew file mode 100644index 0000000..81eeabb--- /dev/null+++ b/mwmbl/templates/mwmbl/domains.html@@ -0,0 +1,23 @@+{% extends "base.html" %}++{% block content %}++ <table>+ <thead>+ <tr>+ <th>Domain name</th><th>Crawled</th><th>Successful</th><th>Links</th><th>New links</th>+ </tr>+ </thead>+ {% for domain in domain_stats %}+ <tr>+ <td>{{ domain.domain_name }}</td>+ <td>{{ domain.num_crawled }}</td>+ <td>{{ domain.num_successful }}</td>+ <td>{{ domain.num_links }}</td>+ <td>{{ domain.num_links_new }}</td>+ </tr>+ {% endfor %}+ </table>+++{% endblock %}\ No newline at end of filediff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex beb8ca6..9f2aed0 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -21,7 +21,7 @@ import mwmbl.crawler.app as crawlerfrom mwmbl.search_setup import queued_batches, ranker, batch_cachefrom mwmbl.tinysearchengine import searchfrom mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \- flag_curation, CurationFlagListView, flag_curation_update+ flag_curation, CurationFlagListView, flag_curation_update, domains_viewurlpatterns = [path('admin/', admin.site.urls),@@ -36,6 +36,7 @@ urlpatterns = [path('app/curation/<int:curation_id>/flag/', flag_curation, name="flag_curation"),path('app/flags/curation', CurationFlagListView.as_view(), name="flag_curation_list"),path('app/flags/curation/<int:flag_curation_id>/update', flag_curation_update, name="flag_curation_update"),+ path('app/domains/', domains_view, name="domains"),# TODO: this is the old API, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),diff --git a/mwmbl/views.py b/mwmbl/views.pyindex f537439..9f2fb14 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -19,6 +19,7 @@ from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revisSTOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessorfrom requests.exceptions import RequestException+from mwmbl.crawler.app import stats_managerfrom mwmbl.models import Curation, FlagCurationfrom mwmbl.search_setup import ranker, index_pathfrom mwmbl.settings import NUM_EXTRACT_CHARS@@ -416,3 +417,8 @@ class CurationFlagListView(LoginRequiredMixin, ListView):def get_queryset(self):return FlagCuration.objects.filter(status="PENDING").order_by("-timestamp")+++def domains_view(request):+ domain_stats = stats_manager.get_domain_stats()+ return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})