An open source, non-profit web search engine
Add domain view
Daoud Clarke485b5332dae5
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 7248b1d..d9b9a97 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -182,6 +182,23 @@ class StatsManager:all_domain_stats.append(domain_stats)return all_domain_stats+ def get_stats_for_domain(self, host: str) -> DomainStats:+ date_time = datetime.utcnow()+ num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)+ num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=date_time.date()), host)+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)+ num_index_results = get_domain_result_count(host)+ domain_stats = DomainStats(+ domain_name=host,+ num_crawled=num_crawled or 0,+ num_successful=num_successful or 0,+ num_links=num_links or 0,+ num_links_new=num_links_new or 0,+ num_index_results=num_index_results,+ )+ return domain_stats+def get_test_batches():for path in glob("./devdata/batches/**/*.json.gz", recursive=True):diff --git a/mwmbl/templates/mwmbl/domain.html b/mwmbl/templates/mwmbl/domain.htmlnew file mode 100644index 0000000..b54157b--- /dev/null+++ b/mwmbl/templates/mwmbl/domain.html@@ -0,0 +1,13 @@+{% extends "base.html" %}++{% block content %}+<h1>Stats for {{ domain_stats.domain_name }}</h1>+<ul>+<li>Number of URLs crawled today: {{ domain_stats.num_crawled }}</li>+<li>Number successfully retrieved: {{ domain_stats.num_successful }}</li>+<li>Number of links extracted from this domain: {{ domain_stats.num_links }}</li>+<li>Number of links which are new: {{ domain_stats.num_links_new }}</li>+<li>Number of results for this domain in the index: {{ domain_stats.num_index_results }}</li>+</ul>++{% endblock content %}diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.htmlindex f02da29..4f0dcfc 100644--- a/mwmbl/templates/mwmbl/domains.html+++ b/mwmbl/templates/mwmbl/domains.html@@ -10,7 +10,7 @@</thead>{% for domain in domain_stats %}<tr>- <td>{{ domain.domain_name }}</td>+ <td><a href="{% url "domain" domain.domain_name %}">{{ domain.domain_name }}</a></td><td>{{ domain.num_crawled }}</td><td>{{ domain.num_successful }}</td><td>{{ domain.num_links }}</td>diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 9f2aed0..7445240 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -21,7 +21,7 @@ import mwmbl.crawler.app as crawlerfrom mwmbl.search_setup import queued_batches, ranker, batch_cachefrom mwmbl.tinysearchengine import searchfrom mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \- flag_curation, CurationFlagListView, flag_curation_update, domains_view+ flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_viewurlpatterns = [path('admin/', admin.site.urls),@@ -37,6 +37,7 @@ urlpatterns = [path('app/flags/curation', CurationFlagListView.as_view(), name="flag_curation_list"),path('app/flags/curation/<int:flag_curation_id>/update', flag_curation_update, name="flag_curation_update"),path('app/domains/', domains_view, name="domains"),+ path('app/domains/<str:domain>/', domain_view, name="domain"),# TODO: this is the old API, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),diff --git a/mwmbl/views.py b/mwmbl/views.pyindex 9f2fb14..d369d2f 100644--- a/mwmbl/views.py+++ b/mwmbl/views.py@@ -422,3 +422,8 @@ class CurationFlagListView(LoginRequiredMixin, ListView):def domains_view(request):domain_stats = stats_manager.get_domain_stats()return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})+++def domain_view(request, domain):+ domain_stats = stats_manager.get_stats_for_domain(domain)+ return render(request, "mwmbl/domain.html", {"domain_stats": domain_stats})