Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add domain view

Daoud Clarke485b5332dae5
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 7248b1d..d9b9a97 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -182,6 +182,23 @@ class StatsManager:
all_domain_stats.append(domain_stats)
return all_domain_stats
+ def get_stats_for_domain(self, host: str) -> DomainStats:
+ date_time = datetime.utcnow()
+ num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)
+ num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=date_time.date()), host)
+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)
+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)
+ num_index_results = get_domain_result_count(host)
+ domain_stats = DomainStats(
+ domain_name=host,
+ num_crawled=num_crawled or 0,
+ num_successful=num_successful or 0,
+ num_links=num_links or 0,
+ num_links_new=num_links_new or 0,
+ num_index_results=num_index_results,
+ )
+ return domain_stats
+
def get_test_batches():
for path in glob("./devdata/batches/**/*.json.gz", recursive=True):
diff --git a/mwmbl/templates/mwmbl/domain.html b/mwmbl/templates/mwmbl/domain.html
new file mode 100644
index 0000000..b54157b
--- /dev/null
+++ b/mwmbl/templates/mwmbl/domain.html
@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+
+{% block content %}
+<h1>Stats for {{ domain_stats.domain_name }}</h1>
+<ul>
+<li>Number of URLs crawled today: {{ domain_stats.num_crawled }}</li>
+<li>Number successfully retrieved: {{ domain_stats.num_successful }}</li>
+<li>Number of links extracted from this domain: {{ domain_stats.num_links }}</li>
+<li>Number of links which are new: {{ domain_stats.num_links_new }}</li>
+<li>Number of results for this domain in the index: {{ domain_stats.num_index_results }}</li>
+</ul>
+
+{% endblock content %}
diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.html
index f02da29..4f0dcfc 100644
--- a/mwmbl/templates/mwmbl/domains.html
+++ b/mwmbl/templates/mwmbl/domains.html
@@ -10,7 +10,7 @@
</thead>
{% for domain in domain_stats %}
<tr>
- <td>{{ domain.domain_name }}</td>
+ <td><a href="{% url "domain" domain.domain_name %}">{{ domain.domain_name }}</a></td>
<td>{{ domain.num_crawled }}</td>
<td>{{ domain.num_successful }}</td>
<td>{{ domain.num_links }}</td>
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 9f2aed0..7445240 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -21,7 +21,7 @@ import mwmbl.crawler.app as crawler
from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
from mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \
- flag_curation, CurationFlagListView, flag_curation_update, domains_view
+ flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view
urlpatterns = [
path('admin/', admin.site.urls),
@@ -37,6 +37,7 @@ urlpatterns = [
path('app/flags/curation', CurationFlagListView.as_view(), name="flag_curation_list"),
path('app/flags/curation/<int:flag_curation_id>/update', flag_curation_update, name="flag_curation_update"),
path('app/domains/', domains_view, name="domains"),
+ path('app/domains/<str:domain>/', domain_view, name="domain"),
# TODO: this is the old API, deprecated and to be removed once all clients have moved over
path("search/", search.create_router(ranker, "0.1").urls),
diff --git a/mwmbl/views.py b/mwmbl/views.py
index 9f2fb14..d369d2f 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -422,3 +422,8 @@ class CurationFlagListView(LoginRequiredMixin, ListView):
def domains_view(request):
domain_stats = stats_manager.get_domain_stats()
return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})
+
+
+def domain_view(request, domain):
+ domain_stats = stats_manager.get_stats_for_domain(domain)
+ return render(request, "mwmbl/domain.html", {"domain_stats": domain_stats})