Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add domain stats page

Daoud Clarke504efc6c3ecf
diff --git a/analyse/analyse_crawled_domains.py b/analyse/analyse_crawled_domains.py
index 5e87abb..28410bf 100644
--- a/analyse/analyse_crawled_domains.py
+++ b/analyse/analyse_crawled_domains.py
@@ -11,10 +11,6 @@ from mwmbl.crawler import HashedBatch
from mwmbl.indexer import CRAWL_GLOB, MWMBL_DATA_DIR
-# TODO: remove this line - temporary override
-CRAWL_GLOB = str(MWMBL_DATA_DIR / "b2") + "/*/*/2022-06-23/*/*/*.json.gz"
-
-
def get_urls():
for path in glob.glob(CRAWL_GLOB):
data = json.load(gzip.open(path))
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 709dfda..7a8bf86 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -46,7 +46,6 @@ class MwmblStats(BaseModel):
users_crawled_daily: dict[str, int]
top_users: dict[str, int]
top_domains: dict[str, int]
- domain_stats: list[DomainStats]
# New stats we want per domain:
@@ -114,6 +113,7 @@ class StatsManager:
self.redis.zincrby(HOST_COUNT_LINK_NEW_KEY.format(date=date), 1, link_host)
self.redis.expire(host_key, SHORT_EXPIRE_SECONDS)
+ self.redis.expire(host_all_key, SHORT_EXPIRE_SECONDS)
def get_stats(self) -> MwmblStats:
date_time = datetime.utcnow()
@@ -148,20 +148,6 @@ class StatsManager:
host_key = HOST_COUNT_KEY.format(date=date_time.date())
host_counts = self.redis.zrevrange(host_key, 0, 100, withscores=True)
- all_domain_stats = []
- for host, count in host_counts:
- num_crawled = self.redis.zscore(HOST_COUNT_ALL_KEY.format(date=date_time.date()), host)
- num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)
- num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)
- domain_stats = DomainStats(
- domain_name=host,
- num_crawled=num_crawled,
- num_successful=count,
- num_links=num_links,
- num_links_new=num_links_new,
- )
- all_domain_stats.append(domain_stats)
-
urls_crawled_today = list(urls_crawled_daily.values())[-1]
index_stats = get_counts()
return MwmblStats(
@@ -171,10 +157,28 @@ class StatsManager:
users_crawled_daily=users_crawled_daily,
top_users=user_counts,
top_domains=host_counts,
- domain_stats=all_domain_stats,
**index_stats,
)
+ def get_domain_stats(self) -> list[DomainStats]:
+ date_time = datetime.utcnow()
+ host_all_key = HOST_COUNT_ALL_KEY.format(date=date_time.date())
+ host_counts_all = self.redis.zrevrange(host_all_key, 0, 1000, withscores=True)
+ all_domain_stats = []
+ for host, count in host_counts_all:
+ num_successful = self.redis.zscore(HOST_COUNT_KEY.format(date=date_time.date()), host)
+ num_links = self.redis.zscore(HOST_COUNT_LINK_KEY.format(date=date_time.date()), host)
+ num_links_new = self.redis.zscore(HOST_COUNT_LINK_NEW_KEY.format(date=date_time.date()), host)
+ domain_stats = DomainStats(
+ domain_name=host,
+ num_crawled=count,
+ num_successful=num_successful or 0,
+ num_links=num_links or 0,
+ num_links_new=num_links_new or 0,
+ )
+ all_domain_stats.append(domain_stats)
+ return all_domain_stats
+
def get_test_batches():
for path in glob("./devdata/batches/**/*.json.gz", recursive=True):
diff --git a/mwmbl/templates/mwmbl/domains.html b/mwmbl/templates/mwmbl/domains.html
new file mode 100644
index 0000000..81eeabb
--- /dev/null
+++ b/mwmbl/templates/mwmbl/domains.html
@@ -0,0 +1,23 @@
+{% extends "base.html" %}
+
+{% block content %}
+
+ <table>
+ <thead>
+ <tr>
+ <th>Domain name</th><th>Crawled</th><th>Successful</th><th>Links</th><th>New links</th>
+ </tr>
+ </thead>
+ {% for domain in domain_stats %}
+ <tr>
+ <td>{{ domain.domain_name }}</td>
+ <td>{{ domain.num_crawled }}</td>
+ <td>{{ domain.num_successful }}</td>
+ <td>{{ domain.num_links }}</td>
+ <td>{{ domain.num_links_new }}</td>
+ </tr>
+ {% endfor %}
+ </table>
+
+
+{% endblock %}
\ No newline at end of file
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index beb8ca6..9f2aed0 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -21,7 +21,7 @@ import mwmbl.crawler.app as crawler
from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
from mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \
- flag_curation, CurationFlagListView, flag_curation_update
+ flag_curation, CurationFlagListView, flag_curation_update, domains_view
urlpatterns = [
path('admin/', admin.site.urls),
@@ -36,6 +36,7 @@ urlpatterns = [
path('app/curation/<int:curation_id>/flag/', flag_curation, name="flag_curation"),
path('app/flags/curation', CurationFlagListView.as_view(), name="flag_curation_list"),
path('app/flags/curation/<int:flag_curation_id>/update', flag_curation_update, name="flag_curation_update"),
+ path('app/domains/', domains_view, name="domains"),
# TODO: this is the old API, deprecated and to be removed once all clients have moved over
path("search/", search.create_router(ranker, "0.1").urls),
diff --git a/mwmbl/views.py b/mwmbl/views.py
index f537439..9f2fb14 100644
--- a/mwmbl/views.py
+++ b/mwmbl/views.py
@@ -19,6 +19,7 @@ from justext.core import html_to_dom, ParagraphMaker, classify_paragraphs, revis
STOPWORDS_HIGH_DEFAULT, MAX_HEADING_DISTANCE_DEFAULT, DEFAULT_ENCODING, DEFAULT_ENC_ERRORS, preprocessor
from requests.exceptions import RequestException
+from mwmbl.crawler.app import stats_manager
from mwmbl.models import Curation, FlagCuration
from mwmbl.search_setup import ranker, index_path
from mwmbl.settings import NUM_EXTRACT_CHARS
@@ -416,3 +417,8 @@ class CurationFlagListView(LoginRequiredMixin, ListView):
def get_queryset(self):
return FlagCuration.objects.filter(status="PENDING").order_by("-timestamp")
+
+
+def domains_view(request):
+ domain_stats = stats_manager.get_domain_stats()
+ return render(request, "mwmbl/domains.html", {"domain_stats": domain_stats})