Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Switch to more efficient unique-user count

blueheron7862f91ff6a85e0
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index c226bbf..93cb5a0 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -36,6 +36,7 @@ class DomainStats(BaseModel):
class MwmblStats(BaseModel):
users_crawled_daily: dict[str, int]
+ users_crawled_30d: int
results_indexed_daily: dict[str, int]
top_user_results: list[tuple[str, int]]
urls_in_index_daily: dict[str, int]
@@ -77,6 +78,11 @@ class StatsManager:
dataset_queries_daily = {}
dataset_results_daily = {}
blacklisted_results_removed_daily = {}
+
+ # Collect all unique users over 30 days for users_crawled_30d
+ keys = [USERS_KEY.format(date=date - timedelta(days=i)) for i in range(30)]
+ all_users = self.redis.sunion(keys)
+
for i in range(29, -1, -1):
date_i = date - timedelta(days=i)
@@ -115,6 +121,7 @@ class StatsManager:
return MwmblStats(
users_crawled_daily=users_crawled_daily,
+ users_crawled_30d=len(all_users),
results_indexed_daily=results_indexed_daily,
top_user_results=user_results_counts,
dataset_queries_daily=dataset_queries_daily,