Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Metric: count seed sites discovered per user

blueheron786a040cc346989
diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.py
index bc41751..b8ca365 100644
--- a/mwmbl/indexer/seed_crawl.py
+++ b/mwmbl/indexer/seed_crawl.py
@@ -29,7 +29,7 @@ take minutes, and in series with the periodic tasks they would hold up the usage
the Polar report. One crawl runs at a time, each user has at most one
queued or running, and at most SEED_CRAWL_MAX_QUEUED are, so nobody can fill the queue.
-The new pages a search and its crawl added are counted on the user for their stats, and what it found on
+The new pages a crawl added are counted on the user for their stats, and what it found on
each of Staan's domains is scored for crawlers as seed domains - see
mwmbl.indexer.seed_domains.
"""
@@ -218,18 +218,9 @@ def crawl_round(frontier: Frontier, batch: list[str], redis) -> list[Document]:
return documents
-def credit_seed_search_pages(user_id: int, count: int) -> None:
- """Add pages a Seed Search or its crawl put into the index to the user's all-time total."""
- if count:
- MwmblUser.objects.filter(id=user_id).update(seed_search_pages_indexed=F("seed_search_pages_indexed") + count)
-
-
-def record_seed_search(user_id: int, pages_added: int) -> None:
- """Count a search made with Seed Search on, and the new pages it wrote to the index, for the user's stats."""
- MwmblUser.objects.filter(id=user_id).update(
- seed_searches=F("seed_searches") + 1,
- seed_search_pages_indexed=F("seed_search_pages_indexed") + pages_added,
- )
+def record_seed_search(user_id: int) -> None:
+ """Count a search made with Seed Search on, for the user's stats."""
+ MwmblUser.objects.filter(id=user_id).update(seed_searches=F("seed_searches") + 1)
def start_seed_crawl(
@@ -515,9 +506,10 @@ def _run_round(redis, record_key: str, token: int, fields: dict[str, str], index
return
logger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))
# After the round is recorded, so a round run twice is not counted twice.
- # Not the new seeds: the search that queued this crawl wrote them, and credited them then.
- crawled_pages = [document for document in new_documents if document.url not in new_seed_urls]
- credit_seed_search_pages(user_id, len(crawled_pages))
+ if new_documents:
+ MwmblUser.objects.filter(id=user_id).update(
+ seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)
+ )
def _finish(
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.py
index 1efe2cd..a8744b7 100644
--- a/mwmbl/platform/api.py
+++ b/mwmbl/platform/api.py
@@ -1213,8 +1213,8 @@ def get_current_user(request):
description=(
"Returns the authenticated user's contribution stats: how many results they "
"have indexed, broken down per day for the last 30 days, how many Seed Searches "
- "(Combined Search with `crawl=true`) they have made, and how many new pages those "
- "searches and their crawls have added to the index, both in all."
+ "(Combined Search with `crawl=true`) they have made, how many sites their seed crawls were the "
+ "first to find, and how many new pages those crawls have added to the index, all in all."
),
tags=["Users"],
)
@@ -1225,6 +1225,7 @@ def get_current_user_stats(request):
**stats,
"seed_search_pages_indexed": user.seed_search_pages_indexed,
"seed_searches": user.seed_searches,
+ "seed_sites_discovered": user.discovered_seed_domains.count(),
}
diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.py
index 1938fca..5cf3645 100644
--- a/mwmbl/platform/schemas.py
+++ b/mwmbl/platform/schemas.py
@@ -24,6 +24,7 @@ class UserStatsResponse(Schema):
results_indexed_daily: dict[str, int]
seed_search_pages_indexed: int
seed_searches: int
+ seed_sites_discovered: int
class SubscriptionResponse(Schema):
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.py
index 4d4755a..a7d37c6 100644
--- a/mwmbl/tinysearchengine/combined_search.py
+++ b/mwmbl/tinysearchengine/combined_search.py
@@ -310,7 +310,7 @@ def init_router(ranker) -> None:
crawl_outcome = None
active_crawl_query = None
if crawl:
- await sync_to_async(record_seed_search)(user.id, len(new_staan_urls))
+ await sync_to_async(record_seed_search)(user.id)
crawl_outcome = await sync_to_async(start_seed_crawl)(
user.id, q, retrieval.pages, staan_results, new_staan_urls
)
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
index b4724ca..3ee67a0 100644
--- a/test/test_combined_search.py
+++ b/test/test_combined_search.py
@@ -555,26 +555,24 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(
@pytest.mark.django_db
-def test_a_seed_search_counts_itself_and_its_pages_in_the_users_totals(
+def test_a_seed_search_counts_itself_in_the_users_total(
client, user, access_token, fresh_quota, redis_cache, crawl_sources
):
- crawl_sources(new_urls=("https://a.example/", "https://b.example/", "https://c.example/"))
+ crawl_sources()
_crawl(client, access_token)
user.refresh_from_db()
- assert user.seed_search_pages_indexed == 3
assert user.seed_searches == 1
@pytest.mark.django_db
-def test_a_search_without_the_crawl_flag_credits_nothing(client, user, access_token, fresh_quota, stub_sources):
+def test_a_search_without_the_crawl_flag_is_not_counted(client, user, access_token, fresh_quota, stub_sources):
stub_sources(new_urls=("https://a.example/",))
_get(client, access_token)
user.refresh_from_db()
- assert user.seed_search_pages_indexed == 0
assert user.seed_searches == 0
diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.py
index 4533922..6dd80a0 100644
--- a/test/test_seed_crawl.py
+++ b/test/test_seed_crawl.py
@@ -444,28 +444,30 @@ def test_new_pages_are_added_to_the_users_total(fake_web, redis_cache, user, ind
@pytest.mark.django_db
-def test_seeds_the_search_found_new_are_not_counted_again_by_the_crawl(fake_web, redis_cache, user, index_path):
+def test_the_users_total_matches_the_pages_the_crawl_lists(fake_web, redis_cache, user, index_path):
_index_staan_snippet_of_the_seed(index_path)
- _run(user, index_path, new_seed_urls=[SEED])
+ record = _run(user, index_path, new_seed_urls=[SEED])
user.refresh_from_db()
- assert SEED in [page["url"] for page in seed_crawl.get_seed_crawl(user.id, "tokio")["pages"]]
- assert user.seed_search_pages_indexed == 2
+ assert SEED in [page["url"] for page in record["pages"]]
+ assert user.seed_search_pages_indexed == len(record["pages"])
@pytest.mark.django_db
-def test_recording_a_seed_search_adds_to_the_users_totals(user):
- seed_crawl.record_seed_search(user.id, 3)
- seed_crawl.record_seed_search(user.id, 0)
+def test_recording_a_seed_search_adds_to_the_users_count(user):
+ seed_crawl.record_seed_search(user.id)
+ seed_crawl.record_seed_search(user.id)
user.refresh_from_db()
assert user.seed_searches == 2
- assert user.seed_search_pages_indexed == 3
@pytest.mark.django_db
-def test_the_users_stats_report_their_seed_search_total(user):
+def test_the_users_stats_report_their_seed_search_totals(user):
User.objects.filter(id=user.id).update(seed_search_pages_indexed=12, seed_searches=4)
+ SeedDomain.objects.create(domain="tokio.rs", discovered_by=user)
+ SeedDomain.objects.create(domain="smol.rs", discovered_by=user)
+ SeedDomain.objects.create(domain="example.com")
token = str(RefreshToken.for_user(user).access_token)
response = Client().get("/api/v1/platform/user/stats", HTTP_AUTHORIZATION=f"Bearer {token}")
@@ -473,6 +475,7 @@ def test_the_users_stats_report_their_seed_search_total(user):
assert response.status_code == 200
assert response.json()["seed_search_pages_indexed"] == 12
assert response.json()["seed_searches"] == 4
+ assert response.json()["seed_sites_discovered"] == 2
# ---------------------------------------------------------------------------