An open source, non-profit web search engine
Metric: count seed sites discovered per user
blueheron786a040cc346989
diff --git a/mwmbl/indexer/seed_crawl.py b/mwmbl/indexer/seed_crawl.pyindex bc41751..b8ca365 100644--- a/mwmbl/indexer/seed_crawl.py+++ b/mwmbl/indexer/seed_crawl.py@@ -29,7 +29,7 @@ take minutes, and in series with the periodic tasks they would hold up the usagethe Polar report. One crawl runs at a time, each user has at most onequeued or running, and at most SEED_CRAWL_MAX_QUEUED are, so nobody can fill the queue.-The new pages a search and its crawl added are counted on the user for their stats, and what it found on+The new pages a crawl added are counted on the user for their stats, and what it found oneach of Staan's domains is scored for crawlers as seed domains - seemwmbl.indexer.seed_domains."""@@ -218,18 +218,9 @@ def crawl_round(frontier: Frontier, batch: list[str], redis) -> list[Document]:return documents-def credit_seed_search_pages(user_id: int, count: int) -> None:- """Add pages a Seed Search or its crawl put into the index to the user's all-time total."""- if count:- MwmblUser.objects.filter(id=user_id).update(seed_search_pages_indexed=F("seed_search_pages_indexed") + count)---def record_seed_search(user_id: int, pages_added: int) -> None:- """Count a search made with Seed Search on, and the new pages it wrote to the index, for the user's stats."""- MwmblUser.objects.filter(id=user_id).update(- seed_searches=F("seed_searches") + 1,- seed_search_pages_indexed=F("seed_search_pages_indexed") + pages_added,- )+def record_seed_search(user_id: int) -> None:+ """Count a search made with Seed Search on, for the user's stats."""+ MwmblUser.objects.filter(id=user_id).update(seed_searches=F("seed_searches") + 1)def start_seed_crawl(@@ -515,9 +506,10 @@ def _run_round(redis, record_key: str, token: int, fields: dict[str, str], indexreturnlogger.info("Seed crawl for user %d added %d new pages", user_id, len(new_documents))# After the round is recorded, so a round run twice is not counted twice.- # Not the new seeds: the search that queued this crawl wrote them, and credited them then.- crawled_pages = [document for document in new_documents if document.url not in new_seed_urls]- credit_seed_search_pages(user_id, len(crawled_pages))+ if new_documents:+ MwmblUser.objects.filter(id=user_id).update(+ seed_search_pages_indexed=F("seed_search_pages_indexed") + len(new_documents)+ )def _finish(diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.pyindex 1efe2cd..a8744b7 100644--- a/mwmbl/platform/api.py+++ b/mwmbl/platform/api.py@@ -1213,8 +1213,8 @@ def get_current_user(request):description=("Returns the authenticated user's contribution stats: how many results they ""have indexed, broken down per day for the last 30 days, how many Seed Searches "- "(Combined Search with `crawl=true`) they have made, and how many new pages those "- "searches and their crawls have added to the index, both in all."+ "(Combined Search with `crawl=true`) they have made, how many sites their seed crawls were the "+ "first to find, and how many new pages those crawls have added to the index, all in all."),tags=["Users"],)@@ -1225,6 +1225,7 @@ def get_current_user_stats(request):**stats,"seed_search_pages_indexed": user.seed_search_pages_indexed,"seed_searches": user.seed_searches,+ "seed_sites_discovered": user.discovered_seed_domains.count(),}diff --git a/mwmbl/platform/schemas.py b/mwmbl/platform/schemas.pyindex 1938fca..5cf3645 100644--- a/mwmbl/platform/schemas.py+++ b/mwmbl/platform/schemas.py@@ -24,6 +24,7 @@ class UserStatsResponse(Schema):results_indexed_daily: dict[str, int]seed_search_pages_indexed: intseed_searches: int+ seed_sites_discovered: intclass SubscriptionResponse(Schema):diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.pyindex 4d4755a..a7d37c6 100644--- a/mwmbl/tinysearchengine/combined_search.py+++ b/mwmbl/tinysearchengine/combined_search.py@@ -310,7 +310,7 @@ def init_router(ranker) -> None:crawl_outcome = Noneactive_crawl_query = Noneif crawl:- await sync_to_async(record_seed_search)(user.id, len(new_staan_urls))+ await sync_to_async(record_seed_search)(user.id)crawl_outcome = await sync_to_async(start_seed_crawl)(user.id, q, retrieval.pages, staan_results, new_staan_urls)diff --git a/test/test_combined_search.py b/test/test_combined_search.pyindex b4724ca..3ee67a0 100644--- a/test/test_combined_search.py+++ b/test/test_combined_search.py@@ -555,26 +555,24 @@ def test_crawl_schedules_a_seed_crawl_of_what_the_index_lacked(@pytest.mark.django_db-def test_a_seed_search_counts_itself_and_its_pages_in_the_users_totals(+def test_a_seed_search_counts_itself_in_the_users_total(client, user, access_token, fresh_quota, redis_cache, crawl_sources):- crawl_sources(new_urls=("https://a.example/", "https://b.example/", "https://c.example/"))+ crawl_sources()_crawl(client, access_token)user.refresh_from_db()- assert user.seed_search_pages_indexed == 3assert user.seed_searches == 1@pytest.mark.django_db-def test_a_search_without_the_crawl_flag_credits_nothing(client, user, access_token, fresh_quota, stub_sources):+def test_a_search_without_the_crawl_flag_is_not_counted(client, user, access_token, fresh_quota, stub_sources):stub_sources(new_urls=("https://a.example/",))_get(client, access_token)user.refresh_from_db()- assert user.seed_search_pages_indexed == 0assert user.seed_searches == 0diff --git a/test/test_seed_crawl.py b/test/test_seed_crawl.pyindex 4533922..6dd80a0 100644--- a/test/test_seed_crawl.py+++ b/test/test_seed_crawl.py@@ -444,28 +444,30 @@ def test_new_pages_are_added_to_the_users_total(fake_web, redis_cache, user, ind@pytest.mark.django_db-def test_seeds_the_search_found_new_are_not_counted_again_by_the_crawl(fake_web, redis_cache, user, index_path):+def test_the_users_total_matches_the_pages_the_crawl_lists(fake_web, redis_cache, user, index_path):_index_staan_snippet_of_the_seed(index_path)- _run(user, index_path, new_seed_urls=[SEED])+ record = _run(user, index_path, new_seed_urls=[SEED])user.refresh_from_db()- assert SEED in [page["url"] for page in seed_crawl.get_seed_crawl(user.id, "tokio")["pages"]]- assert user.seed_search_pages_indexed == 2+ assert SEED in [page["url"] for page in record["pages"]]+ assert user.seed_search_pages_indexed == len(record["pages"])@pytest.mark.django_db-def test_recording_a_seed_search_adds_to_the_users_totals(user):- seed_crawl.record_seed_search(user.id, 3)- seed_crawl.record_seed_search(user.id, 0)+def test_recording_a_seed_search_adds_to_the_users_count(user):+ seed_crawl.record_seed_search(user.id)+ seed_crawl.record_seed_search(user.id)user.refresh_from_db()assert user.seed_searches == 2- assert user.seed_search_pages_indexed == 3@pytest.mark.django_db-def test_the_users_stats_report_their_seed_search_total(user):+def test_the_users_stats_report_their_seed_search_totals(user):User.objects.filter(id=user.id).update(seed_search_pages_indexed=12, seed_searches=4)+ SeedDomain.objects.create(domain="tokio.rs", discovered_by=user)+ SeedDomain.objects.create(domain="smol.rs", discovered_by=user)+ SeedDomain.objects.create(domain="example.com")token = str(RefreshToken.for_user(user).access_token)response = Client().get("/api/v1/platform/user/stats", HTTP_AUTHORIZATION=f"Bearer {token}")@@ -473,6 +475,7 @@ def test_the_users_stats_report_their_seed_search_total(user):assert response.status_code == 200assert response.json()["seed_search_pages_indexed"] == 12assert response.json()["seed_searches"] == 4+ assert response.json()["seed_sites_discovered"] == 2# ---------------------------------------------------------------------------