An open source, non-profit web search engine
Address review of crawler memberships
Claude0a1d5efe6322
diff --git a/mwmbl/admin.py b/mwmbl/admin.pyindex 6a482f8..10f3e6c 100644--- a/mwmbl/admin.py+++ b/mwmbl/admin.py@@ -98,8 +98,8 @@ admin.site.register(Membership, MembershipAdmin)class CrawlerMembershipAdmin(ModelAdmin):- list_display = ("user", "valid_until", "earned")- readonly_fields = ("earned",)+ list_display = ("user", "valid_until", "started")+ readonly_fields = ("started",)search_fields = ("user__username", "user__email")diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.pyindex 344df3f..4b555c1 100644--- a/mwmbl/crawler/stats.py+++ b/mwmbl/crawler/stats.py@@ -224,7 +224,8 @@ class StatsManager:date=today,num_results=num_results,)- update_crawler_membership(user, today)++ update_crawler_membership(user, today)def record_blacklisted_removed(self, num_results: int) -> None:"""Record documents removed from the index by the background blacklist purge.diff --git a/mwmbl/crawler_membership.py b/mwmbl/crawler_membership.pyindex 2b6503e..cc72e38 100644--- a/mwmbl/crawler_membership.py+++ b/mwmbl/crawler_membership.py@@ -18,13 +18,27 @@ def end_of_next_month(day: date) -> date:def update_crawler_membership(user: MwmblUser, today: date) -> None:"""Grant or extend the user's crawler membership if they have crawled on enough days this month.- Called when the user's first results of the day arrive, so the membership starts the moment- they reach the threshold.+ Called after every batch of results, so the membership starts the moment the user reaches the+ threshold, and a grant that failed is retried on their next batch. Once this month has earned+ the membership, it costs one query. A grant never shortens a membership, so a longer one set by+ hand in the admin survives."""- days_crawled = UserStats.objects.filter(user=user, date__gte=today.replace(day=1), date__lte=today).count()+ valid_until = end_of_next_month(today)+ if CrawlerMembership.objects.filter(user=user, valid_until__gte=valid_until).exists():+ return++ days_crawled = UserStats.objects.filter(+ user=user,+ date__gte=today.replace(day=1),+ date__lte=today,+ num_results__gte=settings.CRAWLER_MEMBERSHIP_MIN_RESULTS,+ ).count()if days_crawled < settings.CRAWLER_MEMBERSHIP_DAYS:return- CrawlerMembership.objects.update_or_create(user=user, defaults={"valid_until": end_of_next_month(today)})++ _, created = CrawlerMembership.objects.get_or_create(user=user, defaults={"valid_until": valid_until})+ if not created:+ CrawlerMembership.objects.filter(user=user, valid_until__lt=valid_until).update(valid_until=valid_until)def has_crawler_membership(user_id: int) -> bool:diff --git a/mwmbl/migrations/0045_crawlermembership.py b/mwmbl/migrations/0045_crawlermembership.pyindex 681f2e7..8945870 100644--- a/mwmbl/migrations/0045_crawlermembership.py+++ b/mwmbl/migrations/0045_crawlermembership.py@@ -14,7 +14,7 @@ class Migration(migrations.Migration):fields=[("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),("valid_until", models.DateField()),- ("earned", models.DateTimeField(auto_now=True)),+ ("started", models.DateTimeField(auto_now_add=True)),("user",models.OneToOneField(diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 0b72775..f1603ba 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -278,7 +278,7 @@ class CrawlerMembership(models.Model):user = models.OneToOneField(MwmblUser, on_delete=models.CASCADE, related_name="crawler_membership")valid_until = models.DateField()- earned = models.DateTimeField(auto_now=True)+ started = models.DateTimeField(auto_now_add=True)class AgreementType(models.TextChoices):diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.pyindex e431b1f..5e5646e 100644--- a/mwmbl/platform/api.py+++ b/mwmbl/platform/api.py@@ -26,7 +26,7 @@ from polar_sdk.webhooks import WebhookVerificationErrorfrom mwmbl import membership, pricingfrom mwmbl.background import enrich_domain_submission, stats_manager-from mwmbl.crawler_membership import membership_tier+from mwmbl.crawler_membership import has_crawler_membership, membership_tierfrom mwmbl.exceptions import InvalidRequestfrom mwmbl.membership import MembershipTierfrom mwmbl.models import (@@ -1454,9 +1454,7 @@ def create_membership_checkout(request, body: MembershipCheckoutRequest):)def cancel_membership(request):check_email_verified(request)- user_membership = Membership.objects.filter(user=request.user).first()- if user_membership is None:- raise InvalidRequest("Not a member.", status=404)+ user_membership = _paid_membership(request.user)if user_membership.cancel_at_period_end:raise InvalidRequest("Membership is already scheduled to cancel.", status=409)try:@@ -1482,9 +1480,7 @@ def cancel_membership(request):)def uncancel_membership(request):check_email_verified(request)- user_membership = Membership.objects.filter(user=request.user).first()- if user_membership is None:- raise InvalidRequest("Not a member.", status=404)+ user_membership = _paid_membership(request.user)if not user_membership.cancel_at_period_end:raise InvalidRequest("Membership is not scheduled to cancel.", status=409)try:@@ -1511,9 +1507,7 @@ def uncancel_membership(request):)def change_membership(request, body: MembershipChangeRequest):check_email_verified(request)- user_membership = Membership.objects.filter(user=request.user).first()- if user_membership is None:- raise InvalidRequest("Not a member.", status=404)+ user_membership = _paid_membership(request.user)if user_membership.tier == body.tier:raise InvalidRequest("Already on this tier.", status=409)# Changing tier would leave the cancellation in place, which is rarely what anyone wants;@@ -1540,6 +1534,19 @@ def change_membership(request, body: MembershipChangeRequest):return _save_membership_update(user_membership, tier=tier, current_period_end=result.current_period_end)+def _paid_membership(user) -> Membership:+ """The user's paid membership, for the endpoints that change it in Polar."""+ user_membership = Membership.objects.filter(user=user).first()+ if user_membership is not None:+ return user_membership+ if has_crawler_membership(user.id):+ raise InvalidRequest(+ "Memberships earned by crawling can't be cancelled or changed. Buy a membership to choose a tier.",+ status=409,+ )+ raise InvalidRequest("Not a member.", status=404)++def _save_membership_update(user_membership, **changes):"""Record a change made in Polar on the Membership row, unless a webhook changed it during the Polar call.diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex b50575d..982f29f 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -301,8 +301,10 @@ CURRENT_AGREEMENT_VERSIONS = {# meant to replace. Monthly limits for web (JWT) callers, keyed by# mwmbl.membership.MembershipTier value; "free" is signed-in users without a membership.COMBINED_SEARCH_MONTHLY_LIMITS = {"free": 30, "sprout": 300, "sapling": 1_500, "canopy": 1_500}-# Crawling on this many days in a calendar month earns a free Sprout membership.+# Crawling on this many days in a calendar month earns a free Sprout membership. A day counts+# once the user has submitted at least CRAWLER_MEMBERSHIP_MIN_RESULTS results that day.CRAWLER_MEMBERSHIP_DAYS = 3+CRAWLER_MEMBERSHIP_MIN_RESULTS = 100# Its own model artifact, so Combined Search can be retrained on the pooled candidate set# without moving standard search's ranking. Falls back to RUST_MODEL_PATH until trained.COMBINED_MODEL_PATH = Path(__file__).parent / "resources" / "model-combined.xgb"diff --git a/test/test_admin_paying_users.py b/test/test_admin_paying_users.pyindex d38df13..6254269 100644--- a/test/test_admin_paying_users.py+++ b/test/test_admin_paying_users.py@@ -6,8 +6,9 @@ from django.core.cache import cachefrom fakeredis import FakeConnectionfrom mwmbl import quota-from mwmbl.models import ApiKey, Membership, UserBilling+from mwmbl.models import ApiKey, CrawlerMembership, Membership, UserBillingfrom mwmbl.templatetags.money import minor_units+from mwmbl.utils import utc_todayUser = get_user_model()@@ -143,6 +144,18 @@ def test_seed_search_usage_by_tier(staff_client):assert [row["user"].username for row in seed_search["top_users"]] == ["free_user", "sprout_member"]+def test_seed_search_counts_crawler_members_as_sprout(staff_client):+ crawler = User.objects.create_user(username="crawler_member")+ CrawlerMembership.objects.create(user=crawler, valid_until=utc_today())+ quota.increment_monthly_combined_search(crawler.id)++ response = staff_client.get(PAYING_USERS_URL)++ by_tier = {tier["tier"]: tier for tier in response.context["seed_search"]["by_tier"]}+ assert by_tier["sprout"]["users"] == 1+ assert by_tier["free"]["users"] == 0++def test_seed_search_skips_deleted_users(staff_client):deleted = User.objects.create_user(username="deleted_user")quota.increment_monthly_combined_search(deleted.id)diff --git a/test/test_combined_search.py b/test/test_combined_search.pyindex 413c8b0..832c588 100644--- a/test/test_combined_search.py+++ b/test/test_combined_search.py@@ -24,7 +24,7 @@ import mwmbl.tinysearchengine.combined_search as combined_searchfrom mwmbl import pricingfrom mwmbl.indexer import seed_crawlfrom mwmbl.membership import MembershipTier-from mwmbl.models import ApiKey, Membership, UserBilling, generate_api_key+from mwmbl.models import ApiKey, CrawlerMembership, Membership, UserBilling, generate_api_keyfrom mwmbl.quota import (_combined_search_api_monthly_key,_combined_search_monthly_key,@@ -33,6 +33,7 @@ from mwmbl.quota import (get_monthly_count,)from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState, TinyIndex+from mwmbl.utils import utc_todayUser = get_user_model()@@ -287,6 +288,20 @@ def _join(user, tier):Membership.objects.create(user=user, tier=tier, polar_subscription_id=f"sub_{tier}")+def test_a_crawler_membership_gives_the_sprout_limit(client, user, access_token, fresh_quota, stub_sources):+ stub_sources()+ CrawlerMembership.objects.create(user=user, valid_until=utc_today())+ cache.set(_combined_search_monthly_key(user.id), 299, timeout=3600)++ last_allowed = _get(client, access_token)+ over_the_limit = _get(client, access_token)++ assert last_allowed.status_code == 200+ assert last_allowed.json()["monthly_limit"] == 300+ assert over_the_limit.status_code == 429++@pytest.mark.django_db@pytest.mark.parametrize("tier, limit",diff --git a/test/test_crawler_membership.py b/test/test_crawler_membership.pyindex c76dace..7692826 100644--- a/test/test_crawler_membership.py+++ b/test/test_crawler_membership.py@@ -6,6 +6,7 @@ from unittest.mock import patchimport fakeredisimport pytest+from allauth.account.models import EmailAddressfrom django.test import Clientfrom ninja_jwt.tokens import RefreshToken@@ -13,17 +14,28 @@ from mwmbl.crawler.stats import StatsManagerfrom mwmbl.crawler_membership import end_of_next_month, membership_tierfrom mwmbl.membership import MembershipTierfrom mwmbl.models import CrawlerMembership, Membership, MwmblUser-from mwmbl.utils import utc_todayTODAY = date(2026, 10, 17)+ENOUGH_RESULTS = 100@pytest.fixturedef user(db):- return MwmblUser.objects.create_user(username="crawler", email="[email protected]", password="testpass123")+ user = MwmblUser.objects.create_user(username="crawler", email="[email protected]", password="testpass123")+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)+ return user-def _crawl(user, day: date, num_results: int = 5) -> None:+def today():+ with (+ patch("mwmbl.crawler_membership.utc_today", return_value=TODAY),+ patch("mwmbl.platform.api.utc_today", return_value=TODAY),+ ):+ yield TODAY+++def _crawl(user, day: date, num_results: int = ENOUGH_RESULTS) -> None:results = SimpleNamespace(results=[object()] * num_results)with patch("mwmbl.crawler.stats.utc_today", return_value=day):StatsManager(fakeredis.FakeRedis()).record_results(results, user)@@ -63,6 +75,24 @@ def test_several_batches_on_one_day_count_as_one_day(user):assert not CrawlerMembership.objects.filter(user=user).exists()[email protected]("num_results", [0, ENOUGH_RESULTS - 1])+def test_days_with_too_few_results_do_not_count(user, num_results):+ _crawl(user, date(2026, 10, 1))+ _crawl(user, date(2026, 10, 2))+ _crawl(user, date(2026, 10, 3), num_results=num_results)++ assert not CrawlerMembership.objects.filter(user=user).exists()+++def test_membership_is_granted_when_a_day_reaches_the_minimum(user):+ _crawl(user, date(2026, 10, 1))+ _crawl(user, date(2026, 10, 2))+ _crawl(user, date(2026, 10, 3), num_results=ENOUGH_RESULTS - 1)+ _crawl(user, date(2026, 10, 3), num_results=1)++ assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 11, 30)++def test_days_in_earlier_months_do_not_count(user):_crawl(user, date(2026, 9, 29))_crawl(user, date(2026, 9, 30))@@ -80,28 +110,45 @@ def test_qualifying_again_extends_the_membership(user):assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 12, 31)-def test_crawler_membership_gives_sprout(user):- CrawlerMembership.objects.create(user=user, valid_until=TODAY)- with patch("mwmbl.crawler_membership.utc_today", return_value=TODAY):- assert membership_tier(user.id) == MembershipTier.SPROUT+def test_qualifying_does_not_shorten_a_longer_membership(user):+ CrawlerMembership.objects.create(user=user, valid_until=date(2027, 6, 30))+ for day in (1, 2, 3):+ _crawl(user, date(2026, 10, day))++ assert CrawlerMembership.objects.get(user=user).valid_until == date(2027, 6, 30)+++def test_a_failed_grant_is_retried_on_the_next_batch(user):+ _crawl(user, date(2026, 10, 1))+ _crawl(user, date(2026, 10, 2))+ with patch("mwmbl.crawler_membership.CrawlerMembership.objects.get_or_create", side_effect=RuntimeError):+ with pytest.raises(RuntimeError):+ _crawl(user, date(2026, 10, 3))+ _crawl(user, date(2026, 10, 3))-def test_expired_crawler_membership_gives_nothing(user):+ assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 11, 30)+++def test_crawler_membership_gives_sprout(user, today):+ CrawlerMembership.objects.create(user=user, valid_until=today)+ assert membership_tier(user.id) == MembershipTier.SPROUT+++def test_expired_crawler_membership_gives_nothing(user, today):CrawlerMembership.objects.create(user=user, valid_until=date(2026, 10, 16))- with patch("mwmbl.crawler_membership.utc_today", return_value=TODAY):- assert membership_tier(user.id) is None+ assert membership_tier(user.id) is None-def test_paid_membership_wins_over_crawler_membership(user):- CrawlerMembership.objects.create(user=user, valid_until=TODAY)+def test_paid_membership_wins_over_crawler_membership(user, today):+ CrawlerMembership.objects.create(user=user, valid_until=today)Membership.objects.create(user=user, tier=MembershipTier.SAPLING, polar_subscription_id="sub_1")assert membership_tier(user.id) == MembershipTier.SAPLING-def test_get_membership_returns_crawler_membership(user):+def test_get_membership_returns_crawler_membership(user, today):CrawlerMembership.objects.create(user=user, valid_until=date(2026, 11, 30))- with patch("mwmbl.platform.api.utc_today", return_value=TODAY):- response = Client().get("/api/v1/platform/membership", **auth_headers(user))+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))assert response.status_code == 200assert response.json() == {@@ -112,6 +159,13 @@ def test_get_membership_returns_crawler_membership(user):}+def test_get_membership_is_404_when_the_crawler_membership_has_expired(user, today):+ CrawlerMembership.objects.create(user=user, valid_until=date(2026, 10, 16))+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))++ assert response.status_code == 404++def test_get_membership_marks_paid_membership(user):Membership.objects.create(user=user, tier=MembershipTier.SAPLING, polar_subscription_id="sub_1")response = Client().get("/api/v1/platform/membership", **auth_headers(user))@@ -120,8 +174,30 @@ def test_get_membership_marks_paid_membership(user):assert response.json()["source"] == "paid"-def test_crawler_membership_sets_the_combined_search_limit(user):- CrawlerMembership.objects.create(user=user, valid_until=end_of_next_month(utc_today()))+def test_get_membership_prefers_the_paid_membership(user, today):+ CrawlerMembership.objects.create(user=user, valid_until=today)+ Membership.objects.create(user=user, tier=MembershipTier.CANOPY, polar_subscription_id="sub_1")+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))++ assert response.json()["tier"] == "canopy"+ assert response.json()["source"] == "paid"+++ "path, body",+ [("cancel", None), ("uncancel", None), ("change", {"tier": "sapling"})],+)+def test_crawler_memberships_cannot_be_cancelled_or_changed(user, today, path, body):+ CrawlerMembership.objects.create(user=user, valid_until=today)+ response = Client().post(+ f"/api/v1/platform/membership/{path}", body, content_type="application/json", **auth_headers(user)+ )++ assert response.status_code == 409+++def test_crawler_membership_sets_the_combined_search_limit(user, today):+ CrawlerMembership.objects.create(user=user, valid_until=today)with patch("mwmbl.platform.api.get_monthly_combined_search_count", return_value=0):response = Client().get("/api/v1/platform/combined-search/usage", **auth_headers(user))