Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Address review of crawler memberships

Claude0a1d5efe6322
diff --git a/mwmbl/admin.py b/mwmbl/admin.py
index 6a482f8..10f3e6c 100644
--- a/mwmbl/admin.py
+++ b/mwmbl/admin.py
@@ -98,8 +98,8 @@ admin.site.register(Membership, MembershipAdmin)
class CrawlerMembershipAdmin(ModelAdmin):
- list_display = ("user", "valid_until", "earned")
- readonly_fields = ("earned",)
+ list_display = ("user", "valid_until", "started")
+ readonly_fields = ("started",)
search_fields = ("user__username", "user__email")
diff --git a/mwmbl/crawler/stats.py b/mwmbl/crawler/stats.py
index 344df3f..4b555c1 100644
--- a/mwmbl/crawler/stats.py
+++ b/mwmbl/crawler/stats.py
@@ -224,7 +224,8 @@ class StatsManager:
date=today,
num_results=num_results,
)
- update_crawler_membership(user, today)
+
+ update_crawler_membership(user, today)
def record_blacklisted_removed(self, num_results: int) -> None:
"""Record documents removed from the index by the background blacklist purge.
diff --git a/mwmbl/crawler_membership.py b/mwmbl/crawler_membership.py
index 2b6503e..cc72e38 100644
--- a/mwmbl/crawler_membership.py
+++ b/mwmbl/crawler_membership.py
@@ -18,13 +18,27 @@ def end_of_next_month(day: date) -> date:
def update_crawler_membership(user: MwmblUser, today: date) -> None:
"""Grant or extend the user's crawler membership if they have crawled on enough days this month.
- Called when the user's first results of the day arrive, so the membership starts the moment
- they reach the threshold.
+ Called after every batch of results, so the membership starts the moment the user reaches the
+ threshold, and a grant that failed is retried on their next batch. Once this month has earned
+ the membership, it costs one query. A grant never shortens a membership, so a longer one set by
+ hand in the admin survives.
"""
- days_crawled = UserStats.objects.filter(user=user, date__gte=today.replace(day=1), date__lte=today).count()
+ valid_until = end_of_next_month(today)
+ if CrawlerMembership.objects.filter(user=user, valid_until__gte=valid_until).exists():
+ return
+
+ days_crawled = UserStats.objects.filter(
+ user=user,
+ date__gte=today.replace(day=1),
+ date__lte=today,
+ num_results__gte=settings.CRAWLER_MEMBERSHIP_MIN_RESULTS,
+ ).count()
if days_crawled < settings.CRAWLER_MEMBERSHIP_DAYS:
return
- CrawlerMembership.objects.update_or_create(user=user, defaults={"valid_until": end_of_next_month(today)})
+
+ _, created = CrawlerMembership.objects.get_or_create(user=user, defaults={"valid_until": valid_until})
+ if not created:
+ CrawlerMembership.objects.filter(user=user, valid_until__lt=valid_until).update(valid_until=valid_until)
def has_crawler_membership(user_id: int) -> bool:
diff --git a/mwmbl/migrations/0045_crawlermembership.py b/mwmbl/migrations/0045_crawlermembership.py
index 681f2e7..8945870 100644
--- a/mwmbl/migrations/0045_crawlermembership.py
+++ b/mwmbl/migrations/0045_crawlermembership.py
@@ -14,7 +14,7 @@ class Migration(migrations.Migration):
fields=[
("id", models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
("valid_until", models.DateField()),
- ("earned", models.DateTimeField(auto_now=True)),
+ ("started", models.DateTimeField(auto_now_add=True)),
(
"user",
models.OneToOneField(
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 0b72775..f1603ba 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -278,7 +278,7 @@ class CrawlerMembership(models.Model):
user = models.OneToOneField(MwmblUser, on_delete=models.CASCADE, related_name="crawler_membership")
valid_until = models.DateField()
- earned = models.DateTimeField(auto_now=True)
+ started = models.DateTimeField(auto_now_add=True)
class AgreementType(models.TextChoices):
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.py
index e431b1f..5e5646e 100644
--- a/mwmbl/platform/api.py
+++ b/mwmbl/platform/api.py
@@ -26,7 +26,7 @@ from polar_sdk.webhooks import WebhookVerificationError
from mwmbl import membership, pricing
from mwmbl.background import enrich_domain_submission, stats_manager
-from mwmbl.crawler_membership import membership_tier
+from mwmbl.crawler_membership import has_crawler_membership, membership_tier
from mwmbl.exceptions import InvalidRequest
from mwmbl.membership import MembershipTier
from mwmbl.models import (
@@ -1454,9 +1454,7 @@ def create_membership_checkout(request, body: MembershipCheckoutRequest):
)
def cancel_membership(request):
check_email_verified(request)
- user_membership = Membership.objects.filter(user=request.user).first()
- if user_membership is None:
- raise InvalidRequest("Not a member.", status=404)
+ user_membership = _paid_membership(request.user)
if user_membership.cancel_at_period_end:
raise InvalidRequest("Membership is already scheduled to cancel.", status=409)
try:
@@ -1482,9 +1480,7 @@ def cancel_membership(request):
)
def uncancel_membership(request):
check_email_verified(request)
- user_membership = Membership.objects.filter(user=request.user).first()
- if user_membership is None:
- raise InvalidRequest("Not a member.", status=404)
+ user_membership = _paid_membership(request.user)
if not user_membership.cancel_at_period_end:
raise InvalidRequest("Membership is not scheduled to cancel.", status=409)
try:
@@ -1511,9 +1507,7 @@ def uncancel_membership(request):
)
def change_membership(request, body: MembershipChangeRequest):
check_email_verified(request)
- user_membership = Membership.objects.filter(user=request.user).first()
- if user_membership is None:
- raise InvalidRequest("Not a member.", status=404)
+ user_membership = _paid_membership(request.user)
if user_membership.tier == body.tier:
raise InvalidRequest("Already on this tier.", status=409)
# Changing tier would leave the cancellation in place, which is rarely what anyone wants;
@@ -1540,6 +1534,19 @@ def change_membership(request, body: MembershipChangeRequest):
return _save_membership_update(user_membership, tier=tier, current_period_end=result.current_period_end)
+def _paid_membership(user) -> Membership:
+ """The user's paid membership, for the endpoints that change it in Polar."""
+ user_membership = Membership.objects.filter(user=user).first()
+ if user_membership is not None:
+ return user_membership
+ if has_crawler_membership(user.id):
+ raise InvalidRequest(
+ "Memberships earned by crawling can't be cancelled or changed. Buy a membership to choose a tier.",
+ status=409,
+ )
+ raise InvalidRequest("Not a member.", status=404)
+
+
def _save_membership_update(user_membership, **changes):
"""Record a change made in Polar on the Membership row, unless a webhook changed it during the Polar call.
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index b50575d..982f29f 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -301,8 +301,10 @@ CURRENT_AGREEMENT_VERSIONS = {
# meant to replace. Monthly limits for web (JWT) callers, keyed by
# mwmbl.membership.MembershipTier value; "free" is signed-in users without a membership.
COMBINED_SEARCH_MONTHLY_LIMITS = {"free": 30, "sprout": 300, "sapling": 1_500, "canopy": 1_500}
-# Crawling on this many days in a calendar month earns a free Sprout membership.
+# Crawling on this many days in a calendar month earns a free Sprout membership. A day counts
+# once the user has submitted at least CRAWLER_MEMBERSHIP_MIN_RESULTS results that day.
CRAWLER_MEMBERSHIP_DAYS = 3
+CRAWLER_MEMBERSHIP_MIN_RESULTS = 100
# Its own model artifact, so Combined Search can be retrained on the pooled candidate set
# without moving standard search's ranking. Falls back to RUST_MODEL_PATH until trained.
COMBINED_MODEL_PATH = Path(__file__).parent / "resources" / "model-combined.xgb"
diff --git a/test/test_admin_paying_users.py b/test/test_admin_paying_users.py
index d38df13..6254269 100644
--- a/test/test_admin_paying_users.py
+++ b/test/test_admin_paying_users.py
@@ -6,8 +6,9 @@ from django.core.cache import cache
from fakeredis import FakeConnection
from mwmbl import quota
-from mwmbl.models import ApiKey, Membership, UserBilling
+from mwmbl.models import ApiKey, CrawlerMembership, Membership, UserBilling
from mwmbl.templatetags.money import minor_units
+from mwmbl.utils import utc_today
User = get_user_model()
@@ -143,6 +144,18 @@ def test_seed_search_usage_by_tier(staff_client):
assert [row["user"].username for row in seed_search["top_users"]] == ["free_user", "sprout_member"]
+def test_seed_search_counts_crawler_members_as_sprout(staff_client):
+ crawler = User.objects.create_user(username="crawler_member")
+ CrawlerMembership.objects.create(user=crawler, valid_until=utc_today())
+ quota.increment_monthly_combined_search(crawler.id)
+
+ response = staff_client.get(PAYING_USERS_URL)
+
+ by_tier = {tier["tier"]: tier for tier in response.context["seed_search"]["by_tier"]}
+ assert by_tier["sprout"]["users"] == 1
+ assert by_tier["free"]["users"] == 0
+
+
def test_seed_search_skips_deleted_users(staff_client):
deleted = User.objects.create_user(username="deleted_user")
quota.increment_monthly_combined_search(deleted.id)
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
index 413c8b0..832c588 100644
--- a/test/test_combined_search.py
+++ b/test/test_combined_search.py
@@ -24,7 +24,7 @@ import mwmbl.tinysearchengine.combined_search as combined_search
from mwmbl import pricing
from mwmbl.indexer import seed_crawl
from mwmbl.membership import MembershipTier
-from mwmbl.models import ApiKey, Membership, UserBilling, generate_api_key
+from mwmbl.models import ApiKey, CrawlerMembership, Membership, UserBilling, generate_api_key
from mwmbl.quota import (
_combined_search_api_monthly_key,
_combined_search_monthly_key,
@@ -33,6 +33,7 @@ from mwmbl.quota import (
get_monthly_count,
)
from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState, TinyIndex
+from mwmbl.utils import utc_today
User = get_user_model()
@@ -287,6 +288,20 @@ def _join(user, tier):
Membership.objects.create(user=user, tier=tier, polar_subscription_id=f"sub_{tier}")
+def test_a_crawler_membership_gives_the_sprout_limit(client, user, access_token, fresh_quota, stub_sources):
+ stub_sources()
+ CrawlerMembership.objects.create(user=user, valid_until=utc_today())
+ cache.set(_combined_search_monthly_key(user.id), 299, timeout=3600)
+
+ last_allowed = _get(client, access_token)
+ over_the_limit = _get(client, access_token)
+
+ assert last_allowed.status_code == 200
+ assert last_allowed.json()["monthly_limit"] == 300
+ assert over_the_limit.status_code == 429
+
+
@pytest.mark.django_db
@pytest.mark.parametrize(
"tier, limit",
diff --git a/test/test_crawler_membership.py b/test/test_crawler_membership.py
index c76dace..7692826 100644
--- a/test/test_crawler_membership.py
+++ b/test/test_crawler_membership.py
@@ -6,6 +6,7 @@ from unittest.mock import patch
import fakeredis
import pytest
+from allauth.account.models import EmailAddress
from django.test import Client
from ninja_jwt.tokens import RefreshToken
@@ -13,17 +14,28 @@ from mwmbl.crawler.stats import StatsManager
from mwmbl.crawler_membership import end_of_next_month, membership_tier
from mwmbl.membership import MembershipTier
from mwmbl.models import CrawlerMembership, Membership, MwmblUser
-from mwmbl.utils import utc_today
TODAY = date(2026, 10, 17)
+ENOUGH_RESULTS = 100
@pytest.fixture
def user(db):
- return MwmblUser.objects.create_user(username="crawler", email="[email protected]", password="testpass123")
+ user = MwmblUser.objects.create_user(username="crawler", email="[email protected]", password="testpass123")
+ EmailAddress.objects.create(user=user, email="[email protected]", verified=True, primary=True)
+ return user
-def _crawl(user, day: date, num_results: int = 5) -> None:
+def today():
+ with (
+ patch("mwmbl.crawler_membership.utc_today", return_value=TODAY),
+ patch("mwmbl.platform.api.utc_today", return_value=TODAY),
+ ):
+ yield TODAY
+
+
+def _crawl(user, day: date, num_results: int = ENOUGH_RESULTS) -> None:
results = SimpleNamespace(results=[object()] * num_results)
with patch("mwmbl.crawler.stats.utc_today", return_value=day):
StatsManager(fakeredis.FakeRedis()).record_results(results, user)
@@ -63,6 +75,24 @@ def test_several_batches_on_one_day_count_as_one_day(user):
assert not CrawlerMembership.objects.filter(user=user).exists()
[email protected]("num_results", [0, ENOUGH_RESULTS - 1])
+def test_days_with_too_few_results_do_not_count(user, num_results):
+ _crawl(user, date(2026, 10, 1))
+ _crawl(user, date(2026, 10, 2))
+ _crawl(user, date(2026, 10, 3), num_results=num_results)
+
+ assert not CrawlerMembership.objects.filter(user=user).exists()
+
+
+def test_membership_is_granted_when_a_day_reaches_the_minimum(user):
+ _crawl(user, date(2026, 10, 1))
+ _crawl(user, date(2026, 10, 2))
+ _crawl(user, date(2026, 10, 3), num_results=ENOUGH_RESULTS - 1)
+ _crawl(user, date(2026, 10, 3), num_results=1)
+
+ assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 11, 30)
+
+
def test_days_in_earlier_months_do_not_count(user):
_crawl(user, date(2026, 9, 29))
_crawl(user, date(2026, 9, 30))
@@ -80,28 +110,45 @@ def test_qualifying_again_extends_the_membership(user):
assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 12, 31)
-def test_crawler_membership_gives_sprout(user):
- CrawlerMembership.objects.create(user=user, valid_until=TODAY)
- with patch("mwmbl.crawler_membership.utc_today", return_value=TODAY):
- assert membership_tier(user.id) == MembershipTier.SPROUT
+def test_qualifying_does_not_shorten_a_longer_membership(user):
+ CrawlerMembership.objects.create(user=user, valid_until=date(2027, 6, 30))
+ for day in (1, 2, 3):
+ _crawl(user, date(2026, 10, day))
+
+ assert CrawlerMembership.objects.get(user=user).valid_until == date(2027, 6, 30)
+
+
+def test_a_failed_grant_is_retried_on_the_next_batch(user):
+ _crawl(user, date(2026, 10, 1))
+ _crawl(user, date(2026, 10, 2))
+ with patch("mwmbl.crawler_membership.CrawlerMembership.objects.get_or_create", side_effect=RuntimeError):
+ with pytest.raises(RuntimeError):
+ _crawl(user, date(2026, 10, 3))
+ _crawl(user, date(2026, 10, 3))
-def test_expired_crawler_membership_gives_nothing(user):
+ assert CrawlerMembership.objects.get(user=user).valid_until == date(2026, 11, 30)
+
+
+def test_crawler_membership_gives_sprout(user, today):
+ CrawlerMembership.objects.create(user=user, valid_until=today)
+ assert membership_tier(user.id) == MembershipTier.SPROUT
+
+
+def test_expired_crawler_membership_gives_nothing(user, today):
CrawlerMembership.objects.create(user=user, valid_until=date(2026, 10, 16))
- with patch("mwmbl.crawler_membership.utc_today", return_value=TODAY):
- assert membership_tier(user.id) is None
+ assert membership_tier(user.id) is None
-def test_paid_membership_wins_over_crawler_membership(user):
- CrawlerMembership.objects.create(user=user, valid_until=TODAY)
+def test_paid_membership_wins_over_crawler_membership(user, today):
+ CrawlerMembership.objects.create(user=user, valid_until=today)
Membership.objects.create(user=user, tier=MembershipTier.SAPLING, polar_subscription_id="sub_1")
assert membership_tier(user.id) == MembershipTier.SAPLING
-def test_get_membership_returns_crawler_membership(user):
+def test_get_membership_returns_crawler_membership(user, today):
CrawlerMembership.objects.create(user=user, valid_until=date(2026, 11, 30))
- with patch("mwmbl.platform.api.utc_today", return_value=TODAY):
- response = Client().get("/api/v1/platform/membership", **auth_headers(user))
+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))
assert response.status_code == 200
assert response.json() == {
@@ -112,6 +159,13 @@ def test_get_membership_returns_crawler_membership(user):
}
+def test_get_membership_is_404_when_the_crawler_membership_has_expired(user, today):
+ CrawlerMembership.objects.create(user=user, valid_until=date(2026, 10, 16))
+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))
+
+ assert response.status_code == 404
+
+
def test_get_membership_marks_paid_membership(user):
Membership.objects.create(user=user, tier=MembershipTier.SAPLING, polar_subscription_id="sub_1")
response = Client().get("/api/v1/platform/membership", **auth_headers(user))
@@ -120,8 +174,30 @@ def test_get_membership_marks_paid_membership(user):
assert response.json()["source"] == "paid"
-def test_crawler_membership_sets_the_combined_search_limit(user):
- CrawlerMembership.objects.create(user=user, valid_until=end_of_next_month(utc_today()))
+def test_get_membership_prefers_the_paid_membership(user, today):
+ CrawlerMembership.objects.create(user=user, valid_until=today)
+ Membership.objects.create(user=user, tier=MembershipTier.CANOPY, polar_subscription_id="sub_1")
+ response = Client().get("/api/v1/platform/membership", **auth_headers(user))
+
+ assert response.json()["tier"] == "canopy"
+ assert response.json()["source"] == "paid"
+
+
+ "path, body",
+ [("cancel", None), ("uncancel", None), ("change", {"tier": "sapling"})],
+)
+def test_crawler_memberships_cannot_be_cancelled_or_changed(user, today, path, body):
+ CrawlerMembership.objects.create(user=user, valid_until=today)
+ response = Client().post(
+ f"/api/v1/platform/membership/{path}", body, content_type="application/json", **auth_headers(user)
+ )
+
+ assert response.status_code == 409
+
+
+def test_crawler_membership_sets_the_combined_search_limit(user, today):
+ CrawlerMembership.objects.create(user=user, valid_until=today)
with patch("mwmbl.platform.api.get_monthly_combined_search_count", return_value=0):
response = Client().get("/api/v1/platform/combined-search/usage", **auth_headers(user))