An open source, non-profit web search engine
Fix a missing API key encode; enhanced docs and tests
Daoud Clarke20e8fbeef2f9
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex dfa403b..5a80316 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -257,8 +257,10 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchif not raw_key:return 401, {"message": "API key required. Pass it in the X-API-Key header."}+ import hashlib+ key_hash = hashlib.sha256(raw_key.encode()).hexdigest()api_key = ApiKey.objects.filter(- key=raw_key,+ key=key_hash,scopes__contains=[ApiKey.Scope.CRAWL],).select_related("user").first()if api_key is None:diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex 663d679..7260027 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -216,7 +216,10 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):summary="Search",description=("Search the Mwmbl index and return formatted results.\n\n"- "Requires a valid search-scoped API key passed in the `X-API-Key` header. "+ "Search the Mwmbl index and return formatted results.\n\n"+ "Unauthenticated requests are accepted but return `monthly_usage` and "+ "`monthly_limit` as `null`. To track usage against a quota, pass a valid "+ "search-scoped API key in the `X-API-Key` header. ""Obtain a key via `POST /api/v1/platform/api-keys/`.\n\n""Results are ranked using a heuristic ranker that considers title, extract, ""domain authority, and query-term match quality. "@@ -229,7 +232,8 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):"- `google` — originally suggested via Google\n""- `user` — submitted directly by a user\n\n""The response includes `monthly_usage` (requests used this month) and "- "`monthly_limit` (your plan's monthly cap).\n\n"+ "`monthly_limit` (your plan's monthly cap); both are `null` for "+ "unauthenticated requests.\n\n""**Query parameter:** `s` — the search query string (required)."),openapi_extra={@@ -247,10 +251,12 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):from mwmbl.search_auth import SearchApiKeyAuthfrom mwmbl.models import ApiKey- api_key: ApiKey | None = Noneraw_key = request.headers.get("X-API-Key")+ api_key = Noneif raw_key:api_key = SearchApiKeyAuth().authenticate(request, raw_key)+ if api_key is None:+ raise HttpError(401, "Invalid API key.")if api_key is not None:user: MwmblUser = api_key.user@@ -273,8 +279,7 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):msg += f" {upgrade_msg}"raise HttpError(429, msg)- new_count = increment_monthly(user.id)- monthly_usage = new_count+ monthly_usage = increment_monthly(user.id)else:monthly_limit = Nonemonthly_usage = Nonediff --git a/scripts/check_search_api_key.py b/scripts/check_search_api_key.pynew file mode 100755index 0000000..25ecfa6--- /dev/null+++ b/scripts/check_search_api_key.py@@ -0,0 +1,142 @@+#!/usr/bin/env -S uv run+# /// script+# requires-python = ">=3.11"+# dependencies = ["requests", "python-dotenv"]+# ///+"""+Check search API behaviour with an API key:+ 1. Usage value increases with each successful request+ 2. Rate limiting kicks in and returns 429 when exceeded+"""+import os+import sys+import time+from pathlib import Path++import requests+from dotenv import load_dotenv++load_dotenv(Path(__file__).parent / ".env")++BASE_URL = os.environ.get("MWMBL_BASE_URL", "http://localhost:8000")+API_KEY = os.environ.get("MWMBL_API_KEY", "")+JWT_TOKEN = os.environ.get("MWMBL_JWT_TOKEN", "")++SEARCH_URL = f"{BASE_URL}/api/v1/search/"+SUBSCRIPTION_URL = f"{BASE_URL}/api/v1/platform/billing/subscription"+QUERY = "python"+++def search(session: requests.Session) -> requests.Response:+ return session.get(SEARCH_URL, params={"s": QUERY}, headers={"X-API-Key": API_KEY})+++def check_usage_increases() -> None:+ print("--- Check: monthly_usage increases ---")+ session = requests.Session()+ prev_usage = None+ for i in range(3):+ resp = search(session)+ if resp.status_code != 200:+ print(f" Request {i + 1}: unexpected status {resp.status_code} — {resp.text}")+ continue+ data = resp.json()+ print("Response data", data)+ usage = data.get("monthly_usage")+ limit = data.get("monthly_limit")+ print(f" Request {i + 1}: monthly_usage={usage}, monthly_limit={limit}")+ assert usage is not None, "monthly_usage should not be null when using an API key"+ if prev_usage is not None:+ assert usage > prev_usage, (+ f"monthly_usage did not increase: was {prev_usage}, now {usage}"+ )+ prev_usage = usage+ # Small pause to avoid triggering rate limit between checks+ time.sleep(0.3)+ print(" PASS: usage increases with each request\n")+++def check_invalid_key_rejected() -> None:+ print("--- Check: invalid API key returns 401 ---")+ resp = requests.get(SEARCH_URL, params={"s": QUERY}, headers={"X-API-Key": "invalid-key"})+ print(f" status={resp.status_code}, body={resp.json()}")+ assert resp.status_code == 401, f"Expected 401 for invalid key, got {resp.status_code}"+ print(" PASS: invalid API key returns 401\n")+++def check_rate_limit() -> None:+ print("--- Check: rate limiting returns 429 ---")+ session = requests.Session()+ statuses = []+ # Fire 10 requests as fast as possible; the limit is 5/sec so we expect 429s+ for _ in range(10):+ resp = search(session)+ statuses.append(resp.status_code)++ count_200 = statuses.count(200)+ count_429 = statuses.count(429)+ other = [s for s in statuses if s not in (200, 429)]+ print(f" 200 OK: {count_200}, 429 Too Many Requests: {count_429}, other: {other}")++ assert count_429 > 0, "Expected at least one 429 but got none"+ assert not other, f"Unexpected status codes: {other}"+ print(" PASS: rate limit triggers 429\n")++ # Also inspect the 429 response body+ # Re-fire until we get one to check the message+ for _ in range(10):+ resp = search(session)+ if resp.status_code == 429:+ body = resp.json()+ print(f" 429 response body: {body}")+ assert "detail" in body, "429 response should include a 'detail' field"+ print(" PASS: 429 response has 'detail' field\n")+ return+ print(" WARNING: could not capture a 429 response body in the second burst\n")+++def check_subscription_usage(expected_usage: int) -> None:+ print("--- Check: subscription endpoint reflects correct usage ---")+ if not JWT_TOKEN:+ print(" SKIP: MWMBL_JWT_TOKEN not set\n")+ return+ resp = requests.get(SUBSCRIPTION_URL, headers={"Authorization": f"Bearer {JWT_TOKEN}"})+ assert resp.status_code == 200, f"Expected 200 from subscription, got {resp.status_code}: {resp.text}"+ data = resp.json()+ usage = data.get("monthly_usage")+ limit = data.get("monthly_limit")+ plan = data.get("plan")+ print(f" plan={plan}, monthly_usage={usage}, monthly_limit={limit}")+ assert usage is not None, "monthly_usage missing from subscription response"+ assert limit is not None, "monthly_limit missing from subscription response"+ assert usage >= expected_usage, (+ f"subscription monthly_usage ({usage}) is less than search usage ({expected_usage})"+ )+ print(f" PASS: subscription usage ({usage}) >= search usage ({expected_usage})\n")+++def main() -> None:+ if not API_KEY:+ print("Set MWMBL_API_KEY environment variable before running this script.", file=sys.stderr)+ sys.exit(1)++ print(f"Target: {BASE_URL}\n")++ try:+ check_invalid_key_rejected()+ check_usage_increases()+ # Capture usage after a single search to compare against subscription+ time.sleep(0.3)+ resp = search(requests.Session())+ last_search_usage = resp.json().get("monthly_usage", 0) if resp.status_code == 200 else 0+ check_rate_limit()+ check_subscription_usage(last_search_usage)+ except AssertionError as exc:+ print(f"FAIL: {exc}", file=sys.stderr)+ sys.exit(1)++ print("All checks passed.")+++if __name__ == "__main__":+ main()diff --git a/test/test_search_api_key.py b/test/test_search_api_key.pyindex 0233030..9c4e2e7 100644--- a/test/test_search_api_key.py+++ b/test/test_search_api_key.py@@ -14,7 +14,7 @@ from django.contrib.auth import get_user_modelfrom allauth.account.models import EmailAddressfrom ninja_jwt.tokens import RefreshToken-from mwmbl.models import ApiKey, MwmblUser, UsageBucket+from mwmbl.models import ApiKey, MwmblUser, UsageBucket, generate_api_keyUser = get_user_model()@@ -69,22 +69,30 @@ def other_access_token(other_user):@pytest.fixturedef search_api_key(verified_user):- """A search-scoped ApiKey for verified_user."""- return ApiKey.objects.create(+ """A search-scoped ApiKey for verified_user. Has a .raw_key attribute for use in headers."""+ raw_key, key_hash = generate_api_key()+ obj = ApiKey.objects.create(user=verified_user,+ key=key_hash,name="Test search key",scopes=[ApiKey.Scope.SEARCH],)+ obj.raw_key = raw_key+ return obj@pytest.fixturedef crawl_api_key(verified_user):- """A crawl-scoped ApiKey for verified_user."""- return ApiKey.objects.create(+ """A crawl-scoped ApiKey for verified_user. Has a .raw_key attribute for use in headers."""+ raw_key, key_hash = generate_api_key()+ obj = ApiKey.objects.create(user=verified_user,+ key=key_hash,name="Test crawl key",scopes=[ApiKey.Scope.CRAWL],)+ obj.raw_key = raw_key+ return obj@pytest.fixture@@ -233,9 +241,13 @@ def test_delete_nonexistent_key_returns_404(api_client, access_token):# ---------------------------------------------------------------------------@pytest.mark.django_db-def test_search_missing_key(api_client):+def test_search_missing_key_returns_results_with_null_usage(api_client):+ """Unauthenticated requests are allowed; usage fields are null."""response = api_client.get("/api/v1/search/?s=python")- assert response.status_code == 401+ assert response.status_code == 200+ data = response.json()+ assert data["monthly_usage"] is None+ assert data["monthly_limit"] is None@pytest.mark.django_db@@ -249,7 +261,7 @@ def test_search_crawl_scoped_key_rejected(api_client, crawl_api_key):"""A crawl-scoped key must not grant access to the search endpoint."""response = api_client.get("/api/v1/search/?s=python",- **api_key_header(crawl_api_key.key),+ **api_key_header(crawl_api_key.raw_key),)assert response.status_code == 401@@ -268,7 +280,7 @@ def test_search_with_valid_key(api_client, search_api_key):# Use the actual endpoint via the test clientresponse = api_client.get("/api/v1/search/?s=python",- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)# The response may be 200 or 500 depending on ranker availability in test env;# the key point is it is NOT 401 (auth passed)@@ -284,13 +296,13 @@ def test_search_response_includes_usage_fields(api_client, search_api_key):patch("mwmbl.tinysearchengine.rank.HeuristicRanker.search", return_value=[]):response = api_client.get("/api/v1/search/?s=python",- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)- if response.status_code == 200:- data = response.json()- assert "monthly_usage" in data- assert "monthly_limit" in data- assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]+ assert response.status_code == 200+ data = response.json()+ assert "monthly_usage" in data+ assert "monthly_limit" in data+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]# ---------------------------------------------------------------------------@@ -302,7 +314,7 @@ def test_search_rate_limit_exceeded(api_client, search_api_key):with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=False):response = api_client.get("/api/v1/search/?s=python",- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)assert response.status_code == 429assert "rate limit" in response.json()["detail"].lower()@@ -315,7 +327,7 @@ def test_search_monthly_quota_exceeded(api_client, search_api_key):patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit):response = api_client.get("/api/v1/search/?s=python",- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)assert response.status_code == 429detail = response.json()["detail"]@@ -334,7 +346,7 @@ def test_search_monthly_quota_exceeded_pro_no_upgrade_message(api_client, searchpatch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit):response = api_client.get("/api/v1/search/?s=python",- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)assert response.status_code == 429detail = response.json()["detail"]@@ -364,7 +376,7 @@ def test_post_results_header_key_accepted(api_client, crawl_api_key):"/api/v1/crawler/results",content_type="application/json",data={"results": []},- **api_key_header(crawl_api_key.key),+ **api_key_header(crawl_api_key.raw_key),)assert response.status_code == 200@@ -378,7 +390,7 @@ def test_post_results_body_key_deprecated_still_works(api_client, crawl_api_key)response = api_client.post("/api/v1/crawler/results",content_type="application/json",- data={"api_key": crawl_api_key.key, "results": []},+ data={"api_key": crawl_api_key.raw_key, "results": []},)assert response.status_code == 200@@ -390,7 +402,7 @@ def test_post_results_search_scoped_key_rejected(api_client, search_api_key):"/api/v1/crawler/results",content_type="application/json",data={"results": []},- **api_key_header(search_api_key.key),+ **api_key_header(search_api_key.raw_key),)assert response.status_code == 401@@ -405,8 +417,8 @@ def test_post_results_header_takes_precedence_over_body(api_client, crawl_api_ke"/api/v1/crawler/results",content_type="application/json",# Header has valid crawl key; body has search key (wrong scope)- data={"api_key": search_api_key.key, "results": []},- **api_key_header(crawl_api_key.key),+ data={"api_key": search_api_key.raw_key, "results": []},+ **api_key_header(crawl_api_key.raw_key),)assert response.status_code == 200@@ -506,6 +518,101 @@ def test_sync_search_counts_keeps_redis_value_when_higher(verified_user):cache.delete(key)+# ---------------------------------------------------------------------------+# Subscription endpoint — GET /api/v1/platform/billing/subscription+# ---------------------------------------------------------------------------++def test_subscription_unauthenticated(api_client):+ response = api_client.get("/api/v1/platform/billing/subscription")+ assert response.status_code == 401+++def test_subscription_free_user_no_usage(api_client, access_token, verified_user):+ response = api_client.get(+ "/api/v1/platform/billing/subscription",+ **auth_headers(access_token),+ )+ assert response.status_code == 200+ data = response.json()+ assert data["plan"] == MwmblUser.Tier.FREE+ assert data["status"] == "free"+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]+ assert data["monthly_usage"] == 0+ assert data["current_period_end"] is None+ assert data["polar_customer_id"] is None+++def test_subscription_reflects_usage_bucket(api_client, access_token, verified_user):+ from django.utils import timezone+ now = timezone.now()+ UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=42)++ response = api_client.get(+ "/api/v1/platform/billing/subscription",+ **auth_headers(access_token),+ )+ assert response.status_code == 200+ assert response.json()["monthly_usage"] == 42+++def test_subscription_usage_matches_search_response(api_client, access_token, verified_user):+ """Usage reported by the subscription endpoint should match what the search endpoint returned."""+ from django.utils import timezone+ from mwmbl.models import generate_api_key++ now = timezone.now()+ raw_key, key_hash = generate_api_key()+ ApiKey.objects.create(+ user=verified_user,+ key=key_hash,+ name="Test key",+ scopes=[ApiKey.Scope.SEARCH],+ )+ UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=7)++ with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=7), \+ patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=8):+ search_resp = api_client.get(+ "/api/v1/search/?s=python",+ **api_key_header(raw_key),+ )++ assert search_resp.status_code == 200+ search_usage = search_resp.json()["monthly_usage"] # 8 (post-increment)+ assert search_usage == 8++ # Sync the incremented value into UsageBucket so the subscription endpoint sees it+ UsageBucket.objects.filter(user=verified_user, year=now.year, month=now.month).update(count=search_usage)++ sub_resp = api_client.get(+ "/api/v1/platform/billing/subscription",+ **auth_headers(access_token),+ )+ assert sub_resp.status_code == 200+ assert sub_resp.json()["monthly_usage"] == search_usage+++def test_subscription_correct_limit_for_starter_tier(api_client, access_token, verified_user):+ verified_user.tier = MwmblUser.Tier.STARTER+ verified_user.save()++ response = api_client.get(+ "/api/v1/platform/billing/subscription",+ **auth_headers(access_token),+ )+ assert response.status_code == 200+ data = response.json()+ assert data["plan"] == MwmblUser.Tier.STARTER+ assert data["status"] == "active"+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.STARTER]++# ---------------------------------------------------------------------------# Quota helper unit tests# ---------------------------------------------------------------------------