Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Fix a missing API key encode; enhanced docs and tests

Daoud Clarke20e8fbeef2f9
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index dfa403b..5a80316 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -257,8 +257,10 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
if not raw_key:
return 401, {"message": "API key required. Pass it in the X-API-Key header."}
+ import hashlib
+ key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
api_key = ApiKey.objects.filter(
- key=raw_key,
+ key=key_hash,
scopes__contains=[ApiKey.Scope.CRAWL],
).select_related("user").first()
if api_key is None:
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index 663d679..7260027 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -216,7 +216,10 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
summary="Search",
description=(
"Search the Mwmbl index and return formatted results.\n\n"
- "Requires a valid search-scoped API key passed in the `X-API-Key` header. "
+ "Search the Mwmbl index and return formatted results.\n\n"
+ "Unauthenticated requests are accepted but return `monthly_usage` and "
+ "`monthly_limit` as `null`. To track usage against a quota, pass a valid "
+ "search-scoped API key in the `X-API-Key` header. "
"Obtain a key via `POST /api/v1/platform/api-keys/`.\n\n"
"Results are ranked using a heuristic ranker that considers title, extract, "
"domain authority, and query-term match quality. "
@@ -229,7 +232,8 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
"- `google` — originally suggested via Google\n"
"- `user` — submitted directly by a user\n\n"
"The response includes `monthly_usage` (requests used this month) and "
- "`monthly_limit` (your plan's monthly cap).\n\n"
+ "`monthly_limit` (your plan's monthly cap); both are `null` for "
+ "unauthenticated requests.\n\n"
"**Query parameter:** `s` — the search query string (required)."
),
openapi_extra={
@@ -247,10 +251,12 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
from mwmbl.search_auth import SearchApiKeyAuth
from mwmbl.models import ApiKey
- api_key: ApiKey | None = None
raw_key = request.headers.get("X-API-Key")
+ api_key = None
if raw_key:
api_key = SearchApiKeyAuth().authenticate(request, raw_key)
+ if api_key is None:
+ raise HttpError(401, "Invalid API key.")
if api_key is not None:
user: MwmblUser = api_key.user
@@ -273,8 +279,7 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
msg += f" {upgrade_msg}"
raise HttpError(429, msg)
- new_count = increment_monthly(user.id)
- monthly_usage = new_count
+ monthly_usage = increment_monthly(user.id)
else:
monthly_limit = None
monthly_usage = None
diff --git a/scripts/check_search_api_key.py b/scripts/check_search_api_key.py
new file mode 100755
index 0000000..25ecfa6
--- /dev/null
+++ b/scripts/check_search_api_key.py
@@ -0,0 +1,142 @@
+#!/usr/bin/env -S uv run
+# /// script
+# requires-python = ">=3.11"
+# dependencies = ["requests", "python-dotenv"]
+# ///
+"""
+Check search API behaviour with an API key:
+ 1. Usage value increases with each successful request
+ 2. Rate limiting kicks in and returns 429 when exceeded
+"""
+import os
+import sys
+import time
+from pathlib import Path
+
+import requests
+from dotenv import load_dotenv
+
+load_dotenv(Path(__file__).parent / ".env")
+
+BASE_URL = os.environ.get("MWMBL_BASE_URL", "http://localhost:8000")
+API_KEY = os.environ.get("MWMBL_API_KEY", "")
+JWT_TOKEN = os.environ.get("MWMBL_JWT_TOKEN", "")
+
+SEARCH_URL = f"{BASE_URL}/api/v1/search/"
+SUBSCRIPTION_URL = f"{BASE_URL}/api/v1/platform/billing/subscription"
+QUERY = "python"
+
+
+def search(session: requests.Session) -> requests.Response:
+ return session.get(SEARCH_URL, params={"s": QUERY}, headers={"X-API-Key": API_KEY})
+
+
+def check_usage_increases() -> None:
+ print("--- Check: monthly_usage increases ---")
+ session = requests.Session()
+ prev_usage = None
+ for i in range(3):
+ resp = search(session)
+ if resp.status_code != 200:
+ print(f" Request {i + 1}: unexpected status {resp.status_code} — {resp.text}")
+ continue
+ data = resp.json()
+ print("Response data", data)
+ usage = data.get("monthly_usage")
+ limit = data.get("monthly_limit")
+ print(f" Request {i + 1}: monthly_usage={usage}, monthly_limit={limit}")
+ assert usage is not None, "monthly_usage should not be null when using an API key"
+ if prev_usage is not None:
+ assert usage > prev_usage, (
+ f"monthly_usage did not increase: was {prev_usage}, now {usage}"
+ )
+ prev_usage = usage
+ # Small pause to avoid triggering rate limit between checks
+ time.sleep(0.3)
+ print(" PASS: usage increases with each request\n")
+
+
+def check_invalid_key_rejected() -> None:
+ print("--- Check: invalid API key returns 401 ---")
+ resp = requests.get(SEARCH_URL, params={"s": QUERY}, headers={"X-API-Key": "invalid-key"})
+ print(f" status={resp.status_code}, body={resp.json()}")
+ assert resp.status_code == 401, f"Expected 401 for invalid key, got {resp.status_code}"
+ print(" PASS: invalid API key returns 401\n")
+
+
+def check_rate_limit() -> None:
+ print("--- Check: rate limiting returns 429 ---")
+ session = requests.Session()
+ statuses = []
+ # Fire 10 requests as fast as possible; the limit is 5/sec so we expect 429s
+ for _ in range(10):
+ resp = search(session)
+ statuses.append(resp.status_code)
+
+ count_200 = statuses.count(200)
+ count_429 = statuses.count(429)
+ other = [s for s in statuses if s not in (200, 429)]
+ print(f" 200 OK: {count_200}, 429 Too Many Requests: {count_429}, other: {other}")
+
+ assert count_429 > 0, "Expected at least one 429 but got none"
+ assert not other, f"Unexpected status codes: {other}"
+ print(" PASS: rate limit triggers 429\n")
+
+ # Also inspect the 429 response body
+ # Re-fire until we get one to check the message
+ for _ in range(10):
+ resp = search(session)
+ if resp.status_code == 429:
+ body = resp.json()
+ print(f" 429 response body: {body}")
+ assert "detail" in body, "429 response should include a 'detail' field"
+ print(" PASS: 429 response has 'detail' field\n")
+ return
+ print(" WARNING: could not capture a 429 response body in the second burst\n")
+
+
+def check_subscription_usage(expected_usage: int) -> None:
+ print("--- Check: subscription endpoint reflects correct usage ---")
+ if not JWT_TOKEN:
+ print(" SKIP: MWMBL_JWT_TOKEN not set\n")
+ return
+ resp = requests.get(SUBSCRIPTION_URL, headers={"Authorization": f"Bearer {JWT_TOKEN}"})
+ assert resp.status_code == 200, f"Expected 200 from subscription, got {resp.status_code}: {resp.text}"
+ data = resp.json()
+ usage = data.get("monthly_usage")
+ limit = data.get("monthly_limit")
+ plan = data.get("plan")
+ print(f" plan={plan}, monthly_usage={usage}, monthly_limit={limit}")
+ assert usage is not None, "monthly_usage missing from subscription response"
+ assert limit is not None, "monthly_limit missing from subscription response"
+ assert usage >= expected_usage, (
+ f"subscription monthly_usage ({usage}) is less than search usage ({expected_usage})"
+ )
+ print(f" PASS: subscription usage ({usage}) >= search usage ({expected_usage})\n")
+
+
+def main() -> None:
+ if not API_KEY:
+ print("Set MWMBL_API_KEY environment variable before running this script.", file=sys.stderr)
+ sys.exit(1)
+
+ print(f"Target: {BASE_URL}\n")
+
+ try:
+ check_invalid_key_rejected()
+ check_usage_increases()
+ # Capture usage after a single search to compare against subscription
+ time.sleep(0.3)
+ resp = search(requests.Session())
+ last_search_usage = resp.json().get("monthly_usage", 0) if resp.status_code == 200 else 0
+ check_rate_limit()
+ check_subscription_usage(last_search_usage)
+ except AssertionError as exc:
+ print(f"FAIL: {exc}", file=sys.stderr)
+ sys.exit(1)
+
+ print("All checks passed.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/test/test_search_api_key.py b/test/test_search_api_key.py
index 0233030..9c4e2e7 100644
--- a/test/test_search_api_key.py
+++ b/test/test_search_api_key.py
@@ -14,7 +14,7 @@ from django.contrib.auth import get_user_model
from allauth.account.models import EmailAddress
from ninja_jwt.tokens import RefreshToken
-from mwmbl.models import ApiKey, MwmblUser, UsageBucket
+from mwmbl.models import ApiKey, MwmblUser, UsageBucket, generate_api_key
User = get_user_model()
@@ -69,22 +69,30 @@ def other_access_token(other_user):
@pytest.fixture
def search_api_key(verified_user):
- """A search-scoped ApiKey for verified_user."""
- return ApiKey.objects.create(
+ """A search-scoped ApiKey for verified_user. Has a .raw_key attribute for use in headers."""
+ raw_key, key_hash = generate_api_key()
+ obj = ApiKey.objects.create(
user=verified_user,
+ key=key_hash,
name="Test search key",
scopes=[ApiKey.Scope.SEARCH],
)
+ obj.raw_key = raw_key
+ return obj
@pytest.fixture
def crawl_api_key(verified_user):
- """A crawl-scoped ApiKey for verified_user."""
- return ApiKey.objects.create(
+ """A crawl-scoped ApiKey for verified_user. Has a .raw_key attribute for use in headers."""
+ raw_key, key_hash = generate_api_key()
+ obj = ApiKey.objects.create(
user=verified_user,
+ key=key_hash,
name="Test crawl key",
scopes=[ApiKey.Scope.CRAWL],
)
+ obj.raw_key = raw_key
+ return obj
@pytest.fixture
@@ -233,9 +241,13 @@ def test_delete_nonexistent_key_returns_404(api_client, access_token):
# ---------------------------------------------------------------------------
@pytest.mark.django_db
-def test_search_missing_key(api_client):
+def test_search_missing_key_returns_results_with_null_usage(api_client):
+ """Unauthenticated requests are allowed; usage fields are null."""
response = api_client.get("/api/v1/search/?s=python")
- assert response.status_code == 401
+ assert response.status_code == 200
+ data = response.json()
+ assert data["monthly_usage"] is None
+ assert data["monthly_limit"] is None
@pytest.mark.django_db
@@ -249,7 +261,7 @@ def test_search_crawl_scoped_key_rejected(api_client, crawl_api_key):
"""A crawl-scoped key must not grant access to the search endpoint."""
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(crawl_api_key.key),
+ **api_key_header(crawl_api_key.raw_key),
)
assert response.status_code == 401
@@ -268,7 +280,7 @@ def test_search_with_valid_key(api_client, search_api_key):
# Use the actual endpoint via the test client
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
# The response may be 200 or 500 depending on ranker availability in test env;
# the key point is it is NOT 401 (auth passed)
@@ -284,13 +296,13 @@ def test_search_response_includes_usage_fields(api_client, search_api_key):
patch("mwmbl.tinysearchengine.rank.HeuristicRanker.search", return_value=[]):
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
- if response.status_code == 200:
- data = response.json()
- assert "monthly_usage" in data
- assert "monthly_limit" in data
- assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]
+ assert response.status_code == 200
+ data = response.json()
+ assert "monthly_usage" in data
+ assert "monthly_limit" in data
+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]
# ---------------------------------------------------------------------------
@@ -302,7 +314,7 @@ def test_search_rate_limit_exceeded(api_client, search_api_key):
with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=False):
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
assert response.status_code == 429
assert "rate limit" in response.json()["detail"].lower()
@@ -315,7 +327,7 @@ def test_search_monthly_quota_exceeded(api_client, search_api_key):
patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit):
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
assert response.status_code == 429
detail = response.json()["detail"]
@@ -334,7 +346,7 @@ def test_search_monthly_quota_exceeded_pro_no_upgrade_message(api_client, search
patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=limit):
response = api_client.get(
"/api/v1/search/?s=python",
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
assert response.status_code == 429
detail = response.json()["detail"]
@@ -364,7 +376,7 @@ def test_post_results_header_key_accepted(api_client, crawl_api_key):
"/api/v1/crawler/results",
content_type="application/json",
data={"results": []},
- **api_key_header(crawl_api_key.key),
+ **api_key_header(crawl_api_key.raw_key),
)
assert response.status_code == 200
@@ -378,7 +390,7 @@ def test_post_results_body_key_deprecated_still_works(api_client, crawl_api_key)
response = api_client.post(
"/api/v1/crawler/results",
content_type="application/json",
- data={"api_key": crawl_api_key.key, "results": []},
+ data={"api_key": crawl_api_key.raw_key, "results": []},
)
assert response.status_code == 200
@@ -390,7 +402,7 @@ def test_post_results_search_scoped_key_rejected(api_client, search_api_key):
"/api/v1/crawler/results",
content_type="application/json",
data={"results": []},
- **api_key_header(search_api_key.key),
+ **api_key_header(search_api_key.raw_key),
)
assert response.status_code == 401
@@ -405,8 +417,8 @@ def test_post_results_header_takes_precedence_over_body(api_client, crawl_api_ke
"/api/v1/crawler/results",
content_type="application/json",
# Header has valid crawl key; body has search key (wrong scope)
- data={"api_key": search_api_key.key, "results": []},
- **api_key_header(crawl_api_key.key),
+ data={"api_key": search_api_key.raw_key, "results": []},
+ **api_key_header(crawl_api_key.raw_key),
)
assert response.status_code == 200
@@ -506,6 +518,101 @@ def test_sync_search_counts_keeps_redis_value_when_higher(verified_user):
cache.delete(key)
+# ---------------------------------------------------------------------------
+# Subscription endpoint — GET /api/v1/platform/billing/subscription
+# ---------------------------------------------------------------------------
+
+def test_subscription_unauthenticated(api_client):
+ response = api_client.get("/api/v1/platform/billing/subscription")
+ assert response.status_code == 401
+
+
+def test_subscription_free_user_no_usage(api_client, access_token, verified_user):
+ response = api_client.get(
+ "/api/v1/platform/billing/subscription",
+ **auth_headers(access_token),
+ )
+ assert response.status_code == 200
+ data = response.json()
+ assert data["plan"] == MwmblUser.Tier.FREE
+ assert data["status"] == "free"
+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.FREE]
+ assert data["monthly_usage"] == 0
+ assert data["current_period_end"] is None
+ assert data["polar_customer_id"] is None
+
+
+def test_subscription_reflects_usage_bucket(api_client, access_token, verified_user):
+ from django.utils import timezone
+ now = timezone.now()
+ UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=42)
+
+ response = api_client.get(
+ "/api/v1/platform/billing/subscription",
+ **auth_headers(access_token),
+ )
+ assert response.status_code == 200
+ assert response.json()["monthly_usage"] == 42
+
+
+def test_subscription_usage_matches_search_response(api_client, access_token, verified_user):
+ """Usage reported by the subscription endpoint should match what the search endpoint returned."""
+ from django.utils import timezone
+ from mwmbl.models import generate_api_key
+
+ now = timezone.now()
+ raw_key, key_hash = generate_api_key()
+ ApiKey.objects.create(
+ user=verified_user,
+ key=key_hash,
+ name="Test key",
+ scopes=[ApiKey.Scope.SEARCH],
+ )
+ UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=7)
+
+ with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \
+ patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=7), \
+ patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=8):
+ search_resp = api_client.get(
+ "/api/v1/search/?s=python",
+ **api_key_header(raw_key),
+ )
+
+ assert search_resp.status_code == 200
+ search_usage = search_resp.json()["monthly_usage"] # 8 (post-increment)
+ assert search_usage == 8
+
+ # Sync the incremented value into UsageBucket so the subscription endpoint sees it
+ UsageBucket.objects.filter(user=verified_user, year=now.year, month=now.month).update(count=search_usage)
+
+ sub_resp = api_client.get(
+ "/api/v1/platform/billing/subscription",
+ **auth_headers(access_token),
+ )
+ assert sub_resp.status_code == 200
+ assert sub_resp.json()["monthly_usage"] == search_usage
+
+
+def test_subscription_correct_limit_for_starter_tier(api_client, access_token, verified_user):
+ verified_user.tier = MwmblUser.Tier.STARTER
+ verified_user.save()
+
+ response = api_client.get(
+ "/api/v1/platform/billing/subscription",
+ **auth_headers(access_token),
+ )
+ assert response.status_code == 200
+ data = response.json()
+ assert data["plan"] == MwmblUser.Tier.STARTER
+ assert data["status"] == "active"
+ assert data["monthly_limit"] == MwmblUser.TIER_MONTHLY_LIMITS[MwmblUser.Tier.STARTER]
+
+
# ---------------------------------------------------------------------------
# Quota helper unit tests
# ---------------------------------------------------------------------------