An open source, non-profit web search engine
Move imports to top of file; remove reduntant imports
Daoud Clarke20fd6bc02c4e
diff --git a/mwmbl/background.py b/mwmbl/background.pyindex 0415169..50cca81 100644--- a/mwmbl/background.py+++ b/mwmbl/background.py@@ -6,16 +6,19 @@ Also contains Django Background Tasks for periodic quota maintenance:"""import loggingimport sys+from datetime import datetime, timezonefrom logging import getLogger, basicConfigfrom pathlib import Pathfrom time import sleepfrom background_task import backgroundfrom django.conf import settings+from django.core.cache import cachefrom mwmbl.indexer import index_batches, historicalfrom mwmbl.indexer.batch_cache import BatchCache-from mwmbl.models import OldIndex+from mwmbl.models import OldIndex, UsageBucket+from mwmbl.quota import MONTHLY_TTL, _monthly_key, get_all_monthly_keysfrom mwmbl.tinysearchengine.copy_index import copy_pagesNUM_PAGES_TO_COPY = 1024@@ -96,13 +99,6 @@ def sync_search_counts():Step 2 (Redis → Postgres): update UsageBucket with the live Redis countsso Postgres stays current as a durable backup."""- from datetime import datetime, timezone-- from django.core.cache import cache-- from mwmbl.models import UsageBucket- from mwmbl.quota import MONTHLY_TTL, _monthly_key, get_all_monthly_keys-now = datetime.now(timezone.utc)# Step 1: seed Redis from Postgres, taking the max of the two values.diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 5a80316..3ee48a1 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -257,7 +257,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batchif not raw_key:return 401, {"message": "API key required. Pass it in the X-API-Key header."}- import hashlibkey_hash = hashlib.sha256(raw_key.encode()).hexdigest()api_key = ApiKey.objects.filter(key=key_hash,diff --git a/mwmbl/models.py b/mwmbl/models.pyindex 7208b35..7838701 100644--- a/mwmbl/models.py+++ b/mwmbl/models.py@@ -1,3 +1,4 @@+import hashlibimport secretsfrom django.contrib.postgres.fields import ArrayField@@ -128,7 +129,6 @@ def random_api_key():def generate_api_key() -> tuple[str, str]:"""Return (raw_key, key_hash). Store only the hash; return raw_key to the user once."""- import hashlibraw = secrets.token_urlsafe(64)return raw, hashlib.sha256(raw.encode()).hexdigest()diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex 7260027..26c99a6 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -6,7 +6,9 @@ from ninja import NinjaAPI, Router, Schemafrom ninja.errors import HttpErrorfrom mwmbl.format import format_result+from mwmbl.models import ApiKey, MwmblUserfrom mwmbl.quota import check_rate_limit, get_monthly_count, increment_monthly+from mwmbl.search_auth import SearchApiKeyAuthfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.rank import HeuristicRanker@@ -189,7 +191,6 @@ _COMPLETE_RESPONSE_SCHEMA = {# ---------------------------------------------------------------------------def _upgrade_message(tier: str) -> str:- from mwmbl.models import MwmblUserif tier == MwmblUser.Tier.FREE:return ("Upgrade to Starter (10,000 requests/month) or Pro (50,000 requests/month) "@@ -207,15 +208,12 @@ def _upgrade_message(tier: str) -> str:def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):"""Register search routes on the given router or API instance."""- from mwmbl.models import MwmblUser-@r.get("",response=SearchResponse,auth=None,summary="Search",description=(- "Search the Mwmbl index and return formatted results.\n\n""Search the Mwmbl index and return formatted results.\n\n""Unauthenticated requests are accepted but return `monthly_usage` and ""`monthly_limit` as `null`. To track usage against a quota, pass a valid "@@ -248,9 +246,6 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):},)def search(request, s: str):- from mwmbl.search_auth import SearchApiKeyAuth- from mwmbl.models import ApiKey-raw_key = request.headers.get("X-API-Key")api_key = Noneif raw_key:diff --git a/test/test_search_api_key.py b/test/test_search_api_key.pyindex 53aec3a..e4e3de8 100644--- a/test/test_search_api_key.py+++ b/test/test_search_api_key.py@@ -281,16 +281,11 @@ def test_search_with_valid_key(api_client, search_api_key):with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=0), \patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=1):- # Mock the ranker so we don't need a real index- with patch("mwmbl.tinysearchengine.search.router") as mock_router:- # Use the actual endpoint via the test client- response = api_client.get(- "/api/v1/search/?s=python",- **api_key_header(search_api_key.raw_key),- )- # The response may be 200 or 500 depending on ranker availability in test env;- # the key point is it is NOT 401 (auth passed)- assert response.status_code != 401+ response = api_client.get(+ "/api/v1/search/?s=python",+ **api_key_header(search_api_key.raw_key),+ )+ assert response.status_code == 200@pytest.mark.django_db@@ -441,7 +436,7 @@ def test_sync_search_counts_redis_to_postgres(verified_user):key = _monthly_key(verified_user.id)cache.set(key, 42, timeout=3600)- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[key]):+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[key]):sync_search_counts.now()bucket = UsageBucket.objects.get(user=verified_user, year=now.year, month=now.month)@@ -461,7 +456,7 @@ def test_sync_search_counts_seeds_redis_from_postgres(verified_user):UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=99)cache.delete(key)- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):sync_search_counts.now()assert get_monthly_count(verified_user.id) == 99@@ -481,7 +476,7 @@ def test_sync_search_counts_uses_postgres_value_when_higher(verified_user):UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=70)cache.set(key, 5, timeout=3600)- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):sync_search_counts.now()assert get_monthly_count(verified_user.id) == 70@@ -500,7 +495,7 @@ def test_sync_search_counts_keeps_redis_value_when_higher(verified_user):UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=70)cache.set(key, 85, timeout=3600)- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):sync_search_counts.now()assert get_monthly_count(verified_user.id) == 85