Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Move imports to top of file; remove reduntant imports

Daoud Clarke20fd6bc02c4e
diff --git a/mwmbl/background.py b/mwmbl/background.py
index 0415169..50cca81 100644
--- a/mwmbl/background.py
+++ b/mwmbl/background.py
@@ -6,16 +6,19 @@ Also contains Django Background Tasks for periodic quota maintenance:
"""
import logging
import sys
+from datetime import datetime, timezone
from logging import getLogger, basicConfig
from pathlib import Path
from time import sleep
from background_task import background
from django.conf import settings
+from django.core.cache import cache
from mwmbl.indexer import index_batches, historical
from mwmbl.indexer.batch_cache import BatchCache
-from mwmbl.models import OldIndex
+from mwmbl.models import OldIndex, UsageBucket
+from mwmbl.quota import MONTHLY_TTL, _monthly_key, get_all_monthly_keys
from mwmbl.tinysearchengine.copy_index import copy_pages
NUM_PAGES_TO_COPY = 1024
@@ -96,13 +99,6 @@ def sync_search_counts():
Step 2 (Redis → Postgres): update UsageBucket with the live Redis counts
so Postgres stays current as a durable backup.
"""
- from datetime import datetime, timezone
-
- from django.core.cache import cache
-
- from mwmbl.models import UsageBucket
- from mwmbl.quota import MONTHLY_TTL, _monthly_key, get_all_monthly_keys
-
now = datetime.now(timezone.utc)
# Step 1: seed Redis from Postgres, taking the max of the two values.
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 5a80316..3ee48a1 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -257,7 +257,6 @@ def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batch
if not raw_key:
return 401, {"message": "API key required. Pass it in the X-API-Key header."}
- import hashlib
key_hash = hashlib.sha256(raw_key.encode()).hexdigest()
api_key = ApiKey.objects.filter(
key=key_hash,
diff --git a/mwmbl/models.py b/mwmbl/models.py
index 7208b35..7838701 100644
--- a/mwmbl/models.py
+++ b/mwmbl/models.py
@@ -1,3 +1,4 @@
+import hashlib
import secrets
from django.contrib.postgres.fields import ArrayField
@@ -128,7 +129,6 @@ def random_api_key():
def generate_api_key() -> tuple[str, str]:
"""Return (raw_key, key_hash). Store only the hash; return raw_key to the user once."""
- import hashlib
raw = secrets.token_urlsafe(64)
return raw, hashlib.sha256(raw.encode()).hexdigest()
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index 7260027..26c99a6 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -6,7 +6,9 @@ from ninja import NinjaAPI, Router, Schema
from ninja.errors import HttpError
from mwmbl.format import format_result
+from mwmbl.models import ApiKey, MwmblUser
from mwmbl.quota import check_rate_limit, get_monthly_count, increment_monthly
+from mwmbl.search_auth import SearchApiKeyAuth
from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.rank import HeuristicRanker
@@ -189,7 +191,6 @@ _COMPLETE_RESPONSE_SCHEMA = {
# ---------------------------------------------------------------------------
def _upgrade_message(tier: str) -> str:
- from mwmbl.models import MwmblUser
if tier == MwmblUser.Tier.FREE:
return (
"Upgrade to Starter (10,000 requests/month) or Pro (50,000 requests/month) "
@@ -207,15 +208,12 @@ def _upgrade_message(tier: str) -> str:
def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
"""Register search routes on the given router or API instance."""
- from mwmbl.models import MwmblUser
-
@r.get(
"",
response=SearchResponse,
auth=None,
summary="Search",
description=(
- "Search the Mwmbl index and return formatted results.\n\n"
"Search the Mwmbl index and return formatted results.\n\n"
"Unauthenticated requests are accepted but return `monthly_usage` and "
"`monthly_limit` as `null`. To track usage against a quota, pass a valid "
@@ -248,9 +246,6 @@ def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
},
)
def search(request, s: str):
- from mwmbl.search_auth import SearchApiKeyAuth
- from mwmbl.models import ApiKey
-
raw_key = request.headers.get("X-API-Key")
api_key = None
if raw_key:
diff --git a/test/test_search_api_key.py b/test/test_search_api_key.py
index 53aec3a..e4e3de8 100644
--- a/test/test_search_api_key.py
+++ b/test/test_search_api_key.py
@@ -281,16 +281,11 @@ def test_search_with_valid_key(api_client, search_api_key):
with patch("mwmbl.tinysearchengine.search.check_rate_limit", return_value=True), \
patch("mwmbl.tinysearchengine.search.get_monthly_count", return_value=0), \
patch("mwmbl.tinysearchengine.search.increment_monthly", return_value=1):
- # Mock the ranker so we don't need a real index
- with patch("mwmbl.tinysearchengine.search.router") as mock_router:
- # Use the actual endpoint via the test client
- response = api_client.get(
- "/api/v1/search/?s=python",
- **api_key_header(search_api_key.raw_key),
- )
- # The response may be 200 or 500 depending on ranker availability in test env;
- # the key point is it is NOT 401 (auth passed)
- assert response.status_code != 401
+ response = api_client.get(
+ "/api/v1/search/?s=python",
+ **api_key_header(search_api_key.raw_key),
+ )
+ assert response.status_code == 200
@pytest.mark.django_db
@@ -441,7 +436,7 @@ def test_sync_search_counts_redis_to_postgres(verified_user):
key = _monthly_key(verified_user.id)
cache.set(key, 42, timeout=3600)
- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[key]):
+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[key]):
sync_search_counts.now()
bucket = UsageBucket.objects.get(user=verified_user, year=now.year, month=now.month)
@@ -461,7 +456,7 @@ def test_sync_search_counts_seeds_redis_from_postgres(verified_user):
UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=99)
cache.delete(key)
- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):
+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):
sync_search_counts.now()
assert get_monthly_count(verified_user.id) == 99
@@ -481,7 +476,7 @@ def test_sync_search_counts_uses_postgres_value_when_higher(verified_user):
UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=70)
cache.set(key, 5, timeout=3600)
- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):
+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):
sync_search_counts.now()
assert get_monthly_count(verified_user.id) == 70
@@ -500,7 +495,7 @@ def test_sync_search_counts_keeps_redis_value_when_higher(verified_user):
UsageBucket.objects.create(user=verified_user, year=now.year, month=now.month, count=70)
cache.set(key, 85, timeout=3600)
- with patch("mwmbl.quota.get_all_monthly_keys", return_value=[]):
+ with patch("mwmbl.background.get_all_monthly_keys", return_value=[]):
sync_search_counts.now()
assert get_monthly_count(verified_user.id) == 85