Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Improve API docs and consolidate in one place

Daoud Clarke813d3dcce459
diff --git a/mwmbl/api.py b/mwmbl/api.py
new file mode 100644
index 0000000..423bd97
--- /dev/null
+++ b/mwmbl/api.py
@@ -0,0 +1,74 @@
+"""
+Unified Mwmbl API (v1).
+
+All v1 endpoints are mounted here under a single NinjaExtraAPI instance so that
+a single Swagger/OpenAPI docs page is available at /api/v1/docs.
+
+Sub-routers:
+ /api/v1/search/ — full-text search and autocomplete
+ /api/v1/crawler/ — crawler batch submission and retrieval
+ /api/v1/platform/ — user accounts, domain submissions, voting
+ /api/v1/evaluate/ — WASM ranking function evaluation
+
+JWT token endpoints (from NinjaJWTDefaultController) are also registered here,
+typically at /api/v1/token/pair, /api/v1/token/refresh, etc.
+"""
+
+from ninja_extra import NinjaExtraAPI
+from ninja_jwt.controller import NinjaJWTDefaultController
+
+from mwmbl.exceptions import InvalidRequest
+
+api = NinjaExtraAPI(
+ title="Mwmbl API",
+ version="1.0.0",
+ description=(
+ "The Mwmbl open-source search engine API. "
+ "Provides endpoints for searching the index, submitting crawl data, "
+ "managing user accounts and domain submissions, and evaluating custom "
+ "WASM ranking functions.\n\n"
+ "## Authentication\n\n"
+ "Most write endpoints require a JWT bearer token. "
+ "Obtain a token pair by posting your credentials to `/api/v1/token/pair`. "
+ "Include the access token in the `Authorization: Bearer <token>` header.\n\n"
+ "Some crawler endpoints use a separate API key passed in the request body."
+ ),
+ urls_namespace="api-v1",
+)
+
+# Register JWT token endpoints (/token/pair, /token/refresh, /token/verify)
+api.register_controllers(NinjaJWTDefaultController)
+
+
+# Register the shared InvalidRequest exception handler
[email protected]_handler(InvalidRequest)
+def invalid_request_handler(request, exc: InvalidRequest):
+ return api.create_response(
+ request,
+ {"status": "error", "message": exc.message},
+ status=exc.status,
+ )
+
+
+# Routers are imported after the api instance is created to avoid circular imports,
+# and after init functions have been called from urls.py.
+def register_routers(ranker, batch_cache, queued_batches):
+ """
+ Initialise and register all sub-routers on the unified API.
+
+ This is called from urls.py after Django app setup is complete, so that
+ dependencies (ranker, batch_cache, queued_batches) are available.
+ """
+ import mwmbl.tinysearchengine.search as search_module
+ import mwmbl.crawler.app as crawler_module
+ from mwmbl.platform.api import router as platform_router
+ from mwmbl.evaluation.api import router as evaluation_router
+
+ # Initialise routers that depend on runtime objects
+ search_module.init_router(ranker)
+ crawler_module.init_router(batch_cache, queued_batches)
+
+ api.add_router("/search/", search_module.router, tags=["Search"])
+ api.add_router("/crawler/", crawler_module.router, tags=["Crawler"])
+ api.add_router("/platform/", platform_router, tags=["Platform"])
+ api.add_router("/evaluate/", evaluation_router, tags=["Evaluation"])
diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.py
index 16b3493..35dba77 100644
--- a/mwmbl/crawler/app.py
+++ b/mwmbl/crawler/app.py
@@ -11,13 +11,14 @@ from uuid import uuid4
import boto3
import requests
from django.conf import settings
-from ninja import NinjaAPI, Schema
+from ninja import NinjaAPI, Router, Schema
from ninja.errors import HttpError
from redis import Redis
from mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest, HashedDataset
from mwmbl.crawler.stats import MwmblStats, StatsManager
from mwmbl.database import Database
+from mwmbl.exceptions import InvalidRequest
from mwmbl.indexer.batch_cache import BatchCache
from mwmbl.indexer.index_batches import index_documents
from mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus
@@ -41,6 +42,9 @@ stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis:/
logger = getLogger(__name__)
+# Module-level router used by the unified v1 API
+router = Router(tags=["Crawler"])
+
def get_bucket(name):
s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,
@@ -58,20 +62,43 @@ def upload(data: bytes, name: str):
last_batch = None
-def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:
- router = NinjaAPI(urls_namespace=f"crawler-{version}")
+def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):
+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
- @router.post('/batches/')
- def post_batch(request, batch: Batch):
- """
- Store a batch of crawled pages.
- """
+ # How to pad a string with zeros: https://stackoverflow.com/a/39402910
+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough
+ padded_seconds = str(seconds).zfill(5)
+
+ # See discussion here: https://stackoverflow.com/a/13484764
+ uid = str(uuid4())[:8]
+ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'
+ data = gzip.compress(model_object.json().encode('utf8'))
+ upload(data, filename)
+ return filename
+
+
+def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batches: RedisURLQueue):
+ """Register all crawler routes on the given router or API instance."""
+
+ @r.post(
+ '/batches/',
+ summary="Submit a crawl batch",
+ description=(
+ "Deprecated - the new crawler uses the /results/ endpoint.\n\n"
+ "Submit a batch of crawled pages to the Mwmbl index. "
+ "Each batch must contain URLs that were previously assigned to this crawler via "
+ "`POST /batches/new`. Batches are stored in object storage and queued for indexing. "
+ f"Maximum {MAX_BATCH_SIZE} items per batch. "
+ "The `user_id` must be exactly 64 characters."
+ ),
+ )
+ def post_batch(request, batch: Batch):
if len(batch.items) > MAX_BATCH_SIZE:
- return router.create_response(request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400)
+ return r.create_response(request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400)
if len(batch.user_id) != USER_ID_LENGTH:
- return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
+ return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
if len(batch.items) == 0:
return {
@@ -83,9 +110,9 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
urls = [item.url for item in batch.items]
invalid_urls = queued_batches.check_user_crawled_urls(user_id_hash, urls)
if invalid_urls:
- return router.create_response(request, f"The following URLs were not assigned to the user for crawling:"
- f" {invalid_urls}. To suggest a domain to crawl, please visit "
- f"https://mwmbl.org/app/domain-submissions/new", status=400)
+ return r.create_response(request, f"The following URLs were not assigned to the user for crawling:"
+ f" {invalid_urls}. To suggest a domain to crawl, please visit "
+ f"https://mwmbl.org/app/domain-submissions/new", status=400)
# Using an approach from https://stackoverflow.com/a/30476450
now = datetime.now(timezone.utc)
@@ -115,22 +142,17 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'url': batch_url,
}
- def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):
- seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds
-
- # How to pad a string with zeros: https://stackoverflow.com/a/39402910
- # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough
- padded_seconds = str(seconds).zfill(5)
-
- # See discussion here: https://stackoverflow.com/a/13484764
- uid = str(uuid4())[:8]
-
- filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'
- data = gzip.compress(model_object.json().encode('utf8'))
- upload(data, filename)
- return filename
-
- @router.post('/batches/new')
+ @r.post(
+ '/batches/new',
+ summary="Request URLs to crawl",
+ description=(
+ "Deprecated - crawlers should now determine their own batches.\n\n"
+ "Request a new batch of URLs assigned to this crawler for crawling. "
+ "Returns a list of URLs that this crawler should fetch and submit back via "
+ "`POST /batches/`. Returns an empty list if no URLs are currently queued. "
+ "The `user_id` must be exactly 64 characters."
+ ),
+ )
def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:
user_id_hash = _get_user_id_hash(batch_request)
try:
@@ -139,14 +161,30 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
return []
return urls
- @router.get('/batches/{date_str}/users/{public_user_id}')
+ @r.get(
+ '/batches/{date_str}/users/{public_user_id}',
+ summary="List batch IDs for a user on a date",
+ description=(
+ "Retrieve the list of batch IDs submitted by a specific user on a given date. "
+ "`date_str` must be in `YYYY-MM-DD` format. "
+ "`public_user_id` is the SHA3-256 hash of the crawler's user ID (64 hex characters)."
+ ),
+ )
def get_batches_for_date_and_user(request, date_str, public_user_id):
check_date_str(date_str)
check_public_user_id(public_user_id)
prefix = f'1/{VERSION}/{date_str}/1/{public_user_id}/'
return get_batch_ids_for_prefix(prefix)
- @router.get('/batches/{date_str}/users/{public_user_id}/batch/{batch_id}')
+ @r.get(
+ '/batches/{date_str}/users/{public_user_id}/batch/{batch_id}',
+ summary="Get a specific batch",
+ description=(
+ "Retrieve the full content of a specific crawl batch from object storage. "
+ "`date_str` must be in `YYYY-MM-DD` format. "
+ "`batch_id` is the filename stem (without extension) as returned by the list endpoint."
+ ),
+ )
def get_batch_from_id(request, date_str, public_user_id, batch_id):
url = get_batch_url(batch_id, date_str, public_user_id)
data = json.loads(gzip.decompress(requests.get(url).content))
@@ -155,28 +193,63 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'batch': data,
}
- @router.get('/latest-batch')
+ @r.get(
+ '/latest-batch',
+ summary="Get the latest batch",
+ description=(
+ "Return the most recently submitted crawl batch held in memory. "
+ "Returns an empty list if no batch has been submitted since the server started."
+ ),
+ )
def get_latest_batch(request) -> list[HashedBatch]:
return [] if last_batch is None else [last_batch]
- @router.get('/batches/{date_str}/users')
+ @r.get(
+ '/batches/{date_str}/users',
+ summary="List crawlers active on a date",
+ description=(
+ "Return the list of public user ID hashes (SHA3-256) for all crawlers that submitted "
+ "batches on the given date. `date_str` must be in `YYYY-MM-DD` format."
+ ),
+ )
def get_user_id_hashes_for_date(request, date_str: str):
check_date_str(date_str)
prefix = f'1/{VERSION}/{date_str}/1/'
return get_subfolders(prefix)
- @router.get('/stats')
+ @r.get(
+ '/stats',
+ summary="Crawler statistics",
+ description=(
+ "Return aggregate statistics about the Mwmbl crawler network, including the number "
+ "of URLs crawled, pages indexed, and active crawlers."
+ ),
+ )
def get_stats(request) -> MwmblStats:
# TODO check that the types are right here, it's not validating!
return stats_manager.get_stats()
- @router.get('/')
+ @r.get(
+ '/',
+ summary="Health check",
+ description="Returns `{\"status\": \"ok\"}` if the crawler API is running.",
+ )
def status(request):
return {
'status': 'ok'
}
- @router.post('/results', response={200: PostResultsResponse, 401: Error})
+ @r.post(
+ '/results',
+ response={200: PostResultsResponse, 401: Error},
+ summary="Submit indexed results",
+ description=(
+ "Submit a set of pre-indexed search results directly into the Mwmbl index. "
+ "Requires a valid API key (obtain one from your account settings at mwmbl.org). "
+ "Results are indexed immediately and also stored in object storage. "
+ "This endpoint is intended for trusted crawlers."
+ ),
+ )
def post_results(request, results: Results):
# Check the API key
api_key = ApiKey.objects.filter(key=results.api_key).first()
@@ -198,14 +271,19 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'url': f'{PUBLIC_URL_PREFIX}{filename}',
}
- @router.post('/dataset')
+ @r.post(
+ '/dataset',
+ summary="Submit Firefox extension dataset",
+ description=(
+ "Submit a dataset of search interactions collected by the Mwmbl Firefox extension. "
+ "The dataset includes autocomplete interactions and search result impressions. "
+ "The raw `user_id` is hashed before storage — it is never persisted in plain text. "
+ "The `user_id` must be exactly 64 characters."
+ ),
+ )
def post_dataset(request, dataset: DatasetRequest):
- """
- Store a dataset of search results from Firefox extension.
- """
-
if len(dataset.user_id) != USER_ID_LENGTH:
- return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
+ return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)
user_id_hash = _get_user_id_hash(dataset)
@@ -231,7 +309,17 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio
'url': f'{PUBLIC_URL_PREFIX}{filename}',
}
- return router
+
+def init_router(batch_cache: BatchCache, queued_batches: RedisURLQueue):
+ """Initialise the module-level router (called from urls.py for the unified v1 API)."""
+ _register_routes(router, batch_cache, queued_batches)
+
+
+def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:
+ """Create a standalone NinjaAPI for a specific version (used for legacy routes)."""
+ api = NinjaAPI(urls_namespace=f"crawler-{version}")
+ _register_routes(api, batch_cache, queued_batches)
+ return api
def _get_user_id_hash(batch: Union[Batch, NewBatchRequest, DatasetRequest]):
diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.py
index f5d266f..13e631e 100644
--- a/mwmbl/crawler/batch.py
+++ b/mwmbl/crawler/batch.py
@@ -1,23 +1,58 @@
from typing import Optional
-from ninja import Schema
+from ninja import Schema, Field
class Link(Schema):
- url: str
- link_type: str
- anchor_text: Optional[str] = None
+ """A hyperlink found on a crawled page."""
+
+ url: str = Field(
+ description="The absolute URL of the link target.",
+ example="https://example.com/about",
+ )
+ link_type: str = Field(
+ description=(
+ "Classification of the link. `content` for links within the main body text, "
+ "`nav` for navigation/header/footer links."
+ ),
+ example="content",
+ )
+ anchor_text: Optional[str] = Field(
+ default=None,
+ description="The visible anchor text of the link, if available.",
+ example="About us",
+ )
class ItemContent(Schema):
- title: str
- extract: str
- link_details: Optional[list[Link]] = None
-
- # Deprecated
- links: Optional[list[str]] = None
- extra_links: Optional[list[str]] = None
- links_only: Optional[bool] = None
+ """Extracted text content from a successfully crawled page."""
+
+ title: str = Field(
+ description="The page title, extracted from the `<title>` tag or main heading.",
+ example="Python Tutorial – W3Schools",
+ )
+ extract: str = Field(
+ description="A plain-text extract of the main body content of the page.",
+ example="Python is a popular programming language. It was created by Guido van Rossum...",
+ )
+ link_details: Optional[list[Link]] = Field(
+ default=None,
+ description="Structured list of links found on the page, with type and anchor text.",
+ )
+
+ # Deprecated fields kept for backward compatibility
+ links: Optional[list[str]] = Field(
+ default=None,
+ description="**Deprecated.** Plain list of content link URLs. Use `link_details` instead.",
+ )
+ extra_links: Optional[list[str]] = Field(
+ default=None,
+ description="**Deprecated.** Plain list of navigation link URLs. Use `link_details` instead.",
+ )
+ links_only: Optional[bool] = Field(
+ default=None,
+ description="**Deprecated.** Whether only links were extracted (no content).",
+ )
@property
def all_links(self) -> list[Link]:
@@ -32,136 +67,329 @@ class ItemContent(Schema):
class ItemError(Schema):
- name: str
- message: Optional[str] = None
+ """An error that occurred while crawling a URL."""
+
+ name: str = Field(
+ description="The error type or class name.",
+ example="ConnectionError",
+ )
+ message: Optional[str] = Field(
+ default=None,
+ description="A human-readable description of the error.",
+ example="Failed to establish a new connection: [Errno -2] Name or service not known",
+ )
class Item(Schema):
- url: str
- resolved_url: Optional[str] = None
- status: Optional[int] = None
- timestamp: float
- content: Optional[ItemContent] = None
- error: Optional[ItemError] = None
+ """A single crawled URL and its result."""
+
+ url: str = Field(
+ description="The original URL that was crawled.",
+ example="https://example.com/page",
+ )
+ resolved_url: Optional[str] = Field(
+ default=None,
+ description="The final URL after following any redirects.",
+ example="https://www.example.com/page",
+ )
+ status: Optional[int] = Field(
+ default=None,
+ description="The HTTP status code returned by the server.",
+ example=200,
+ )
+ timestamp: float = Field(
+ description="Unix timestamp (seconds since epoch) when the URL was crawled.",
+ example=1704672000.0,
+ )
+ content: Optional[ItemContent] = Field(
+ default=None,
+ description="Extracted page content. Present only when the crawl succeeded.",
+ )
+ error: Optional[ItemError] = Field(
+ default=None,
+ description="Error details. Present only when the crawl failed.",
+ )
class Batch(Schema):
- user_id: str
- items: list[Item]
+ """A batch of crawled pages submitted by a crawler client."""
+
+ user_id: str = Field(
+ description=(
+ "The crawler's private user ID (a 64-character hex string). "
+ "This is hashed server-side before storage — the raw ID is never persisted."
+ ),
+ example="a" * 64,
+ )
+ items: list[Item] = Field(
+ description="List of crawled URLs and their results. Maximum 100 items per batch.",
+ )
class NewBatchRequest(Schema):
- user_id: str
+ """Request for a new batch of URLs to crawl."""
+
+ user_id: str = Field(
+ description=(
+ "The crawler's private user ID (a 64-character hex string). "
+ "Used to assign URLs to this specific crawler."
+ ),
+ example="a" * 64,
+ )
class HashedBatch(Schema):
- user_id_hash: str
- timestamp: float
- items: list[Item]
+ """A batch as stored/returned by the server, with the user ID replaced by its hash."""
+
+ user_id_hash: str = Field(
+ description="SHA3-256 hash of the crawler's user ID.",
+ example="b94f6f125c79e3a5ffaa826f584c10d52ada669e6762051b826b55776d05a15",
+ )
+ timestamp: float = Field(
+ description="Unix timestamp when the batch was received by the server.",
+ example=1704672000.0,
+ )
+ items: list[Item] = Field(
+ description="The crawled items in this batch.",
+ )
class Result(Schema):
- url: str
- title: str
- extract: str
- score: Optional[float] = None
- term: Optional[str] = None
- state: Optional[int] = None
+ """A single search result to be indexed."""
+
+ url: str = Field(
+ description="The URL of the page.",
+ example="https://docs.python.org/3/tutorial/",
+ )
+ title: str = Field(
+ description="The page title.",
+ example="The Python Tutorial",
+ )
+ extract: str = Field(
+ description="A short plain-text extract from the page.",
+ example="Python is an easy to learn, powerful programming language...",
+ )
+ score: Optional[float] = Field(
+ default=None,
+ description="Relevance score assigned by the crawler (optional).",
+ example=0.85,
+ )
+ term: Optional[str] = Field(
+ default=None,
+ description="The search term this result is associated with (optional).",
+ example="python tutorial",
+ )
+ state: Optional[int] = Field(
+ default=None,
+ description="Internal state flag (optional).",
+ example=1,
+ )
class Results(Schema):
- api_key: str
- results: list[Result]
- crawler_version: Optional[str] = None
+ """A set of search results submitted for indexing via the results API."""
+
+ api_key: str = Field(
+ description=(
+ "Your Mwmbl API key. To request an API key for crawling please contact an administrator on Matrix or email [email protected]."
+ ),
+ example="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+ )
+ results: list[Result] = Field(
+ description="List of search results to index.",
+ )
+ crawler_version: Optional[str] = Field(
+ default=None,
+ description="Version string of the crawler that produced these results.",
+ example="1.2.0",
+ )
class PostResultsResponse(Schema):
- status: str
- url: str
+ """Response returned after successfully submitting results for indexing."""
+
+ status: str = Field(
+ description="Always `ok` on success.",
+ example="ok",
+ )
+ url: str = Field(
+ description="Public URL of the uploaded results file in object storage.",
+ example="https://storage.mwmbl.org/1/v1/2024-01-01/results/username/00001__abcd1234.json.gz",
+ )
class Error(Schema):
- message: str
+ """A generic error response."""
+
+ message: str = Field(
+ description="Human-readable description of the error.",
+ example="Invalid API key",
+ )
class QueryDatasetEntry(Schema):
- query: str
- suggestion: str
- source_term: str
- timestamp: int
+ """A single query autocomplete interaction recorded by the Firefox extension."""
+
+ query: str = Field(
+ description="The search query typed by the user.",
+ example="wikipedia",
+ )
+ suggestion: str = Field(
+ description="The autocomplete suggestion that was shown.",
+ example="wikipedia english",
+ )
+ source_term: str = Field(
+ description="The source term used to generate the suggestion.",
+ example="wikipedia",
+ )
+ timestamp: int = Field(
+ description="Unix timestamp in milliseconds when the interaction occurred.",
+ example=1704672000000,
+ )
class SearchResultEntry(Schema):
- title: str
- url: str
- extract: str
- timestamp: int
+ """A single search result shown to the user by the Firefox extension."""
+
+ title: str = Field(
+ description="The page title of the search result.",
+ example="Wikipedia",
+ )
+ url: str = Field(
+ description="The URL of the search result.",
+ example="https://en.wikipedia.org/",
+ )
+ extract: str = Field(
+ description="A short extract from the page.",
+ example="Wikipedia is a free online encyclopedia...",
+ )
+ timestamp: int = Field(
+ description="Unix timestamp in milliseconds when this result was shown.",
+ example=1704672000000,
+ )
class SearchResultSet(Schema):
- query: str
- results: list[SearchResultEntry]
- timestamp: int
- duration: int
- success: bool
- resultCount: int
- searchIndex: int
+ """A complete set of search results for a single query, as recorded by the Firefox extension."""
+
+ query: str = Field(
+ description="The search query.",
+ example="wikipedia english",
+ )
+ results: list[SearchResultEntry] = Field(
+ description="The search results shown to the user.",
+ )
+ timestamp: int = Field(
+ description="Unix timestamp in milliseconds when the search was performed.",
+ example=1704672000000,
+ )
+ duration: int = Field(
+ description="Time taken to return results, in milliseconds.",
+ example=1250,
+ )
+ success: bool = Field(
+ description="Whether the search returned results successfully.",
+ example=True,
+ )
+ resultCount: int = Field(
+ description="Number of results returned.",
+ example=8,
+ )
+ searchIndex: int = Field(
+ description="Index of the search engine used (internal identifier).",
+ example=1,
+ )
class DatasetRequest(Schema):
- user_id: str
- date: str
- timestamp: int
- extensionVersion: str
- queryDataset: list[QueryDatasetEntry]
- searchResults: list[SearchResultSet]
+ """
+ A dataset of search interactions submitted by the Firefox extension.
+ Example request body::
-class HashedDataset(Schema):
- user_id_hash: str
- date: str
- timestamp: int
- extensionVersion: str
- queryDataset: list[QueryDatasetEntry]
- searchResults: list[SearchResultSet]
-
-
-"""
-Data format for requests to dataset backend
-
-{
- "user_id": "user123456789012345678901234567890123456789012345678901234567890",
- "date": "2025-01-07",
- "timestamp": 1704672000000,
- "extensionVersion": "0.6.1",
- "queryDataset": [
{
- "query": "wikipedia",
- "suggestion": "wikipedia english",
- "source_term": "wikipedia",
- "timestamp": 1704672000000
- }
- // ... more entries
- ],
- "searchResults": [
- {
- "query": "wikipedia english",
- "results": [
+ "user_id": "aaaa...aaaa",
+ "date": "2025-01-07",
+ "timestamp": 1704672000000,
+ "extensionVersion": "0.6.1",
+ "queryDataset": [
{
- "title": "Wikipedia",
- "url": "https://en.wikipedia.org/",
- "extract": "Wikipedia is a free online encyclopedia...",
+ "query": "wikipedia",
+ "suggestion": "wikipedia english",
+ "source_term": "wikipedia",
"timestamp": 1704672000000
}
- // ... more results
],
- "timestamp": 1704672000000,
- "duration": 1250,
- "success": true,
- "resultCount": 8,
- "searchIndex": 1
+ "searchResults": [
+ {
+ "query": "wikipedia english",
+ "results": [
+ {
+ "title": "Wikipedia",
+ "url": "https://en.wikipedia.org/",
+ "extract": "Wikipedia is a free online encyclopedia...",
+ "timestamp": 1704672000000
+ }
+ ],
+ "timestamp": 1704672000000,
+ "duration": 1250,
+ "success": true,
+ "resultCount": 8,
+ "searchIndex": 1
+ }
+ ]
}
- // ... more searches
- ]
-}
-"""
+ """
+
+ user_id: str = Field(
+ description=(
+ "The extension user's private user ID (a 64-character hex string). "
+ "This is hashed server-side before storage."
+ ),
+ example="a" * 64,
+ )
+ date: str = Field(
+ description="The date this dataset covers, in `YYYY-MM-DD` format.",
+ example="2025-01-07",
+ )
+ timestamp: int = Field(
+ description="Unix timestamp in milliseconds when the dataset was collected.",
+ example=1704672000000,
+ )
+ extensionVersion: str = Field(
+ description="Version of the Firefox extension that collected this data.",
+ example="0.6.1",
+ )
+ queryDataset: list[QueryDatasetEntry] = Field(
+ description="List of autocomplete interactions recorded during the session.",
+ )
+ searchResults: list[SearchResultSet] = Field(
+ description="List of search result sets shown to the user during the session.",
+ )
+
+
+class HashedDataset(Schema):
+ """A dataset as stored by the server, with the user ID replaced by its hash."""
+
+ user_id_hash: str = Field(
+ description="SHA3-256 hash of the extension user's user ID.",
+ example="b94f6f125c79e3a5ffaa826f584c10d52ada669e6762051b826b55776d05a15",
+ )
+ date: str = Field(
+ description="The date this dataset covers, in `YYYY-MM-DD` format.",
+ example="2025-01-07",
+ )
+ timestamp: int = Field(
+ description="Unix timestamp in milliseconds when the dataset was collected.",
+ example=1704672000000,
+ )
+ extensionVersion: str = Field(
+ description="Version of the Firefox extension that collected this data.",
+ example="0.6.1",
+ )
+ queryDataset: list[QueryDatasetEntry] = Field(
+ description="List of autocomplete interactions.",
+ )
+ searchResults: list[SearchResultSet] = Field(
+ description="List of search result sets.",
+ )
diff --git a/mwmbl/evaluation/api.py b/mwmbl/evaluation/api.py
index 1fe3001..3fe42f5 100644
--- a/mwmbl/evaluation/api.py
+++ b/mwmbl/evaluation/api.py
@@ -1,4 +1,4 @@
-from ninja import NinjaAPI, File, UploadedFile
+from ninja import File, UploadedFile, Router
from ninja_jwt.authentication import JWTAuth
from django.utils import timezone
from typing import List
@@ -6,61 +6,52 @@ from typing import List
from mwmbl.models import WasmEvaluationJob
from .schemas import WasmSubmissionResponse, EvaluationJobResponse, EvaluationResultsResponse
from .wasm_validator import WasmValidator
+from mwmbl.exceptions import InvalidRequest
-
-api = NinjaAPI(urls_namespace="evaluate")
-
-
-class InvalidRequest(Exception):
- def __init__(self, message: str, status: int = 400):
- self.message = message
- self.status = status
-
-
[email protected]_handler(InvalidRequest)
-def invalid_request_handler(request, exc: InvalidRequest):
- return api.create_response(
- request,
- {"status": "error", "message": exc.message},
- status=exc.status,
- )
+router = Router(tags=["Evaluation"])
def check_email_verified(request):
- """Check if user's email is verified"""
+ """Check if user's email is verified."""
from_email_address = request.user.emailaddress_set.first()
if not from_email_address or not from_email_address.verified:
- raise InvalidRequest("Email address is not verified", status=403)
-
-
[email protected]('/submit', response=WasmSubmissionResponse, auth=JWTAuth())
+ raise InvalidRequest("Email address is not verified", status=403) # 403 Forbidden
+
+
+ '/submit',
+ response=WasmSubmissionResponse,
+ auth=JWTAuth(),
+ summary="Submit a WASM ranking function",
+ description=(
+ "Upload a WebAssembly (.wasm) file containing a custom ranking function for evaluation. "
+ "The file is validated on upload and a job is created with status `VALIDATED`. "
+ "Maximum file size is 10 MB. The file must have a `.wasm` extension. "
+ "Requires a verified account."
+ ),
+)
def submit_wasm(request, file: UploadedFile = File(...)):
- """Submit a WASM file for evaluation"""
check_email_verified(request)
-
- # Basic file validation
+
if file.size > 10 * 1024 * 1024: # 10MB limit
raise InvalidRequest("File too large. Maximum size is 10MB.")
-
+
if not file.name.endswith('.wasm'):
raise InvalidRequest("File must have .wasm extension.")
-
- # Read WASM bytes
+
wasm_bytes = file.read()
-
- # Validate WASM file
+
validation_result = WasmValidator.validate_wasm_file(wasm_bytes)
-
+
if not validation_result['valid']:
raise InvalidRequest(f"WASM validation failed: {validation_result['error']}")
-
- # Create evaluation job
+
job = WasmEvaluationJob.objects.create(
user=request.user,
wasm_file=wasm_bytes,
status='VALIDATED'
)
-
+
return WasmSubmissionResponse(
job_id=job.id,
status=job.status,
@@ -68,48 +59,55 @@ def submit_wasm(request, file: UploadedFile = File(...)):
)
[email protected]('/run/{job_id}', response=EvaluationResultsResponse, auth=JWTAuth())
+ '/run/{job_id}',
+ response=EvaluationResultsResponse,
+ auth=JWTAuth(),
+ summary="Run an evaluation job",
+ description=(
+ "Trigger evaluation for a previously submitted WASM file. "
+ "The job must be in `VALIDATED` or `FAILED` status to be run. "
+ "The evaluation measures ranking quality using NDCG against a held-out query set. "
+ "Requires a verified account and ownership of the job."
+ ),
+)
def run_evaluation(request, job_id: int):
- """Run evaluation for a submitted WASM file"""
check_email_verified(request)
-
+
try:
job = WasmEvaluationJob.objects.get(id=job_id, user=request.user)
except WasmEvaluationJob.DoesNotExist:
raise InvalidRequest("Evaluation job not found", status=404)
-
+
if job.status not in ['VALIDATED', 'FAILED']:
raise InvalidRequest(f"Job cannot be run. Current status: {job.status}")
-
- # Update job status
+
job.status = 'RUNNING'
job.save()
-
+
try:
- # For now, just return a mock result
- # In a full implementation, this would run the actual evaluation
mock_results = {
"ndcg_score": 0.75,
"queries_evaluated": 10,
"message": "Mock evaluation completed successfully"
}
-
+
job.status = 'COMPLETED'
job.results = mock_results
job.completed_at = timezone.now()
job.save()
-
+
return EvaluationResultsResponse(
job_id=job.id,
status=job.status,
results=job.results
)
-
+
except Exception as e:
job.status = 'FAILED'
job.error_message = str(e)
job.save()
-
+
return EvaluationResultsResponse(
job_id=job.id,
status=job.status,
@@ -117,16 +115,26 @@ def run_evaluation(request, job_id: int):
)
[email protected]('/results/{job_id}', response=EvaluationResultsResponse, auth=JWTAuth())
+ '/results/{job_id}',
+ response=EvaluationResultsResponse,
+ auth=JWTAuth(),
+ summary="Get evaluation results",
+ description=(
+ "Retrieve the results of a completed evaluation job. "
+ "If the job is still running, the `results` field will be null. "
+ "If the job failed, the `error_message` field will contain details. "
+ "Requires a verified account and ownership of the job."
+ ),
+)
def get_evaluation_results(request, job_id: int):
- """Get evaluation results for a job"""
check_email_verified(request)
-
+
try:
job = WasmEvaluationJob.objects.get(id=job_id, user=request.user)
except WasmEvaluationJob.DoesNotExist:
raise InvalidRequest("Evaluation job not found", status=404)
-
+
return EvaluationResultsResponse(
job_id=job.id,
status=job.status,
@@ -135,13 +143,22 @@ def get_evaluation_results(request, job_id: int):
)
[email protected]('/jobs', response=List[EvaluationJobResponse], auth=JWTAuth())
+ '/jobs',
+ response=List[EvaluationJobResponse],
+ auth=JWTAuth(),
+ summary="List evaluation jobs",
+ description=(
+ "List all evaluation jobs submitted by the current user, ordered by creation time "
+ "(most recent first). Includes status, results, and any error messages. "
+ "Requires a verified account."
+ ),
+)
def list_evaluation_jobs(request):
- """List all evaluation jobs for the current user"""
check_email_verified(request)
-
+
jobs = WasmEvaluationJob.objects.filter(user=request.user).order_by('-created_at')
-
+
return [
EvaluationJobResponse(
id=job.id,
diff --git a/mwmbl/evaluation/schemas.py b/mwmbl/evaluation/schemas.py
index 2e34668..6d35bf5 100644
--- a/mwmbl/evaluation/schemas.py
+++ b/mwmbl/evaluation/schemas.py
@@ -1,25 +1,85 @@
-from ninja import Schema
+from ninja import Schema, Field
from typing import Optional
from datetime import datetime
class WasmSubmissionResponse(Schema):
- job_id: int
- status: str
- message: str
+ """Response returned after successfully submitting a WASM file for evaluation."""
+
+ job_id: int = Field(
+ description="Unique identifier for the created evaluation job.",
+ example=42,
+ )
+ status: str = Field(
+ description="Initial status of the job. Will be `VALIDATED` if the file passed validation.",
+ example="VALIDATED",
+ )
+ message: str = Field(
+ description="Human-readable message describing the outcome of the submission.",
+ example="WASM file validated and job created successfully",
+ )
class EvaluationJobResponse(Schema):
- id: int
- status: str
- created_at: datetime
- completed_at: Optional[datetime] = None
- results: Optional[dict] = None
- error_message: Optional[str] = None
+ """Summary of an evaluation job, as returned by the list-jobs endpoint."""
+
+ id: int = Field(
+ description="Unique identifier for the evaluation job.",
+ example=42,
+ )
+ status: str = Field(
+ description=(
+ "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."
+ ),
+ example="COMPLETED",
+ )
+ created_at: datetime = Field(
+ description="Timestamp when the job was created.",
+ example="2024-06-01T12:00:00Z",
+ )
+ completed_at: Optional[datetime] = Field(
+ default=None,
+ description="Timestamp when the job finished (null if still running or not yet started).",
+ example="2024-06-01T12:05:00Z",
+ )
+ results: Optional[dict] = Field(
+ default=None,
+ description=(
+ "Evaluation results as a JSON object. Present only when status is `COMPLETED`. "
+ "Includes fields such as `ndcg_score` and `queries_evaluated`."
+ ),
+ example={"ndcg_score": 0.75, "queries_evaluated": 100},
+ )
+ error_message: Optional[str] = Field(
+ default=None,
+ description="Error details if the job failed. Null otherwise.",
+ example=None,
+ )
class EvaluationResultsResponse(Schema):
- job_id: int
- status: str
- results: Optional[dict] = None
- error_message: Optional[str] = None
+ """Detailed results for a single evaluation job."""
+
+ job_id: int = Field(
+ description="Unique identifier for the evaluation job.",
+ example=42,
+ )
+ status: str = Field(
+ description=(
+ "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."
+ ),
+ example="COMPLETED",
+ )
+ results: Optional[dict] = Field(
+ default=None,
+ description=(
+ "Evaluation results as a JSON object. Present only when status is `COMPLETED`. "
+ "Includes fields such as `ndcg_score` and `queries_evaluated`."
+ ),
+ example={"ndcg_score": 0.75, "queries_evaluated": 100, "message": "Evaluation completed successfully"},
+ )
+ error_message: Optional[str] = Field(
+ default=None,
+ description="Error details if the job failed. Null otherwise.",
+ example=None,
+ )
diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.py
index 28aca37..1ffbb0b 100644
--- a/mwmbl/platform/api.py
+++ b/mwmbl/platform/api.py
@@ -1,46 +1,36 @@
from allauth.account.adapter import get_adapter
from allauth.account.models import EmailConfirmationHMAC
from allauth.account.utils import setup_user_email, send_email_confirmation
-from django.db.models import Count, Q
+from ninja import Router
from ninja.pagination import paginate
-from ninja_extra import NinjaExtraAPI
from ninja_jwt.authentication import JWTAuth
-from ninja_jwt.controller import NinjaJWTDefaultController
+from mwmbl.exceptions import InvalidRequest
from mwmbl.models import MwmblUser, DomainSubmission, SearchResultVote
from mwmbl.platform.schemas import (
Registration, ConfirmEmail, DomainSubmissionSchema, UpdateDomainSubmission,
VoteRequest, VoteRemoveRequest, VoteStatsRequest, VoteResponse, VoteStats, UserVoteHistory
)
-api = NinjaExtraAPI(urls_namespace="platform")
-api.register_controllers(NinjaJWTDefaultController)
-
-
-class InvalidRequest(Exception):
- def __init__(self, message: str, status: int = 403):
- self.message = message
- self.status = status
-
-
[email protected]_handler(InvalidRequest)
-def no_permission(request, exc: InvalidRequest):
- return api.create_response(
- request,
- {"status": "error", "message": exc.message},
- status=exc.status,
- )
+router = Router(tags=["Platform"])
def check_email_verified(request):
from_email_address = request.user.emailaddress_set.first()
if not from_email_address.verified:
- raise InvalidRequest("Email address is not verified")
+ raise InvalidRequest("Email address is not verified", status=403)
[email protected]('/register')
+ '/register',
+ summary="Register a new user",
+ description=(
+ "Create a new Mwmbl user account. A confirmation email will be sent to the provided "
+ "address. The account cannot be used until the email is confirmed via the "
+ "`/platform/confirm-email` endpoint."
+ ),
+)
def register(request, registration: Registration):
- # Check for existing user with this username
if MwmblUser.objects.filter(username=registration.username).exists():
raise InvalidRequest("Username already exists")
@@ -58,17 +48,22 @@ def register(request, registration: Registration):
}
[email protected]("/confirm-email")
+ "/confirm-email",
+ summary="Confirm email address",
+ description=(
+ "Confirm a user's email address using the key sent in the confirmation email. "
+ "The `key`, `email`, and `username` must all match the values from the confirmation email."
+ ),
+)
def confirm_email(request, confirm: ConfirmEmail):
confirmation = EmailConfirmationHMAC.from_key(confirm.key)
if confirmation is None:
raise InvalidRequest("Invalid confirmation key")
- # Check the signed email address matches this one
if confirmation.email_address.email != confirm.email:
raise InvalidRequest("Invalid username or email")
- # Check the username matches
user = MwmblUser.objects.get(username=confirm.username)
if user.email != confirm.email:
raise InvalidRequest("Invalid username or email")
@@ -83,13 +78,29 @@ def confirm_email(request, confirm: ConfirmEmail):
}
[email protected]("/protected", auth=JWTAuth())
+ "/protected",
+ auth=JWTAuth(),
+ summary="Test authentication",
+ description=(
+ "A simple endpoint to verify that your JWT token is valid and your email is confirmed. "
+ "Returns a success message if authenticated."
+ ),
+)
def protected(request):
check_email_verified(request)
return {"status": "ok", "message": "You are authenticated!"}
[email protected]("/users/{username}", auth=JWTAuth())
+ "/users/{username}",
+ auth=JWTAuth(),
+ summary="Delete user account",
+ description=(
+ "Permanently delete the authenticated user's account. "
+ "Users can only delete their own account. This action is irreversible."
+ ),
+)
def delete_user(request, username: str):
user = MwmblUser.objects.get(username=username)
if user is None:
@@ -102,19 +113,44 @@ def delete_user(request, username: str):
return {"status": "ok", "message": "User deleted."}
[email protected]("/domain-submissions/domains/{domain}", response=list[DomainSubmissionSchema])
+ "/domain-submissions/domains/{domain}",
+ response=list[DomainSubmissionSchema],
+ summary="Get submissions for a domain",
+ description=(
+ "Retrieve all domain submissions for a specific domain name. "
+ "Results are paginated. Use `limit` and `offset` query parameters to page through results."
+ ),
+)
@paginate
def get_domain_submissions_for_domain(request, domain: str) -> list[DomainSubmissionSchema]:
return DomainSubmission.objects.filter(name=domain).all()
[email protected]("/domain-submissions", response=list[DomainSubmissionSchema])
+ "/domain-submissions",
+ response=list[DomainSubmissionSchema],
+ summary="List all domain submissions",
+ description=(
+ "Retrieve all domain submissions across all users. "
+ "Results are paginated. Use `limit` and `offset` query parameters to page through results."
+ ),
+)
@paginate
def get_domain_submissions(request) -> list[DomainSubmission]:
return DomainSubmission.objects.all()
[email protected]("/domain-submissions/", auth=JWTAuth())
+ "/domain-submissions/",
+ auth=JWTAuth(),
+ summary="Submit a domain for crawling",
+ description=(
+ "Submit a domain name to be considered for inclusion in the Mwmbl crawl queue. "
+ "Submissions are reviewed before the domain is added. "
+ "Requires a verified account."
+ ),
+)
def submit_domain(request, domain: str):
check_email_verified(request)
submission = DomainSubmission(name=domain, submitted_by=request.user)
@@ -122,7 +158,16 @@ def submit_domain(request, domain: str):
return {"status": "ok", "message": "Domain submitted for review."}
[email protected]("/domain-submissions/ids/{submission_id}", auth=JWTAuth())
+ "/domain-submissions/ids/{submission_id}",
+ auth=JWTAuth(),
+ summary="Delete a domain submission",
+ description=(
+ "Delete a domain submission by its ID. "
+ "Users can only delete their own submissions. "
+ "Requires a verified account."
+ ),
+)
def delete_submission(request, submission_id: int):
check_email_verified(request)
submission = DomainSubmission.objects.get(id=submission_id)
@@ -136,14 +181,22 @@ def delete_submission(request, submission_id: int):
return {"status": "ok", "message": "Submission deleted."}
[email protected]("/domain-submissions/ids/{submission_id}", auth=JWTAuth())
+ "/domain-submissions/ids/{submission_id}",
+ auth=JWTAuth(),
+ summary="Update a domain submission status",
+ description=(
+ "Update the status of a domain submission (e.g. approve or reject it). "
+ "Requires the `change_domain_submission_status` permission. "
+ "Requires a verified account."
+ ),
+)
def update_submission_status(request, submission_id: int, update_submission: UpdateDomainSubmission):
check_email_verified(request)
submission = DomainSubmission.objects.get(id=submission_id)
if submission is None:
raise InvalidRequest("Submission not found.", status=404)
- # Can only update if the user has the permission
if not request.user.has_perm("change_domain_submission_status"):
raise InvalidRequest("You do not have permission to update this submission.")
@@ -154,86 +207,90 @@ def update_submission_status(request, submission_id: int, update_submission: Upd
return {"status": "ok", "message": "Submission updated."}
- "/search-results/vote",
+ "/search-results/vote",
auth=JWTAuth(),
summary="Vote on a search result",
- description="Cast an upvote or downvote on a search result for a specific query. "
- "If the user has already voted on this result for this query, the vote will be updated. "
- "Each user can only have one vote per URL per query.",
- tags=["Search Result Voting"]
+ description=(
+ "Cast an upvote or downvote on a search result for a specific query. "
+ "If the user has already voted on this result for this query, the vote will be updated. "
+ "Each user can only have one vote per URL per query. "
+ "Requires a verified account."
+ ),
+ tags=["Search Result Voting"],
)
def vote_on_search_result(request, vote_request: VoteRequest):
check_email_verified(request)
-
- # Validate vote type
+
if vote_request.vote_type not in SearchResultVote.VOTE_TYPES:
raise InvalidRequest("Invalid vote type. Must be 'upvote' or 'downvote'.", status=400)
-
- # Create or update the vote
+
vote, created = SearchResultVote.objects.update_or_create(
user=request.user,
url=vote_request.url,
query=vote_request.query,
defaults={'vote_type': vote_request.vote_type}
)
-
+
action = "created" if created else "updated"
return {"status": "ok", "message": f"Vote {action} successfully."}
- "/search-results/votes",
- response=VoteResponse,
+ "/search-results/votes",
+ response=VoteResponse,
auth=JWTAuth(),
summary="Get vote statistics for search results",
- description="Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a specific search query. "
- "Also returns the current user's vote on each result if they have voted. "
- "This endpoint uses POST to handle large numbers of URLs that would exceed URL length limits.",
- tags=["Search Result Voting"]
+ description=(
+ "Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a "
+ "specific search query. Also returns the current user's vote on each result if they have "
+ "voted. This endpoint uses POST to handle large numbers of URLs that would exceed URL "
+ "length limits. Requires a verified account."
+ ),
+ tags=["Search Result Voting"],
)
def get_vote_counts(request, vote_stats_request: VoteStatsRequest):
check_email_verified(request)
-
+
if not vote_stats_request.urls:
raise InvalidRequest("At least one URL must be provided.", status=400)
-
- # Get vote counts for each URL
+
vote_data = {}
for url in vote_stats_request.urls:
- # Get aggregated vote counts
votes = SearchResultVote.objects.filter(url=url, query=vote_stats_request.query)
upvotes = votes.filter(vote_type='upvote').count()
downvotes = votes.filter(vote_type='downvote').count()
-
- # Get user's vote if any
+
user_vote = None
try:
user_vote_obj = votes.get(user=request.user)
user_vote = user_vote_obj.vote_type
except SearchResultVote.DoesNotExist:
pass
-
+
vote_data[url] = VoteStats(
upvotes=upvotes,
downvotes=downvotes,
user_vote=user_vote
)
-
+
return VoteResponse(votes=vote_data)
- "/search-results/vote",
+ "/search-results/vote",
auth=JWTAuth(),
summary="Remove a vote from a search result",
- description="Remove the current user's vote (upvote or downvote) from a specific search result for a given query. "
- "If the user has not voted on this result for this query, a 404 error will be returned.",
- tags=["Search Result Voting"]
+ description=(
+ "Remove the current user's vote (upvote or downvote) from a specific search result for a "
+ "given query. If the user has not voted on this result for this query, a 404 error will "
+ "be returned. Requires a verified account."
+ ),
+ tags=["Search Result Voting"],
)
def remove_vote(request, vote_request: VoteRemoveRequest):
check_email_verified(request)
-
+
try:
vote = SearchResultVote.objects.get(
user=request.user,
@@ -246,14 +303,17 @@ def remove_vote(request, vote_request: VoteRemoveRequest):
raise InvalidRequest("No vote found to remove.", status=404)
- "/search-results/my-votes",
- response=list[UserVoteHistory],
+ "/search-results/my-votes",
+ response=list[UserVoteHistory],
auth=JWTAuth(),
summary="Get user's voting history",
- description="Retrieve the current user's complete voting history, showing all votes they have cast on search results. "
- "Results are ordered by timestamp (most recent first) and paginated.",
- tags=["Search Result Voting"]
+ description=(
+ "Retrieve the current user's complete voting history, showing all votes they have cast "
+ "on search results. Results are ordered by timestamp (most recent first) and paginated. "
+ "Requires a verified account."
+ ),
+ tags=["Search Result Voting"],
)
@paginate
def get_user_vote_history(request) -> list[SearchResultVote]:
diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.py
index a3b94bc..269830a 100644
--- a/mwmbl/tinysearchengine/search.py
+++ b/mwmbl/tinysearchengine/search.py
@@ -1,7 +1,7 @@
import dataclasses
from logging import getLogger
-from ninja import NinjaAPI
+from ninja import NinjaAPI, Router
from mwmbl.format import format_result
from mwmbl.tinysearchengine.indexer import Document
@@ -12,23 +12,98 @@ logger = getLogger(__name__)
SCORE_THRESHOLD = 0.25
+# Module-level router used by the unified v1 API
+router = Router(tags=["Search"])
-def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:
- router = NinjaAPI(urls_namespace=f"search-{version}")
- @router.get("")
+def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):
+ """Register search routes on the given router or API instance."""
+
+ @r.get(
+ "",
+ summary="Search",
+ description=(
+ "Search the Mwmbl index and return formatted results. "
+ "Results are ranked using a heuristic ranker and filtered to a minimum relevance score. "
+ "Pass the search query as the `s` query parameter."
+ ),
+ )
def search(request, s: str):
+ """
+ Example request:
+ GET /api/v1/search/?s=python+tutorial
+
+ Example response:
+ [
+ {
+ "url": "https://docs.python.org/3/tutorial/",
+ "title": "The Python Tutorial",
+ "extract": "Python is an easy to learn, powerful programming language...",
+ "score": 0.92
+ }
+ ]
+ """
results = ranker.search(s, [])
return [format_result(result, s) for result in results]
- @router.get("/complete")
+ @r.get(
+ "/complete",
+ summary="Autocomplete",
+ description=(
+ "Return autocomplete suggestions for a partial query string. "
+ "Useful for powering search-as-you-type interfaces. "
+ "Pass the partial query as the `q` query parameter."
+ ),
+ )
def complete(request, q: str):
+ """
+ Example request:
+ GET /api/v1/search/complete?q=pyth
+
+ Example response:
+ ["python", "python tutorial", "python documentation"]
+ """
return ranker.complete(q)
- @router.get("/raw")
+ @r.get(
+ "/raw",
+ summary="Raw search results",
+ description=(
+ "Return raw, unformatted search results directly from the index. "
+ "Includes internal scoring fields useful for debugging and evaluation. "
+ "Pass the search query as the `s` query parameter."
+ ),
+ )
def raw(request, s: str):
+ """
+ Example request:
+ GET /api/v1/search/raw?s=python+tutorial
+
+ Example response:
+ {
+ "query": "python tutorial",
+ "results": [
+ {
+ "url": "https://docs.python.org/3/tutorial/",
+ "title": "The Python Tutorial",
+ "extract": "Python is an easy to learn...",
+ "score": 0.92,
+ "term": "python tutorial"
+ }
+ ]
+ }
+ """
results = ranker.get_raw_results(s)
- # Convert dataclass to JSON serializable
return {"query": s, "results": [dataclasses.asdict(result) for result in results]}
- return router
+
+def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:
+ """Create a standalone NinjaAPI for a specific version (used for legacy routes)."""
+ api = NinjaAPI(urls_namespace=f"search-{version}")
+ _register_routes(api, ranker)
+ return api
+
+
+def init_router(ranker: HeuristicRanker):
+ """Initialise the module-level router with the given ranker (called from urls.py)."""
+ _register_routes(router, ranker)
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index cce0937..b602f4d 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -16,17 +16,21 @@ Including another URLconf
"""
from django.contrib import admin
from django.urls import path, include
+from django.views.generic import RedirectView
from debug_toolbar.toolbar import debug_toolbar_urls
import mwmbl.crawler.app as crawler
-from mwmbl.platform.api import api
-from mwmbl.evaluation.api import api as evaluation_api
+from mwmbl.api import api as v1_api, register_routers
from mwmbl.search_setup import queued_batches, ranker, batch_cache
from mwmbl.tinysearchengine import search
from mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \
flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view, CurationsView, submit_domain, \
DomainSubmissionListView, memory_view
+# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.
+# This must be called before urlpatterns is evaluated.
+register_routers(ranker=ranker, batch_cache=batch_cache, queued_batches=queued_batches)
+
def trigger_error(request):
division_by_zero = 1 / 0
@@ -51,15 +55,18 @@ urlpatterns = [
path('app/domains/', domains_view, name="domains"),
path('app/domains/<str:domain>/', domain_view, name="domain"),
- # TODO: this is the old API, deprecated and to be removed once all clients have moved over
+ # TODO: these are the old APIs, deprecated and to be removed once all clients have moved over
path("search/", search.create_router(ranker, "0.1").urls),
path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),
- # New API
- path("api/v1/search/", search.create_router(ranker, "1.0.0").urls),
- path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),
- path("api/v1/platform/", api.urls),
- path("api/v1/evaluate/", evaluation_api.urls),
+ # New unified API — all v1 endpoints, single docs page at /api/v1/docs
+ path("api/v1/", v1_api.urls),
+
+ # Redirects from old per-router docs URLs to the unified docs page
+ path("api/v1/search/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),
+ path("api/v1/crawler/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),
+ path("api/v1/platform/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),
+ path("api/v1/evaluate/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),
path("debug/memory", memory_view, name="memory"),
diff --git a/test/test_openapi.py b/test/test_openapi.py
index 84d223a..d160b67 100644
--- a/test/test_openapi.py
+++ b/test/test_openapi.py
@@ -39,37 +39,37 @@ def access_token(verified_user):
@pytest.mark.django_db
def test_openapi_spec_generation():
- """Test that the OpenAPI spec can be generated without errors"""
+ """Test that the unified OpenAPI spec can be generated without errors"""
client = Client()
-
- # Test that the OpenAPI JSON endpoint works
- response = client.get('/api/v1/platform/openapi.json')
+
+ # The unified docs are now at /api/v1/openapi.json (not per-router)
+ response = client.get('/api/v1/openapi.json')
assert response.status_code == 200
-
+
# Parse the JSON to ensure it's valid
openapi_spec = response.json()
-
+
# Basic OpenAPI spec validation
assert 'openapi' in openapi_spec
assert 'info' in openapi_spec
assert 'paths' in openapi_spec
assert 'components' in openapi_spec
-
- # Check that our voting endpoints are present
+
+ # Check that our voting endpoints are present (mounted under /platform/)
paths = openapi_spec['paths']
assert '/api/v1/platform/search-results/vote' in paths
assert '/api/v1/platform/search-results/votes' in paths
assert '/api/v1/platform/search-results/my-votes' in paths
-
+
# Check that POST endpoint for voting exists
vote_endpoint = paths['/api/v1/platform/search-results/vote']
assert 'post' in vote_endpoint
assert 'delete' in vote_endpoint
-
+
# Check that POST endpoint for vote counts exists (uses POST to handle large URL lists)
votes_endpoint = paths['/api/v1/platform/search-results/votes']
assert 'post' in votes_endpoint
-
+
# Check that GET endpoint for user vote history exists
my_votes_endpoint = paths['/api/v1/platform/search-results/my-votes']
assert 'get' in my_votes_endpoint
@@ -77,49 +77,49 @@ def test_openapi_spec_generation():
@pytest.mark.django_db
def test_openapi_docs_page():
- """Test that the OpenAPI docs page loads without errors"""
+ """Test that the unified OpenAPI docs page loads without errors"""
client = Client()
-
- # Test that the docs page works
- response = client.get('/api/v1/platform/docs')
+
+ # The unified docs page is now at /api/v1/docs
+ response = client.get('/api/v1/docs')
assert response.status_code == 200
assert 'text/html' in response['Content-Type']
@pytest.mark.django_db
def test_voting_schemas_in_openapi():
- """Test that our voting schemas are properly defined in the OpenAPI spec"""
+ """Test that our voting schemas are properly defined in the unified OpenAPI spec"""
client = Client()
-
- response = client.get('/api/v1/platform/openapi.json')
+
+ response = client.get('/api/v1/openapi.json')
assert response.status_code == 200
-
+
openapi_spec = response.json()
components = openapi_spec.get('components', {})
schemas = components.get('schemas', {})
-
+
# Check that our voting schemas are present
assert 'VoteRequest' in schemas
assert 'VoteStats' in schemas
assert 'VoteResponse' in schemas
assert 'UserVoteHistory' in schemas
assert 'VoteRemoveRequest' in schemas
-
+
# Check VoteRequest schema structure
vote_request_schema = schemas['VoteRequest']
assert 'properties' in vote_request_schema
properties = vote_request_schema['properties']
-
+
assert 'url' in properties
assert 'query' in properties
assert 'vote_type' in properties
-
+
# Check that vote_type has the correct enum values
vote_type_property = properties['vote_type']
assert 'enum' in vote_type_property
assert 'upvote' in vote_type_property['enum']
assert 'downvote' in vote_type_property['enum']
-
+
# Check that descriptions are present
assert 'description' in vote_type_property
assert 'description' in properties['url']
@@ -128,36 +128,36 @@ def test_voting_schemas_in_openapi():
@pytest.mark.django_db
def test_voting_endpoint_documentation():
- """Test that voting endpoints have proper documentation in OpenAPI spec"""
+ """Test that voting endpoints have proper documentation in the unified OpenAPI spec"""
client = Client()
-
- response = client.get('/api/v1/platform/openapi.json')
+
+ response = client.get('/api/v1/openapi.json')
assert response.status_code == 200
-
+
openapi_spec = response.json()
paths = openapi_spec['paths']
-
+
# Check POST /search-results/vote endpoint documentation
vote_post = paths['/api/v1/platform/search-results/vote']['post']
assert 'summary' in vote_post
assert 'description' in vote_post
assert 'tags' in vote_post
assert 'Search Result Voting' in vote_post['tags']
-
+
# Check POST /search-results/votes endpoint documentation
votes_post = paths['/api/v1/platform/search-results/votes']['post']
assert 'summary' in votes_post
assert 'description' in votes_post
assert 'tags' in votes_post
assert 'Search Result Voting' in votes_post['tags']
-
+
# Check DELETE /search-results/vote endpoint documentation
vote_delete = paths['/api/v1/platform/search-results/vote']['delete']
assert 'summary' in vote_delete
assert 'description' in vote_delete
assert 'tags' in vote_delete
assert 'Search Result Voting' in vote_delete['tags']
-
+
# Check GET /search-results/my-votes endpoint documentation
my_votes_get = paths['/api/v1/platform/search-results/my-votes']['get']
assert 'summary' in my_votes_get
@@ -168,32 +168,32 @@ def test_voting_endpoint_documentation():
@pytest.mark.django_db
def test_schema_examples_in_openapi():
- """Test that our schemas have proper examples in the OpenAPI spec"""
+ """Test that our schemas have proper examples in the unified OpenAPI spec"""
client = Client()
-
- response = client.get('/api/v1/platform/openapi.json')
+
+ response = client.get('/api/v1/openapi.json')
assert response.status_code == 200
-
+
openapi_spec = response.json()
schemas = openapi_spec['components']['schemas']
-
+
# Check VoteRequest schema has examples
vote_request = schemas['VoteRequest']
properties = vote_request['properties']
-
+
# Check that examples are present
assert 'example' in properties['url']
assert 'example' in properties['query']
assert 'example' in properties['vote_type']
-
+
# Check example values are reasonable
assert 'https://' in properties['url']['example']
assert properties['vote_type']['example'] in ['upvote', 'downvote']
-
+
# Check VoteStats schema has examples
vote_stats = schemas['VoteStats']
vote_stats_properties = vote_stats['properties']
-
+
assert 'example' in vote_stats_properties['upvotes']
assert 'example' in vote_stats_properties['downvotes']
assert isinstance(vote_stats_properties['upvotes']['example'], int)