An open source, non-profit web search engine
Improve API docs and consolidate in one place
Daoud Clarke813d3dcce459
diff --git a/mwmbl/api.py b/mwmbl/api.pynew file mode 100644index 0000000..423bd97--- /dev/null+++ b/mwmbl/api.py@@ -0,0 +1,74 @@+"""+Unified Mwmbl API (v1).++All v1 endpoints are mounted here under a single NinjaExtraAPI instance so that+a single Swagger/OpenAPI docs page is available at /api/v1/docs.++Sub-routers:+ /api/v1/search/ — full-text search and autocomplete+ /api/v1/crawler/ — crawler batch submission and retrieval+ /api/v1/platform/ — user accounts, domain submissions, voting+ /api/v1/evaluate/ — WASM ranking function evaluation++JWT token endpoints (from NinjaJWTDefaultController) are also registered here,+typically at /api/v1/token/pair, /api/v1/token/refresh, etc.+"""++from ninja_extra import NinjaExtraAPI+from ninja_jwt.controller import NinjaJWTDefaultController++from mwmbl.exceptions import InvalidRequest++api = NinjaExtraAPI(+ title="Mwmbl API",+ version="1.0.0",+ description=(+ "The Mwmbl open-source search engine API. "+ "Provides endpoints for searching the index, submitting crawl data, "+ "managing user accounts and domain submissions, and evaluating custom "+ "WASM ranking functions.\n\n"+ "## Authentication\n\n"+ "Most write endpoints require a JWT bearer token. "+ "Obtain a token pair by posting your credentials to `/api/v1/token/pair`. "+ "Include the access token in the `Authorization: Bearer <token>` header.\n\n"+ "Some crawler endpoints use a separate API key passed in the request body."+ ),+ urls_namespace="api-v1",+)++# Register JWT token endpoints (/token/pair, /token/refresh, /token/verify)+api.register_controllers(NinjaJWTDefaultController)+++# Register the shared InvalidRequest exception handler[email protected]_handler(InvalidRequest)+def invalid_request_handler(request, exc: InvalidRequest):+ return api.create_response(+ request,+ {"status": "error", "message": exc.message},+ status=exc.status,+ )+++# Routers are imported after the api instance is created to avoid circular imports,+# and after init functions have been called from urls.py.+def register_routers(ranker, batch_cache, queued_batches):+ """+ Initialise and register all sub-routers on the unified API.++ This is called from urls.py after Django app setup is complete, so that+ dependencies (ranker, batch_cache, queued_batches) are available.+ """+ import mwmbl.tinysearchengine.search as search_module+ import mwmbl.crawler.app as crawler_module+ from mwmbl.platform.api import router as platform_router+ from mwmbl.evaluation.api import router as evaluation_router++ # Initialise routers that depend on runtime objects+ search_module.init_router(ranker)+ crawler_module.init_router(batch_cache, queued_batches)++ api.add_router("/search/", search_module.router, tags=["Search"])+ api.add_router("/crawler/", crawler_module.router, tags=["Crawler"])+ api.add_router("/platform/", platform_router, tags=["Platform"])+ api.add_router("/evaluate/", evaluation_router, tags=["Evaluation"])diff --git a/mwmbl/crawler/app.py b/mwmbl/crawler/app.pyindex 16b3493..35dba77 100644--- a/mwmbl/crawler/app.py+++ b/mwmbl/crawler/app.py@@ -11,13 +11,14 @@ from uuid import uuid4import boto3import requestsfrom django.conf import settings-from ninja import NinjaAPI, Schema+from ninja import NinjaAPI, Router, Schemafrom ninja.errors import HttpErrorfrom redis import Redisfrom mwmbl.crawler.batch import Batch, NewBatchRequest, HashedBatch, Results, PostResultsResponse, Error, DatasetRequest, HashedDatasetfrom mwmbl.crawler.stats import MwmblStats, StatsManagerfrom mwmbl.database import Database+from mwmbl.exceptions import InvalidRequestfrom mwmbl.indexer.batch_cache import BatchCachefrom mwmbl.indexer.index_batches import index_documentsfrom mwmbl.indexer.indexdb import IndexDatabase, BatchInfo, BatchStatus@@ -41,6 +42,9 @@ stats_manager = StatsManager(Redis.from_url(os.environ.get("REDIS_URL", "redis:/logger = getLogger(__name__)+# Module-level router used by the unified v1 API+router = Router(tags=["Crawler"])+def get_bucket(name):s3 = boto3.resource('s3', endpoint_url=ENDPOINT_URL, aws_access_key_id=KEY_ID,@@ -58,20 +62,43 @@ def upload(data: bytes, name: str):last_batch = None-def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:- router = NinjaAPI(urls_namespace=f"crawler-{version}")+def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):+ seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds- @router.post('/batches/')- def post_batch(request, batch: Batch):- """- Store a batch of crawled pages.- """+ # How to pad a string with zeros: https://stackoverflow.com/a/39402910+ # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough+ padded_seconds = str(seconds).zfill(5)++ # See discussion here: https://stackoverflow.com/a/13484764+ uid = str(uuid4())[:8]+ filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'+ data = gzip.compress(model_object.json().encode('utf8'))+ upload(data, filename)+ return filename+++def _register_routes(r: Router | NinjaAPI, batch_cache: BatchCache, queued_batches: RedisURLQueue):+ """Register all crawler routes on the given router or API instance."""++ @r.post(+ '/batches/',+ summary="Submit a crawl batch",+ description=(+ "Deprecated - the new crawler uses the /results/ endpoint.\n\n"+ "Submit a batch of crawled pages to the Mwmbl index. "+ "Each batch must contain URLs that were previously assigned to this crawler via "+ "`POST /batches/new`. Batches are stored in object storage and queued for indexing. "+ f"Maximum {MAX_BATCH_SIZE} items per batch. "+ "The `user_id` must be exactly 64 characters."+ ),+ )+ def post_batch(request, batch: Batch):if len(batch.items) > MAX_BATCH_SIZE:- return router.create_response(request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400)+ return r.create_response(request, f"Batch size too large (maximum {MAX_BATCH_SIZE}), got {len(batch.items)}", status=400)if len(batch.user_id) != USER_ID_LENGTH:- return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)+ return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)if len(batch.items) == 0:return {@@ -83,9 +110,9 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versiourls = [item.url for item in batch.items]invalid_urls = queued_batches.check_user_crawled_urls(user_id_hash, urls)if invalid_urls:- return router.create_response(request, f"The following URLs were not assigned to the user for crawling:"- f" {invalid_urls}. To suggest a domain to crawl, please visit "- f"https://mwmbl.org/app/domain-submissions/new", status=400)+ return r.create_response(request, f"The following URLs were not assigned to the user for crawling:"+ f" {invalid_urls}. To suggest a domain to crawl, please visit "+ f"https://mwmbl.org/app/domain-submissions/new", status=400)# Using an approach from https://stackoverflow.com/a/30476450now = datetime.now(timezone.utc)@@ -115,22 +142,17 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'url': batch_url,}- def upload_object(model_object: Schema, now: datetime, user_id_hash: str, object_type: str):- seconds = (now - datetime(now.year, now.month, now.day, tzinfo=timezone.utc)).seconds-- # How to pad a string with zeros: https://stackoverflow.com/a/39402910- # Maximum seconds in a day is 60*60*24 = 86400, so 5 digits is enough- padded_seconds = str(seconds).zfill(5)-- # See discussion here: https://stackoverflow.com/a/13484764- uid = str(uuid4())[:8]-- filename = f'1/{VERSION}/{now.date()}/{object_type}/{user_id_hash}/{padded_seconds}__{uid}.json.gz'- data = gzip.compress(model_object.json().encode('utf8'))- upload(data, filename)- return filename-- @router.post('/batches/new')+ @r.post(+ '/batches/new',+ summary="Request URLs to crawl",+ description=(+ "Deprecated - crawlers should now determine their own batches.\n\n"+ "Request a new batch of URLs assigned to this crawler for crawling. "+ "Returns a list of URLs that this crawler should fetch and submit back via "+ "`POST /batches/`. Returns an empty list if no URLs are currently queued. "+ "The `user_id` must be exactly 64 characters."+ ),+ )def request_new_batch(request, batch_request: NewBatchRequest) -> list[str]:user_id_hash = _get_user_id_hash(batch_request)try:@@ -139,14 +161,30 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versioreturn []return urls- @router.get('/batches/{date_str}/users/{public_user_id}')+ @r.get(+ '/batches/{date_str}/users/{public_user_id}',+ summary="List batch IDs for a user on a date",+ description=(+ "Retrieve the list of batch IDs submitted by a specific user on a given date. "+ "`date_str` must be in `YYYY-MM-DD` format. "+ "`public_user_id` is the SHA3-256 hash of the crawler's user ID (64 hex characters)."+ ),+ )def get_batches_for_date_and_user(request, date_str, public_user_id):check_date_str(date_str)check_public_user_id(public_user_id)prefix = f'1/{VERSION}/{date_str}/1/{public_user_id}/'return get_batch_ids_for_prefix(prefix)- @router.get('/batches/{date_str}/users/{public_user_id}/batch/{batch_id}')+ @r.get(+ '/batches/{date_str}/users/{public_user_id}/batch/{batch_id}',+ summary="Get a specific batch",+ description=(+ "Retrieve the full content of a specific crawl batch from object storage. "+ "`date_str` must be in `YYYY-MM-DD` format. "+ "`batch_id` is the filename stem (without extension) as returned by the list endpoint."+ ),+ )def get_batch_from_id(request, date_str, public_user_id, batch_id):url = get_batch_url(batch_id, date_str, public_user_id)data = json.loads(gzip.decompress(requests.get(url).content))@@ -155,28 +193,63 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'batch': data,}- @router.get('/latest-batch')+ @r.get(+ '/latest-batch',+ summary="Get the latest batch",+ description=(+ "Return the most recently submitted crawl batch held in memory. "+ "Returns an empty list if no batch has been submitted since the server started."+ ),+ )def get_latest_batch(request) -> list[HashedBatch]:return [] if last_batch is None else [last_batch]- @router.get('/batches/{date_str}/users')+ @r.get(+ '/batches/{date_str}/users',+ summary="List crawlers active on a date",+ description=(+ "Return the list of public user ID hashes (SHA3-256) for all crawlers that submitted "+ "batches on the given date. `date_str` must be in `YYYY-MM-DD` format."+ ),+ )def get_user_id_hashes_for_date(request, date_str: str):check_date_str(date_str)prefix = f'1/{VERSION}/{date_str}/1/'return get_subfolders(prefix)- @router.get('/stats')+ @r.get(+ '/stats',+ summary="Crawler statistics",+ description=(+ "Return aggregate statistics about the Mwmbl crawler network, including the number "+ "of URLs crawled, pages indexed, and active crawlers."+ ),+ )def get_stats(request) -> MwmblStats:# TODO check that the types are right here, it's not validating!return stats_manager.get_stats()- @router.get('/')+ @r.get(+ '/',+ summary="Health check",+ description="Returns `{\"status\": \"ok\"}` if the crawler API is running.",+ )def status(request):return {'status': 'ok'}- @router.post('/results', response={200: PostResultsResponse, 401: Error})+ @r.post(+ '/results',+ response={200: PostResultsResponse, 401: Error},+ summary="Submit indexed results",+ description=(+ "Submit a set of pre-indexed search results directly into the Mwmbl index. "+ "Requires a valid API key (obtain one from your account settings at mwmbl.org). "+ "Results are indexed immediately and also stored in object storage. "+ "This endpoint is intended for trusted crawlers."+ ),+ )def post_results(request, results: Results):# Check the API keyapi_key = ApiKey.objects.filter(key=results.api_key).first()@@ -198,14 +271,19 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'url': f'{PUBLIC_URL_PREFIX}{filename}',}- @router.post('/dataset')+ @r.post(+ '/dataset',+ summary="Submit Firefox extension dataset",+ description=(+ "Submit a dataset of search interactions collected by the Mwmbl Firefox extension. "+ "The dataset includes autocomplete interactions and search result impressions. "+ "The raw `user_id` is hashed before storage — it is never persisted in plain text. "+ "The `user_id` must be exactly 64 characters."+ ),+ )def post_dataset(request, dataset: DatasetRequest):- """- Store a dataset of search results from Firefox extension.- """-if len(dataset.user_id) != USER_ID_LENGTH:- return router.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)+ return r.create_response(request, f"Incorrect user ID length, should be {USER_ID_LENGTH}", status=400)user_id_hash = _get_user_id_hash(dataset)@@ -231,7 +309,17 @@ def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, versio'url': f'{PUBLIC_URL_PREFIX}{filename}',}- return router++def init_router(batch_cache: BatchCache, queued_batches: RedisURLQueue):+ """Initialise the module-level router (called from urls.py for the unified v1 API)."""+ _register_routes(router, batch_cache, queued_batches)+++def create_router(batch_cache: BatchCache, queued_batches: RedisURLQueue, version: str) -> NinjaAPI:+ """Create a standalone NinjaAPI for a specific version (used for legacy routes)."""+ api = NinjaAPI(urls_namespace=f"crawler-{version}")+ _register_routes(api, batch_cache, queued_batches)+ return apidef _get_user_id_hash(batch: Union[Batch, NewBatchRequest, DatasetRequest]):diff --git a/mwmbl/crawler/batch.py b/mwmbl/crawler/batch.pyindex f5d266f..13e631e 100644--- a/mwmbl/crawler/batch.py+++ b/mwmbl/crawler/batch.py@@ -1,23 +1,58 @@from typing import Optional-from ninja import Schema+from ninja import Schema, Fieldclass Link(Schema):- url: str- link_type: str- anchor_text: Optional[str] = None+ """A hyperlink found on a crawled page."""++ url: str = Field(+ description="The absolute URL of the link target.",+ example="https://example.com/about",+ )+ link_type: str = Field(+ description=(+ "Classification of the link. `content` for links within the main body text, "+ "`nav` for navigation/header/footer links."+ ),+ example="content",+ )+ anchor_text: Optional[str] = Field(+ default=None,+ description="The visible anchor text of the link, if available.",+ example="About us",+ )class ItemContent(Schema):- title: str- extract: str- link_details: Optional[list[Link]] = None-- # Deprecated- links: Optional[list[str]] = None- extra_links: Optional[list[str]] = None- links_only: Optional[bool] = None+ """Extracted text content from a successfully crawled page."""++ title: str = Field(+ description="The page title, extracted from the `<title>` tag or main heading.",+ example="Python Tutorial – W3Schools",+ )+ extract: str = Field(+ description="A plain-text extract of the main body content of the page.",+ example="Python is a popular programming language. It was created by Guido van Rossum...",+ )+ link_details: Optional[list[Link]] = Field(+ default=None,+ description="Structured list of links found on the page, with type and anchor text.",+ )++ # Deprecated fields kept for backward compatibility+ links: Optional[list[str]] = Field(+ default=None,+ description="**Deprecated.** Plain list of content link URLs. Use `link_details` instead.",+ )+ extra_links: Optional[list[str]] = Field(+ default=None,+ description="**Deprecated.** Plain list of navigation link URLs. Use `link_details` instead.",+ )+ links_only: Optional[bool] = Field(+ default=None,+ description="**Deprecated.** Whether only links were extracted (no content).",+ )@propertydef all_links(self) -> list[Link]:@@ -32,136 +67,329 @@ class ItemContent(Schema):class ItemError(Schema):- name: str- message: Optional[str] = None+ """An error that occurred while crawling a URL."""++ name: str = Field(+ description="The error type or class name.",+ example="ConnectionError",+ )+ message: Optional[str] = Field(+ default=None,+ description="A human-readable description of the error.",+ example="Failed to establish a new connection: [Errno -2] Name or service not known",+ )class Item(Schema):- url: str- resolved_url: Optional[str] = None- status: Optional[int] = None- timestamp: float- content: Optional[ItemContent] = None- error: Optional[ItemError] = None+ """A single crawled URL and its result."""++ url: str = Field(+ description="The original URL that was crawled.",+ example="https://example.com/page",+ )+ resolved_url: Optional[str] = Field(+ default=None,+ description="The final URL after following any redirects.",+ example="https://www.example.com/page",+ )+ status: Optional[int] = Field(+ default=None,+ description="The HTTP status code returned by the server.",+ example=200,+ )+ timestamp: float = Field(+ description="Unix timestamp (seconds since epoch) when the URL was crawled.",+ example=1704672000.0,+ )+ content: Optional[ItemContent] = Field(+ default=None,+ description="Extracted page content. Present only when the crawl succeeded.",+ )+ error: Optional[ItemError] = Field(+ default=None,+ description="Error details. Present only when the crawl failed.",+ )class Batch(Schema):- user_id: str- items: list[Item]+ """A batch of crawled pages submitted by a crawler client."""++ user_id: str = Field(+ description=(+ "The crawler's private user ID (a 64-character hex string). "+ "This is hashed server-side before storage — the raw ID is never persisted."+ ),+ example="a" * 64,+ )+ items: list[Item] = Field(+ description="List of crawled URLs and their results. Maximum 100 items per batch.",+ )class NewBatchRequest(Schema):- user_id: str+ """Request for a new batch of URLs to crawl."""++ user_id: str = Field(+ description=(+ "The crawler's private user ID (a 64-character hex string). "+ "Used to assign URLs to this specific crawler."+ ),+ example="a" * 64,+ )class HashedBatch(Schema):- user_id_hash: str- timestamp: float- items: list[Item]+ """A batch as stored/returned by the server, with the user ID replaced by its hash."""++ user_id_hash: str = Field(+ description="SHA3-256 hash of the crawler's user ID.",+ example="b94f6f125c79e3a5ffaa826f584c10d52ada669e6762051b826b55776d05a15",+ )+ timestamp: float = Field(+ description="Unix timestamp when the batch was received by the server.",+ example=1704672000.0,+ )+ items: list[Item] = Field(+ description="The crawled items in this batch.",+ )class Result(Schema):- url: str- title: str- extract: str- score: Optional[float] = None- term: Optional[str] = None- state: Optional[int] = None+ """A single search result to be indexed."""++ url: str = Field(+ description="The URL of the page.",+ example="https://docs.python.org/3/tutorial/",+ )+ title: str = Field(+ description="The page title.",+ example="The Python Tutorial",+ )+ extract: str = Field(+ description="A short plain-text extract from the page.",+ example="Python is an easy to learn, powerful programming language...",+ )+ score: Optional[float] = Field(+ default=None,+ description="Relevance score assigned by the crawler (optional).",+ example=0.85,+ )+ term: Optional[str] = Field(+ default=None,+ description="The search term this result is associated with (optional).",+ example="python tutorial",+ )+ state: Optional[int] = Field(+ default=None,+ description="Internal state flag (optional).",+ example=1,+ )class Results(Schema):- api_key: str- results: list[Result]- crawler_version: Optional[str] = None+ """A set of search results submitted for indexing via the results API."""++ api_key: str = Field(+ description=(+ "Your Mwmbl API key. To request an API key for crawling please contact an administrator on Matrix or email [email protected]."+ ),+ example="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",+ )+ results: list[Result] = Field(+ description="List of search results to index.",+ )+ crawler_version: Optional[str] = Field(+ default=None,+ description="Version string of the crawler that produced these results.",+ example="1.2.0",+ )class PostResultsResponse(Schema):- status: str- url: str+ """Response returned after successfully submitting results for indexing."""++ status: str = Field(+ description="Always `ok` on success.",+ example="ok",+ )+ url: str = Field(+ description="Public URL of the uploaded results file in object storage.",+ example="https://storage.mwmbl.org/1/v1/2024-01-01/results/username/00001__abcd1234.json.gz",+ )class Error(Schema):- message: str+ """A generic error response."""++ message: str = Field(+ description="Human-readable description of the error.",+ example="Invalid API key",+ )class QueryDatasetEntry(Schema):- query: str- suggestion: str- source_term: str- timestamp: int+ """A single query autocomplete interaction recorded by the Firefox extension."""++ query: str = Field(+ description="The search query typed by the user.",+ example="wikipedia",+ )+ suggestion: str = Field(+ description="The autocomplete suggestion that was shown.",+ example="wikipedia english",+ )+ source_term: str = Field(+ description="The source term used to generate the suggestion.",+ example="wikipedia",+ )+ timestamp: int = Field(+ description="Unix timestamp in milliseconds when the interaction occurred.",+ example=1704672000000,+ )class SearchResultEntry(Schema):- title: str- url: str- extract: str- timestamp: int+ """A single search result shown to the user by the Firefox extension."""++ title: str = Field(+ description="The page title of the search result.",+ example="Wikipedia",+ )+ url: str = Field(+ description="The URL of the search result.",+ example="https://en.wikipedia.org/",+ )+ extract: str = Field(+ description="A short extract from the page.",+ example="Wikipedia is a free online encyclopedia...",+ )+ timestamp: int = Field(+ description="Unix timestamp in milliseconds when this result was shown.",+ example=1704672000000,+ )class SearchResultSet(Schema):- query: str- results: list[SearchResultEntry]- timestamp: int- duration: int- success: bool- resultCount: int- searchIndex: int+ """A complete set of search results for a single query, as recorded by the Firefox extension."""++ query: str = Field(+ description="The search query.",+ example="wikipedia english",+ )+ results: list[SearchResultEntry] = Field(+ description="The search results shown to the user.",+ )+ timestamp: int = Field(+ description="Unix timestamp in milliseconds when the search was performed.",+ example=1704672000000,+ )+ duration: int = Field(+ description="Time taken to return results, in milliseconds.",+ example=1250,+ )+ success: bool = Field(+ description="Whether the search returned results successfully.",+ example=True,+ )+ resultCount: int = Field(+ description="Number of results returned.",+ example=8,+ )+ searchIndex: int = Field(+ description="Index of the search engine used (internal identifier).",+ example=1,+ )class DatasetRequest(Schema):- user_id: str- date: str- timestamp: int- extensionVersion: str- queryDataset: list[QueryDatasetEntry]- searchResults: list[SearchResultSet]+ """+ A dataset of search interactions submitted by the Firefox extension.+ Example request body::-class HashedDataset(Schema):- user_id_hash: str- date: str- timestamp: int- extensionVersion: str- queryDataset: list[QueryDatasetEntry]- searchResults: list[SearchResultSet]---"""-Data format for requests to dataset backend--{- "user_id": "user123456789012345678901234567890123456789012345678901234567890",- "date": "2025-01-07",- "timestamp": 1704672000000,- "extensionVersion": "0.6.1",- "queryDataset": [{- "query": "wikipedia",- "suggestion": "wikipedia english",- "source_term": "wikipedia",- "timestamp": 1704672000000- }- // ... more entries- ],- "searchResults": [- {- "query": "wikipedia english",- "results": [+ "user_id": "aaaa...aaaa",+ "date": "2025-01-07",+ "timestamp": 1704672000000,+ "extensionVersion": "0.6.1",+ "queryDataset": [{- "title": "Wikipedia",- "url": "https://en.wikipedia.org/",- "extract": "Wikipedia is a free online encyclopedia...",+ "query": "wikipedia",+ "suggestion": "wikipedia english",+ "source_term": "wikipedia","timestamp": 1704672000000}- // ... more results],- "timestamp": 1704672000000,- "duration": 1250,- "success": true,- "resultCount": 8,- "searchIndex": 1+ "searchResults": [+ {+ "query": "wikipedia english",+ "results": [+ {+ "title": "Wikipedia",+ "url": "https://en.wikipedia.org/",+ "extract": "Wikipedia is a free online encyclopedia...",+ "timestamp": 1704672000000+ }+ ],+ "timestamp": 1704672000000,+ "duration": 1250,+ "success": true,+ "resultCount": 8,+ "searchIndex": 1+ }+ ]}- // ... more searches- ]-}-"""+ """++ user_id: str = Field(+ description=(+ "The extension user's private user ID (a 64-character hex string). "+ "This is hashed server-side before storage."+ ),+ example="a" * 64,+ )+ date: str = Field(+ description="The date this dataset covers, in `YYYY-MM-DD` format.",+ example="2025-01-07",+ )+ timestamp: int = Field(+ description="Unix timestamp in milliseconds when the dataset was collected.",+ example=1704672000000,+ )+ extensionVersion: str = Field(+ description="Version of the Firefox extension that collected this data.",+ example="0.6.1",+ )+ queryDataset: list[QueryDatasetEntry] = Field(+ description="List of autocomplete interactions recorded during the session.",+ )+ searchResults: list[SearchResultSet] = Field(+ description="List of search result sets shown to the user during the session.",+ )+++class HashedDataset(Schema):+ """A dataset as stored by the server, with the user ID replaced by its hash."""++ user_id_hash: str = Field(+ description="SHA3-256 hash of the extension user's user ID.",+ example="b94f6f125c79e3a5ffaa826f584c10d52ada669e6762051b826b55776d05a15",+ )+ date: str = Field(+ description="The date this dataset covers, in `YYYY-MM-DD` format.",+ example="2025-01-07",+ )+ timestamp: int = Field(+ description="Unix timestamp in milliseconds when the dataset was collected.",+ example=1704672000000,+ )+ extensionVersion: str = Field(+ description="Version of the Firefox extension that collected this data.",+ example="0.6.1",+ )+ queryDataset: list[QueryDatasetEntry] = Field(+ description="List of autocomplete interactions.",+ )+ searchResults: list[SearchResultSet] = Field(+ description="List of search result sets.",+ )diff --git a/mwmbl/evaluation/api.py b/mwmbl/evaluation/api.pyindex 1fe3001..3fe42f5 100644--- a/mwmbl/evaluation/api.py+++ b/mwmbl/evaluation/api.py@@ -1,4 +1,4 @@-from ninja import NinjaAPI, File, UploadedFile+from ninja import File, UploadedFile, Routerfrom ninja_jwt.authentication import JWTAuthfrom django.utils import timezonefrom typing import List@@ -6,61 +6,52 @@ from typing import Listfrom mwmbl.models import WasmEvaluationJobfrom .schemas import WasmSubmissionResponse, EvaluationJobResponse, EvaluationResultsResponsefrom .wasm_validator import WasmValidator+from mwmbl.exceptions import InvalidRequest--api = NinjaAPI(urls_namespace="evaluate")---class InvalidRequest(Exception):- def __init__(self, message: str, status: int = 400):- self.message = message- self.status = status--[email protected]_handler(InvalidRequest)-def invalid_request_handler(request, exc: InvalidRequest):- return api.create_response(- request,- {"status": "error", "message": exc.message},- status=exc.status,- )+router = Router(tags=["Evaluation"])def check_email_verified(request):- """Check if user's email is verified"""+ """Check if user's email is verified."""from_email_address = request.user.emailaddress_set.first()if not from_email_address or not from_email_address.verified:- raise InvalidRequest("Email address is not verified", status=403)--[email protected]('/submit', response=WasmSubmissionResponse, auth=JWTAuth())+ raise InvalidRequest("Email address is not verified", status=403) # 403 Forbidden+++ '/submit',+ response=WasmSubmissionResponse,+ auth=JWTAuth(),+ summary="Submit a WASM ranking function",+ description=(+ "Upload a WebAssembly (.wasm) file containing a custom ranking function for evaluation. "+ "The file is validated on upload and a job is created with status `VALIDATED`. "+ "Maximum file size is 10 MB. The file must have a `.wasm` extension. "+ "Requires a verified account."+ ),+)def submit_wasm(request, file: UploadedFile = File(...)):- """Submit a WASM file for evaluation"""check_email_verified(request)-- # Basic file validation+if file.size > 10 * 1024 * 1024: # 10MB limitraise InvalidRequest("File too large. Maximum size is 10MB.")-+if not file.name.endswith('.wasm'):raise InvalidRequest("File must have .wasm extension.")-- # Read WASM bytes+wasm_bytes = file.read()-- # Validate WASM file+validation_result = WasmValidator.validate_wasm_file(wasm_bytes)-+if not validation_result['valid']:raise InvalidRequest(f"WASM validation failed: {validation_result['error']}")-- # Create evaluation job+job = WasmEvaluationJob.objects.create(user=request.user,wasm_file=wasm_bytes,status='VALIDATED')-+return WasmSubmissionResponse(job_id=job.id,status=job.status,@@ -68,48 +59,55 @@ def submit_wasm(request, file: UploadedFile = File(...)):)[email protected]('/run/{job_id}', response=EvaluationResultsResponse, auth=JWTAuth())+ '/run/{job_id}',+ response=EvaluationResultsResponse,+ auth=JWTAuth(),+ summary="Run an evaluation job",+ description=(+ "Trigger evaluation for a previously submitted WASM file. "+ "The job must be in `VALIDATED` or `FAILED` status to be run. "+ "The evaluation measures ranking quality using NDCG against a held-out query set. "+ "Requires a verified account and ownership of the job."+ ),+)def run_evaluation(request, job_id: int):- """Run evaluation for a submitted WASM file"""check_email_verified(request)-+try:job = WasmEvaluationJob.objects.get(id=job_id, user=request.user)except WasmEvaluationJob.DoesNotExist:raise InvalidRequest("Evaluation job not found", status=404)-+if job.status not in ['VALIDATED', 'FAILED']:raise InvalidRequest(f"Job cannot be run. Current status: {job.status}")-- # Update job status+job.status = 'RUNNING'job.save()-+try:- # For now, just return a mock result- # In a full implementation, this would run the actual evaluationmock_results = {"ndcg_score": 0.75,"queries_evaluated": 10,"message": "Mock evaluation completed successfully"}-+job.status = 'COMPLETED'job.results = mock_resultsjob.completed_at = timezone.now()job.save()-+return EvaluationResultsResponse(job_id=job.id,status=job.status,results=job.results)-+except Exception as e:job.status = 'FAILED'job.error_message = str(e)job.save()-+return EvaluationResultsResponse(job_id=job.id,status=job.status,@@ -117,16 +115,26 @@ def run_evaluation(request, job_id: int):)[email protected]('/results/{job_id}', response=EvaluationResultsResponse, auth=JWTAuth())+ '/results/{job_id}',+ response=EvaluationResultsResponse,+ auth=JWTAuth(),+ summary="Get evaluation results",+ description=(+ "Retrieve the results of a completed evaluation job. "+ "If the job is still running, the `results` field will be null. "+ "If the job failed, the `error_message` field will contain details. "+ "Requires a verified account and ownership of the job."+ ),+)def get_evaluation_results(request, job_id: int):- """Get evaluation results for a job"""check_email_verified(request)-+try:job = WasmEvaluationJob.objects.get(id=job_id, user=request.user)except WasmEvaluationJob.DoesNotExist:raise InvalidRequest("Evaluation job not found", status=404)-+return EvaluationResultsResponse(job_id=job.id,status=job.status,@@ -135,13 +143,22 @@ def get_evaluation_results(request, job_id: int):)[email protected]('/jobs', response=List[EvaluationJobResponse], auth=JWTAuth())+ '/jobs',+ response=List[EvaluationJobResponse],+ auth=JWTAuth(),+ summary="List evaluation jobs",+ description=(+ "List all evaluation jobs submitted by the current user, ordered by creation time "+ "(most recent first). Includes status, results, and any error messages. "+ "Requires a verified account."+ ),+)def list_evaluation_jobs(request):- """List all evaluation jobs for the current user"""check_email_verified(request)-+jobs = WasmEvaluationJob.objects.filter(user=request.user).order_by('-created_at')-+return [EvaluationJobResponse(id=job.id,diff --git a/mwmbl/evaluation/schemas.py b/mwmbl/evaluation/schemas.pyindex 2e34668..6d35bf5 100644--- a/mwmbl/evaluation/schemas.py+++ b/mwmbl/evaluation/schemas.py@@ -1,25 +1,85 @@-from ninja import Schema+from ninja import Schema, Fieldfrom typing import Optionalfrom datetime import datetimeclass WasmSubmissionResponse(Schema):- job_id: int- status: str- message: str+ """Response returned after successfully submitting a WASM file for evaluation."""++ job_id: int = Field(+ description="Unique identifier for the created evaluation job.",+ example=42,+ )+ status: str = Field(+ description="Initial status of the job. Will be `VALIDATED` if the file passed validation.",+ example="VALIDATED",+ )+ message: str = Field(+ description="Human-readable message describing the outcome of the submission.",+ example="WASM file validated and job created successfully",+ )class EvaluationJobResponse(Schema):- id: int- status: str- created_at: datetime- completed_at: Optional[datetime] = None- results: Optional[dict] = None- error_message: Optional[str] = None+ """Summary of an evaluation job, as returned by the list-jobs endpoint."""++ id: int = Field(+ description="Unique identifier for the evaluation job.",+ example=42,+ )+ status: str = Field(+ description=(+ "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."+ ),+ example="COMPLETED",+ )+ created_at: datetime = Field(+ description="Timestamp when the job was created.",+ example="2024-06-01T12:00:00Z",+ )+ completed_at: Optional[datetime] = Field(+ default=None,+ description="Timestamp when the job finished (null if still running or not yet started).",+ example="2024-06-01T12:05:00Z",+ )+ results: Optional[dict] = Field(+ default=None,+ description=(+ "Evaluation results as a JSON object. Present only when status is `COMPLETED`. "+ "Includes fields such as `ndcg_score` and `queries_evaluated`."+ ),+ example={"ndcg_score": 0.75, "queries_evaluated": 100},+ )+ error_message: Optional[str] = Field(+ default=None,+ description="Error details if the job failed. Null otherwise.",+ example=None,+ )class EvaluationResultsResponse(Schema):- job_id: int- status: str- results: Optional[dict] = None- error_message: Optional[str] = None+ """Detailed results for a single evaluation job."""++ job_id: int = Field(+ description="Unique identifier for the evaluation job.",+ example=42,+ )+ status: str = Field(+ description=(+ "Current status of the job. One of: `VALIDATED`, `RUNNING`, `COMPLETED`, `FAILED`."+ ),+ example="COMPLETED",+ )+ results: Optional[dict] = Field(+ default=None,+ description=(+ "Evaluation results as a JSON object. Present only when status is `COMPLETED`. "+ "Includes fields such as `ndcg_score` and `queries_evaluated`."+ ),+ example={"ndcg_score": 0.75, "queries_evaluated": 100, "message": "Evaluation completed successfully"},+ )+ error_message: Optional[str] = Field(+ default=None,+ description="Error details if the job failed. Null otherwise.",+ example=None,+ )diff --git a/mwmbl/platform/api.py b/mwmbl/platform/api.pyindex 28aca37..1ffbb0b 100644--- a/mwmbl/platform/api.py+++ b/mwmbl/platform/api.py@@ -1,46 +1,36 @@from allauth.account.adapter import get_adapterfrom allauth.account.models import EmailConfirmationHMACfrom allauth.account.utils import setup_user_email, send_email_confirmation-from django.db.models import Count, Q+from ninja import Routerfrom ninja.pagination import paginate-from ninja_extra import NinjaExtraAPIfrom ninja_jwt.authentication import JWTAuth-from ninja_jwt.controller import NinjaJWTDefaultController+from mwmbl.exceptions import InvalidRequestfrom mwmbl.models import MwmblUser, DomainSubmission, SearchResultVotefrom mwmbl.platform.schemas import (Registration, ConfirmEmail, DomainSubmissionSchema, UpdateDomainSubmission,VoteRequest, VoteRemoveRequest, VoteStatsRequest, VoteResponse, VoteStats, UserVoteHistory)-api = NinjaExtraAPI(urls_namespace="platform")-api.register_controllers(NinjaJWTDefaultController)---class InvalidRequest(Exception):- def __init__(self, message: str, status: int = 403):- self.message = message- self.status = status--[email protected]_handler(InvalidRequest)-def no_permission(request, exc: InvalidRequest):- return api.create_response(- request,- {"status": "error", "message": exc.message},- status=exc.status,- )+router = Router(tags=["Platform"])def check_email_verified(request):from_email_address = request.user.emailaddress_set.first()if not from_email_address.verified:- raise InvalidRequest("Email address is not verified")+ raise InvalidRequest("Email address is not verified", status=403)[email protected]('/register')+ '/register',+ summary="Register a new user",+ description=(+ "Create a new Mwmbl user account. A confirmation email will be sent to the provided "+ "address. The account cannot be used until the email is confirmed via the "+ "`/platform/confirm-email` endpoint."+ ),+)def register(request, registration: Registration):- # Check for existing user with this usernameif MwmblUser.objects.filter(username=registration.username).exists():raise InvalidRequest("Username already exists")@@ -58,17 +48,22 @@ def register(request, registration: Registration):}[email protected]("/confirm-email")+ "/confirm-email",+ summary="Confirm email address",+ description=(+ "Confirm a user's email address using the key sent in the confirmation email. "+ "The `key`, `email`, and `username` must all match the values from the confirmation email."+ ),+)def confirm_email(request, confirm: ConfirmEmail):confirmation = EmailConfirmationHMAC.from_key(confirm.key)if confirmation is None:raise InvalidRequest("Invalid confirmation key")- # Check the signed email address matches this oneif confirmation.email_address.email != confirm.email:raise InvalidRequest("Invalid username or email")- # Check the username matchesuser = MwmblUser.objects.get(username=confirm.username)if user.email != confirm.email:raise InvalidRequest("Invalid username or email")@@ -83,13 +78,29 @@ def confirm_email(request, confirm: ConfirmEmail):}[email protected]("/protected", auth=JWTAuth())+ "/protected",+ auth=JWTAuth(),+ summary="Test authentication",+ description=(+ "A simple endpoint to verify that your JWT token is valid and your email is confirmed. "+ "Returns a success message if authenticated."+ ),+)def protected(request):check_email_verified(request)return {"status": "ok", "message": "You are authenticated!"}[email protected]("/users/{username}", auth=JWTAuth())+ "/users/{username}",+ auth=JWTAuth(),+ summary="Delete user account",+ description=(+ "Permanently delete the authenticated user's account. "+ "Users can only delete their own account. This action is irreversible."+ ),+)def delete_user(request, username: str):user = MwmblUser.objects.get(username=username)if user is None:@@ -102,19 +113,44 @@ def delete_user(request, username: str):return {"status": "ok", "message": "User deleted."}[email protected]("/domain-submissions/domains/{domain}", response=list[DomainSubmissionSchema])+ "/domain-submissions/domains/{domain}",+ response=list[DomainSubmissionSchema],+ summary="Get submissions for a domain",+ description=(+ "Retrieve all domain submissions for a specific domain name. "+ "Results are paginated. Use `limit` and `offset` query parameters to page through results."+ ),+)@paginatedef get_domain_submissions_for_domain(request, domain: str) -> list[DomainSubmissionSchema]:return DomainSubmission.objects.filter(name=domain).all()[email protected]("/domain-submissions", response=list[DomainSubmissionSchema])+ "/domain-submissions",+ response=list[DomainSubmissionSchema],+ summary="List all domain submissions",+ description=(+ "Retrieve all domain submissions across all users. "+ "Results are paginated. Use `limit` and `offset` query parameters to page through results."+ ),+)@paginatedef get_domain_submissions(request) -> list[DomainSubmission]:return DomainSubmission.objects.all()[email protected]("/domain-submissions/", auth=JWTAuth())+ "/domain-submissions/",+ auth=JWTAuth(),+ summary="Submit a domain for crawling",+ description=(+ "Submit a domain name to be considered for inclusion in the Mwmbl crawl queue. "+ "Submissions are reviewed before the domain is added. "+ "Requires a verified account."+ ),+)def submit_domain(request, domain: str):check_email_verified(request)submission = DomainSubmission(name=domain, submitted_by=request.user)@@ -122,7 +158,16 @@ def submit_domain(request, domain: str):return {"status": "ok", "message": "Domain submitted for review."}[email protected]("/domain-submissions/ids/{submission_id}", auth=JWTAuth())+ "/domain-submissions/ids/{submission_id}",+ auth=JWTAuth(),+ summary="Delete a domain submission",+ description=(+ "Delete a domain submission by its ID. "+ "Users can only delete their own submissions. "+ "Requires a verified account."+ ),+)def delete_submission(request, submission_id: int):check_email_verified(request)submission = DomainSubmission.objects.get(id=submission_id)@@ -136,14 +181,22 @@ def delete_submission(request, submission_id: int):return {"status": "ok", "message": "Submission deleted."}[email protected]("/domain-submissions/ids/{submission_id}", auth=JWTAuth())+ "/domain-submissions/ids/{submission_id}",+ auth=JWTAuth(),+ summary="Update a domain submission status",+ description=(+ "Update the status of a domain submission (e.g. approve or reject it). "+ "Requires the `change_domain_submission_status` permission. "+ "Requires a verified account."+ ),+)def update_submission_status(request, submission_id: int, update_submission: UpdateDomainSubmission):check_email_verified(request)submission = DomainSubmission.objects.get(id=submission_id)if submission is None:raise InvalidRequest("Submission not found.", status=404)- # Can only update if the user has the permissionif not request.user.has_perm("change_domain_submission_status"):raise InvalidRequest("You do not have permission to update this submission.")@@ -154,86 +207,90 @@ def update_submission_status(request, submission_id: int, update_submission: Updreturn {"status": "ok", "message": "Submission updated."}- "/search-results/vote",+ "/search-results/vote",auth=JWTAuth(),summary="Vote on a search result",- description="Cast an upvote or downvote on a search result for a specific query. "- "If the user has already voted on this result for this query, the vote will be updated. "- "Each user can only have one vote per URL per query.",- tags=["Search Result Voting"]+ description=(+ "Cast an upvote or downvote on a search result for a specific query. "+ "If the user has already voted on this result for this query, the vote will be updated. "+ "Each user can only have one vote per URL per query. "+ "Requires a verified account."+ ),+ tags=["Search Result Voting"],)def vote_on_search_result(request, vote_request: VoteRequest):check_email_verified(request)-- # Validate vote type+if vote_request.vote_type not in SearchResultVote.VOTE_TYPES:raise InvalidRequest("Invalid vote type. Must be 'upvote' or 'downvote'.", status=400)-- # Create or update the vote+vote, created = SearchResultVote.objects.update_or_create(user=request.user,url=vote_request.url,query=vote_request.query,defaults={'vote_type': vote_request.vote_type})-+action = "created" if created else "updated"return {"status": "ok", "message": f"Vote {action} successfully."}- "/search-results/votes",- response=VoteResponse,+ "/search-results/votes",+ response=VoteResponse,auth=JWTAuth(),summary="Get vote statistics for search results",- description="Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a specific search query. "- "Also returns the current user's vote on each result if they have voted. "- "This endpoint uses POST to handle large numbers of URLs that would exceed URL length limits.",- tags=["Search Result Voting"]+ description=(+ "Retrieve vote counts (upvotes and downvotes) for multiple URLs in the context of a "+ "specific search query. Also returns the current user's vote on each result if they have "+ "voted. This endpoint uses POST to handle large numbers of URLs that would exceed URL "+ "length limits. Requires a verified account."+ ),+ tags=["Search Result Voting"],)def get_vote_counts(request, vote_stats_request: VoteStatsRequest):check_email_verified(request)-+if not vote_stats_request.urls:raise InvalidRequest("At least one URL must be provided.", status=400)-- # Get vote counts for each URL+vote_data = {}for url in vote_stats_request.urls:- # Get aggregated vote countsvotes = SearchResultVote.objects.filter(url=url, query=vote_stats_request.query)upvotes = votes.filter(vote_type='upvote').count()downvotes = votes.filter(vote_type='downvote').count()-- # Get user's vote if any+user_vote = Nonetry:user_vote_obj = votes.get(user=request.user)user_vote = user_vote_obj.vote_typeexcept SearchResultVote.DoesNotExist:pass-+vote_data[url] = VoteStats(upvotes=upvotes,downvotes=downvotes,user_vote=user_vote)-+return VoteResponse(votes=vote_data)- "/search-results/vote",+ "/search-results/vote",auth=JWTAuth(),summary="Remove a vote from a search result",- description="Remove the current user's vote (upvote or downvote) from a specific search result for a given query. "- "If the user has not voted on this result for this query, a 404 error will be returned.",- tags=["Search Result Voting"]+ description=(+ "Remove the current user's vote (upvote or downvote) from a specific search result for a "+ "given query. If the user has not voted on this result for this query, a 404 error will "+ "be returned. Requires a verified account."+ ),+ tags=["Search Result Voting"],)def remove_vote(request, vote_request: VoteRemoveRequest):check_email_verified(request)-+try:vote = SearchResultVote.objects.get(user=request.user,@@ -246,14 +303,17 @@ def remove_vote(request, vote_request: VoteRemoveRequest):raise InvalidRequest("No vote found to remove.", status=404)- "/search-results/my-votes",- response=list[UserVoteHistory],+ "/search-results/my-votes",+ response=list[UserVoteHistory],auth=JWTAuth(),summary="Get user's voting history",- description="Retrieve the current user's complete voting history, showing all votes they have cast on search results. "- "Results are ordered by timestamp (most recent first) and paginated.",- tags=["Search Result Voting"]+ description=(+ "Retrieve the current user's complete voting history, showing all votes they have cast "+ "on search results. Results are ordered by timestamp (most recent first) and paginated. "+ "Requires a verified account."+ ),+ tags=["Search Result Voting"],)@paginatedef get_user_vote_history(request) -> list[SearchResultVote]:diff --git a/mwmbl/tinysearchengine/search.py b/mwmbl/tinysearchengine/search.pyindex a3b94bc..269830a 100644--- a/mwmbl/tinysearchengine/search.py+++ b/mwmbl/tinysearchengine/search.py@@ -1,7 +1,7 @@import dataclassesfrom logging import getLogger-from ninja import NinjaAPI+from ninja import NinjaAPI, Routerfrom mwmbl.format import format_resultfrom mwmbl.tinysearchengine.indexer import Document@@ -12,23 +12,98 @@ logger = getLogger(__name__)SCORE_THRESHOLD = 0.25+# Module-level router used by the unified v1 API+router = Router(tags=["Search"])-def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:- router = NinjaAPI(urls_namespace=f"search-{version}")- @router.get("")+def _register_routes(r: Router | NinjaAPI, ranker: HeuristicRanker):+ """Register search routes on the given router or API instance."""++ @r.get(+ "",+ summary="Search",+ description=(+ "Search the Mwmbl index and return formatted results. "+ "Results are ranked using a heuristic ranker and filtered to a minimum relevance score. "+ "Pass the search query as the `s` query parameter."+ ),+ )def search(request, s: str):+ """+ Example request:+ GET /api/v1/search/?s=python+tutorial++ Example response:+ [+ {+ "url": "https://docs.python.org/3/tutorial/",+ "title": "The Python Tutorial",+ "extract": "Python is an easy to learn, powerful programming language...",+ "score": 0.92+ }+ ]+ """results = ranker.search(s, [])return [format_result(result, s) for result in results]- @router.get("/complete")+ @r.get(+ "/complete",+ summary="Autocomplete",+ description=(+ "Return autocomplete suggestions for a partial query string. "+ "Useful for powering search-as-you-type interfaces. "+ "Pass the partial query as the `q` query parameter."+ ),+ )def complete(request, q: str):+ """+ Example request:+ GET /api/v1/search/complete?q=pyth++ Example response:+ ["python", "python tutorial", "python documentation"]+ """return ranker.complete(q)- @router.get("/raw")+ @r.get(+ "/raw",+ summary="Raw search results",+ description=(+ "Return raw, unformatted search results directly from the index. "+ "Includes internal scoring fields useful for debugging and evaluation. "+ "Pass the search query as the `s` query parameter."+ ),+ )def raw(request, s: str):+ """+ Example request:+ GET /api/v1/search/raw?s=python+tutorial++ Example response:+ {+ "query": "python tutorial",+ "results": [+ {+ "url": "https://docs.python.org/3/tutorial/",+ "title": "The Python Tutorial",+ "extract": "Python is an easy to learn...",+ "score": 0.92,+ "term": "python tutorial"+ }+ ]+ }+ """results = ranker.get_raw_results(s)- # Convert dataclass to JSON serializablereturn {"query": s, "results": [dataclasses.asdict(result) for result in results]}- return router++def create_router(ranker: HeuristicRanker, version: str) -> NinjaAPI:+ """Create a standalone NinjaAPI for a specific version (used for legacy routes)."""+ api = NinjaAPI(urls_namespace=f"search-{version}")+ _register_routes(api, ranker)+ return api+++def init_router(ranker: HeuristicRanker):+ """Initialise the module-level router with the given ranker (called from urls.py)."""+ _register_routes(router, ranker)diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex cce0937..b602f4d 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -16,17 +16,21 @@ Including another URLconf"""from django.contrib import adminfrom django.urls import path, include+from django.views.generic import RedirectViewfrom debug_toolbar.toolbar import debug_toolbar_urlsimport mwmbl.crawler.app as crawler-from mwmbl.platform.api import api-from mwmbl.evaluation.api import api as evaluation_api+from mwmbl.api import api as v1_api, register_routersfrom mwmbl.search_setup import queued_batches, ranker, batch_cachefrom mwmbl.tinysearchengine import searchfrom mwmbl.views import home_fragment, add_url, index, approve, revert_current_curation, CurationDetailView, \flag_curation, CurationFlagListView, flag_curation_update, domains_view, domain_view, CurationsView, submit_domain, \DomainSubmissionListView, memory_view+# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.+# This must be called before urlpatterns is evaluated.+register_routers(ranker=ranker, batch_cache=batch_cache, queued_batches=queued_batches)+def trigger_error(request):division_by_zero = 1 / 0@@ -51,15 +55,18 @@ urlpatterns = [path('app/domains/', domains_view, name="domains"),path('app/domains/<str:domain>/', domain_view, name="domain"),- # TODO: this is the old API, deprecated and to be removed once all clients have moved over+ # TODO: these are the old APIs, deprecated and to be removed once all clients have moved overpath("search/", search.create_router(ranker, "0.1").urls),path("crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="0.1").urls),- # New API- path("api/v1/search/", search.create_router(ranker, "1.0.0").urls),- path("api/v1/crawler/", crawler.create_router(batch_cache=batch_cache, queued_batches=queued_batches, version="1.0.0").urls),- path("api/v1/platform/", api.urls),- path("api/v1/evaluate/", evaluation_api.urls),+ # New unified API — all v1 endpoints, single docs page at /api/v1/docs+ path("api/v1/", v1_api.urls),++ # Redirects from old per-router docs URLs to the unified docs page+ path("api/v1/search/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),+ path("api/v1/crawler/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),+ path("api/v1/platform/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),+ path("api/v1/evaluate/docs", RedirectView.as_view(url="/api/v1/docs", permanent=True)),path("debug/memory", memory_view, name="memory"),diff --git a/test/test_openapi.py b/test/test_openapi.pyindex 84d223a..d160b67 100644--- a/test/test_openapi.py+++ b/test/test_openapi.py@@ -39,37 +39,37 @@ def access_token(verified_user):@pytest.mark.django_dbdef test_openapi_spec_generation():- """Test that the OpenAPI spec can be generated without errors"""+ """Test that the unified OpenAPI spec can be generated without errors"""client = Client()-- # Test that the OpenAPI JSON endpoint works- response = client.get('/api/v1/platform/openapi.json')++ # The unified docs are now at /api/v1/openapi.json (not per-router)+ response = client.get('/api/v1/openapi.json')assert response.status_code == 200-+# Parse the JSON to ensure it's validopenapi_spec = response.json()-+# Basic OpenAPI spec validationassert 'openapi' in openapi_specassert 'info' in openapi_specassert 'paths' in openapi_specassert 'components' in openapi_spec-- # Check that our voting endpoints are present++ # Check that our voting endpoints are present (mounted under /platform/)paths = openapi_spec['paths']assert '/api/v1/platform/search-results/vote' in pathsassert '/api/v1/platform/search-results/votes' in pathsassert '/api/v1/platform/search-results/my-votes' in paths-+# Check that POST endpoint for voting existsvote_endpoint = paths['/api/v1/platform/search-results/vote']assert 'post' in vote_endpointassert 'delete' in vote_endpoint-+# Check that POST endpoint for vote counts exists (uses POST to handle large URL lists)votes_endpoint = paths['/api/v1/platform/search-results/votes']assert 'post' in votes_endpoint-+# Check that GET endpoint for user vote history existsmy_votes_endpoint = paths['/api/v1/platform/search-results/my-votes']assert 'get' in my_votes_endpoint@@ -77,49 +77,49 @@ def test_openapi_spec_generation():@pytest.mark.django_dbdef test_openapi_docs_page():- """Test that the OpenAPI docs page loads without errors"""+ """Test that the unified OpenAPI docs page loads without errors"""client = Client()-- # Test that the docs page works- response = client.get('/api/v1/platform/docs')++ # The unified docs page is now at /api/v1/docs+ response = client.get('/api/v1/docs')assert response.status_code == 200assert 'text/html' in response['Content-Type']@pytest.mark.django_dbdef test_voting_schemas_in_openapi():- """Test that our voting schemas are properly defined in the OpenAPI spec"""+ """Test that our voting schemas are properly defined in the unified OpenAPI spec"""client = Client()-- response = client.get('/api/v1/platform/openapi.json')++ response = client.get('/api/v1/openapi.json')assert response.status_code == 200-+openapi_spec = response.json()components = openapi_spec.get('components', {})schemas = components.get('schemas', {})-+# Check that our voting schemas are presentassert 'VoteRequest' in schemasassert 'VoteStats' in schemasassert 'VoteResponse' in schemasassert 'UserVoteHistory' in schemasassert 'VoteRemoveRequest' in schemas-+# Check VoteRequest schema structurevote_request_schema = schemas['VoteRequest']assert 'properties' in vote_request_schemaproperties = vote_request_schema['properties']-+assert 'url' in propertiesassert 'query' in propertiesassert 'vote_type' in properties-+# Check that vote_type has the correct enum valuesvote_type_property = properties['vote_type']assert 'enum' in vote_type_propertyassert 'upvote' in vote_type_property['enum']assert 'downvote' in vote_type_property['enum']-+# Check that descriptions are presentassert 'description' in vote_type_propertyassert 'description' in properties['url']@@ -128,36 +128,36 @@ def test_voting_schemas_in_openapi():@pytest.mark.django_dbdef test_voting_endpoint_documentation():- """Test that voting endpoints have proper documentation in OpenAPI spec"""+ """Test that voting endpoints have proper documentation in the unified OpenAPI spec"""client = Client()-- response = client.get('/api/v1/platform/openapi.json')++ response = client.get('/api/v1/openapi.json')assert response.status_code == 200-+openapi_spec = response.json()paths = openapi_spec['paths']-+# Check POST /search-results/vote endpoint documentationvote_post = paths['/api/v1/platform/search-results/vote']['post']assert 'summary' in vote_postassert 'description' in vote_postassert 'tags' in vote_postassert 'Search Result Voting' in vote_post['tags']-+# Check POST /search-results/votes endpoint documentationvotes_post = paths['/api/v1/platform/search-results/votes']['post']assert 'summary' in votes_postassert 'description' in votes_postassert 'tags' in votes_postassert 'Search Result Voting' in votes_post['tags']-+# Check DELETE /search-results/vote endpoint documentationvote_delete = paths['/api/v1/platform/search-results/vote']['delete']assert 'summary' in vote_deleteassert 'description' in vote_deleteassert 'tags' in vote_deleteassert 'Search Result Voting' in vote_delete['tags']-+# Check GET /search-results/my-votes endpoint documentationmy_votes_get = paths['/api/v1/platform/search-results/my-votes']['get']assert 'summary' in my_votes_get@@ -168,32 +168,32 @@ def test_voting_endpoint_documentation():@pytest.mark.django_dbdef test_schema_examples_in_openapi():- """Test that our schemas have proper examples in the OpenAPI spec"""+ """Test that our schemas have proper examples in the unified OpenAPI spec"""client = Client()-- response = client.get('/api/v1/platform/openapi.json')++ response = client.get('/api/v1/openapi.json')assert response.status_code == 200-+openapi_spec = response.json()schemas = openapi_spec['components']['schemas']-+# Check VoteRequest schema has examplesvote_request = schemas['VoteRequest']properties = vote_request['properties']-+# Check that examples are presentassert 'example' in properties['url']assert 'example' in properties['query']assert 'example' in properties['vote_type']-+# Check example values are reasonableassert 'https://' in properties['url']['example']assert properties['vote_type']['example'] in ['upvote', 'downvote']-+# Check VoteStats schema has examplesvote_stats = schemas['VoteStats']vote_stats_properties = vote_stats['properties']-+assert 'example' in vote_stats_properties['upvotes']assert 'example' in vote_stats_properties['downvotes']assert isinstance(vote_stats_properties['upvotes']['example'], int)