An open source, non-profit web search engine
Add a Combined Search endpoint over Mwmbl, Staan and Wikipedia
Daoud Clarke97bb2f83105d
diff --git a/mwmbl/api.py b/mwmbl/api.pyindex 63f9143..951bc56 100644--- a/mwmbl/api.py+++ b/mwmbl/api.py@@ -10,7 +10,9 @@ Sub-routers (v1):/api/v1/platform/ — user accounts, domain submissions, votingSub-routers (v2):- /api/v2/search/ — full-text search with optional API-key auth and quota info+ /api/v2/search/ — full-text search with optional API-key auth and quota info+ /api/v2/super-search/ — multi-source streaming search (SSE), authenticated+ /api/v2/combined-search/ — Mwmbl + Staan + Wikipedia in one ranked response, authenticatedJWT token endpoints (from NinjaJWTDefaultController) are registered on v1,typically at /api/v1/platform/token/pair, /api/v1/platform/token/refresh, etc.@@ -131,14 +133,15 @@ def v2_invalid_request_handler(request, exc: InvalidRequest):# Routers are imported after the api instances are created to avoid circular imports,# and after init functions have been called from urls.py.-def register_routers(ranker):+def register_routers(ranker, combined_ranker):"""Initialise and register all sub-routers on the v1 and v2 APIs.This is called from urls.py after Django app setup is complete, so that- dependencies (ranker) are available.+ dependencies (the rankers) are available."""import mwmbl.crawler.app as crawler_module+ import mwmbl.tinysearchengine.combined_search as combined_search_moduleimport mwmbl.tinysearchengine.search as search_moduleimport mwmbl.tinysearchengine.super_search as super_search_modulefrom mwmbl.platform.api import router as platform_router@@ -147,6 +150,7 @@ def register_routers(ranker):search_module.init_router(ranker)search_module.init_v2_router(ranker)super_search_module.init_router()+ combined_search_module.init_router(combined_ranker)crawler_module.init_router()api.add_router("/search/", search_module.router, tags=["Search"])@@ -155,3 +159,4 @@ def register_routers(ranker):v2_api.add_router("/search/", search_module.v2_router, tags=["Search"])v2_api.add_router("/super-search/", super_search_module.router, tags=["Super Search"])+ v2_api.add_router("/combined-search/", combined_search_module.router, tags=["Combined Search"])diff --git a/mwmbl/format.py b/mwmbl/format.pyindex ce34372..bcc2d8b 100644--- a/mwmbl/format.py+++ b/mwmbl/format.py@@ -1,6 +1,6 @@import re-from mwmbl.tinysearchengine.indexer import DocumentState+from mwmbl.tinysearchengine.indexer import DocumentSource, DocumentStatefrom mwmbl.tokenizer import clean_unicode, tokenizeDOCUMENT_SOURCES = {@@ -13,6 +13,14 @@ DOCUMENT_SOURCES = {DocumentState.FROM_WIKI_APPROVED: "wikipedia",}+# What each external provider is called on the wire. DocumentSource is append-only and every+# member must appear here, or a result from a new provider cannot be named - test_format+# asserts the map is complete rather than leaving that to a KeyError in a response.+DOCUMENT_PROVIDERS = {+ DocumentSource.WIKIPEDIA: "wikipedia",+ DocumentSource.STAAN: "staan",+}+HIGHLIGHT_STOPWORDS = {# Articles & Determiners@@ -113,6 +121,19 @@ def get_document_source(state: DocumentState):return DOCUMENT_SOURCES.get(state, "mwmbl")+def get_result_source(result):+ """Where a result came from, for the API's `source` / `engine` field.++ DocumentSource names the external provider a document was fetched from, so it is the more+ specific answer and wins when set; DocumentState answers for everything that came out of+ our own index. Without this a Staan result would report as "mwmbl": it has no+ DocumentState of its own, because it never enters the index.+ """+ if result.source is not None:+ return DOCUMENT_PROVIDERS[result.source]+ return get_document_source(result.state)++def format_result_with_pattern(pattern, result):formatted_result = {}for content_type, content_raw in [("title", result.title), ("extract", result.extract)]:@@ -128,7 +149,7 @@ def format_result_with_pattern(pattern, result):content_result.append({"value": content[start:end], "is_bold": is_bold})formatted_result[content_type] = content_resultformatted_result["url"] = result.url- formatted_result["source"] = get_document_source(result.state)+ formatted_result["source"] = get_result_source(result)return formatted_result@@ -187,6 +208,6 @@ def format_result_v2(result, position: int, query: str) -> dict:"title_highlights": _extract_highlights(v1["title"]),"content": content,"content_highlights": _extract_highlights(v1["extract"]),- "engine": get_document_source(result.state),+ "engine": get_result_source(result),"score": 1.0 / position,}diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.pyindex 8af54b4..ffd5828 100644--- a/mwmbl/indexer/external_cache.py+++ b/mwmbl/indexer/external_cache.py@@ -138,19 +138,26 @@ def external_cache_term(query: str) -> str:return digest.hexdigest()[:16]-def is_fresh(document: Document, now: int) -> bool:- """Whether a cached document is still within its TTL. Untimestamped means stale.+def _ttl_seconds(document: Document) -> int:+ """How long this entry stays fresh.++ An empty-result sentinel expires sooner than a real entry, and a provider whose results+ turn over faster than an encyclopedia's gets its own TTL. document.source is the only+ thing that identifies a provider here - this runs over a page's other entries too, which+ belong to other queries and other providers, so the term says nothing about them.- This runs over a page's other entries too, which belong to other queries and other- providers, so it cannot recover the source from the term - document.source is the only- thing that identifies them, and it is where a per-source TTL would key off.+ Built per call rather than at import: a setting read at import time is a setting that+ override_settings cannot move, and the tests do move it."""- ttl = (- settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS- if document.url == EXTERNAL_CACHE_EMPTY_URL- else settings.EXTERNAL_CACHE_TTL_SECONDS- )- return document.last_crawled is not None and now - document.last_crawled < ttl+ if document.url == EXTERNAL_CACHE_EMPTY_URL:+ return settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS+ ttl_by_source = {DocumentSource.STAAN: settings.EXTERNAL_CACHE_STAAN_TTL_SECONDS}+ return ttl_by_source.get(document.source, settings.EXTERNAL_CACHE_TTL_SECONDS)+++def is_fresh(document: Document, now: int) -> bool:+ """Whether a cached document is still within its TTL. Untimestamped means stale."""+ return document.last_crawled is not None and now - document.last_crawled < _ttl_seconds(document)def get_cached_external_results(diff --git a/mwmbl/quota.py b/mwmbl/quota.pyindex ef66072..a46f297 100644--- a/mwmbl/quota.py+++ b/mwmbl/quota.py@@ -34,6 +34,13 @@ def _super_search_monthly_key(user_id: int, year: int | None = None, month: intreturn f"super_search:monthly:{user_id}:{y}:{m:02d}"+def _combined_search_monthly_key(user_id: int, year: int | None = None, month: int | None = None) -> str:+ now = datetime.now(timezone.utc)+ y = year if year is not None else now.year+ m = month if month is not None else now.month+ return f"combined_search:monthly:{user_id}:{y}:{m:02d}"++def _rate_key(user_id: int) -> str:return f"search:rate:{user_id}"@@ -110,6 +117,33 @@ def decrement_monthly_super_search(user_id: int) -> None:pass+def get_monthly_combined_search_count(user_id: int) -> int:+ """Return the current monthly combined-search request count for a user (0 if not set)."""+ return cache.get(_combined_search_monthly_key(user_id), default=0)+++def increment_monthly_combined_search(user_id: int) -> int:+ """Increment the monthly combined-search counter and return the new value."""+ key = _combined_search_monthly_key(user_id)+ if cache.add(key, 1, timeout=MONTHLY_TTL):+ return 1+ return cache.incr(key)+++def decrement_monthly_combined_search(user_id: int) -> None:+ """Refund one combined-search increment (e.g. when the request is rejected over-limit).++ Never drops below 0. No-op if the counter is missing.+ """+ key = _combined_search_monthly_key(user_id)+ try:+ if cache.get(key, default=0) > 0:+ cache.decr(key)+ except ValueError:+ # decr() raises if the key vanished between the get and the decr; ignore.+ pass++# ---------------------------------------------------------------------------# Key scanning (used by background jobs only — requires django-redis)# ---------------------------------------------------------------------------diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 812818c..0cdbe23 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -25,3 +25,17 @@ ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))# passed 3 here while mwmbl.rankeval.ltr.dataset trained on 5, so the model was ranking a# pool a size it had never been fitted to.ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True))++# Combined Search gets its own model and its own ranker, so retraining on the pooled+# Mwmbl + Staan + Wikipedia candidate set never moves standard search's ranking. Until+# model-combined.xgb has been trained the endpoint serves the deployed model, which is a+# working answer rather than a missing endpoint - the artifact ships in Stage 4.+combined_model_path = Path(settings.COMBINED_MODEL_PATH)+if not combined_model_path.exists():+ combined_model_path = Path(settings.RUST_MODEL_PATH)+combined_ltr_model = RustXGBPipeline.from_model_path(str(combined_model_path))++# include_wiki=False on purpose: the endpoint fetches Wikipedia itself so that call can run+# concurrently with Staan's, and passes both in as additional_results. Leaving it True would+# fetch Wikipedia a second time, serially, inside get_results.+combined_ranker = MMRRanker(LTRRanker(tiny_index, completer, combined_ltr_model, include_wiki=False))diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex b511bdd..9752484 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -285,6 +285,21 @@ CURRENT_AGREEMENT_VERSIONS = {"TERMS_OF_SERVICE_API": "v2026-04-A",}+# Combined Search - the v2 endpoint that pools the Mwmbl index, Staan and Wikipedia and+# ranks the union with its own LTR model. Gated behind login at a flat monthly limit, like+# Super Search, which it is meant to replace.+COMBINED_SEARCH_MONTHLY_LIMIT = 100+# Its own model artifact, so Combined Search can be retrained on the pooled candidate set+# without moving standard search's ranking. Falls back to RUST_MODEL_PATH until trained.+COMBINED_MODEL_PATH = Path(__file__).parent / "resources" / "model-combined.xgb"++# Staan, the external web-search provider Combined Search pools in. Results are cached in+# the external results index - see mwmbl.tinysearchengine.staan.+STAAN_SEARCH_API_KEY = os.environ.get("STAAN_SEARCH_API_KEY", "")+STAAN_SEARCH_URL = os.environ.get("STAAN_SEARCH_URL", "https://api.staan.ai/v2/search/web")+STAAN_MARKET = os.environ.get("STAAN_MARKET", "en-us")+STAAN_TIMEOUT_SECONDS = 5+# Super SearchSUPER_SEARCH_MONTHLY_LIMIT = 100SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawling@@ -345,10 +360,15 @@ EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower(# Counting search traffic (mwmbl.traffic). Off in the test settings: it is the only thing in# the request path that reaches for Redis, and the CI test job runs none.SEARCH_TRAFFIC_COUNTING = os.environ.get("SEARCH_TRAFFIC_COUNTING", "true").lower() != "false"-# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six-# months; a provider whose results turn over faster wants its own, and the place to add that-# is here, keyed by source.+# The default TTL, for any provider that does not name its own below. Wikipedia articles+# move rarely, which is what justifies six months.EXTERNAL_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks+# A provider that turns over faster gets its own, read by external_cache.is_fresh. A general+# web-search index moves far faster than an encyclopedia, so six months of Staan results+# would be six months of stale answers. One scalar per provider rather than a dict keyed by+# DocumentSource: the settings module must not import the indexer, which would pull the Rust+# extension in at settings-import time.+EXTERNAL_CACHE_STAAN_TTL_SECONDS = 7 * 24 * 60 * 60 # 1 week# A query a provider has nothing for is worth remembering too, or it is re-fetched forever.# Shorter, because a result appearing is a likelier change than an existing one moving.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.pynew file mode 100644index 0000000..00aac26--- /dev/null+++ b/mwmbl/tinysearchengine/combined_search.py@@ -0,0 +1,135 @@+"""Combined Search - one request over the Mwmbl index, Staan and Wikipedia.++Pools three sources, ranks the union with a model of its own, and answers with the same+SearXNG-shaped JSON as /api/v2/search/. Authentication is required and a flat monthly quota+applies, exactly as for Super Search, which this endpoint is meant to replace.++There is deliberately no streaming and no crawling here. Super Search streams because it+crawls promoted pages and follows their outbound links, which takes seconds; all three+sources here answer in well under a second, and two of the three are cached, so there is+nothing to stream and a plain response is what a client actually wants.++The ranking core is three calls, because Ranker.get_results already pools+``pages + additional_results + external_search_items`` and blacklist-filters the lot. Staan+and Wikipedia go in as additional_results, fetched concurrently; the index side and the+ranking are the ranker's own job.++Nothing found here is written back to the search index. Super Search indexes what it finds;+whether third-party SERP results belong in our index is a separate question with its own+answer, and this endpoint does not pre-empt it.+"""++import asyncio+from logging import getLogger++from asgiref.sync import sync_to_async+from django.conf import settings+from ninja import Router+from ninja.errors import HttpError++from mwmbl.format import format_result_v2+from mwmbl.quota import (+ check_rate_limit,+ decrement_monthly_combined_search,+ increment_monthly_combined_search,+)+from mwmbl.search_auth import authenticate_user+from mwmbl.tinysearchengine.rank import get_wiki_results+from mwmbl.tinysearchengine.search import SearchResponse+from mwmbl.tinysearchengine.staan import get_staan_results++logger = getLogger(__name__)++router = Router(tags=["Combined Search"])+++DESCRIPTION = (+ "Search the Mwmbl index, Staan and Wikipedia in one request and return the ranked "+ "union.\n\n"+ "Every candidate - crawled by Mwmbl, returned by Staan, or fetched from Wikipedia - is "+ "scored by one learning-to-rank model trained on the pooled candidate set, then "+ "diversified so a single domain cannot take the whole page. The response is the same "+ "SearXNG-compatible shape as `/api/v2/search/`.\n\n"+ "The `engine` field names the provider a result came from:\n"+ "- `mwmbl` - organically crawled by the Mwmbl crawler\n"+ "- `staan` - returned by the Staan web-search API\n"+ "- `wikipedia` - fetched from Wikipedia\n"+ "- `google`, `user` - originally suggested via Google, or submitted by a user\n\n"+ "Authentication is required: a search-scoped API key in `X-API-Key`, or a JWT bearer "+ "token. Obtain a key via `POST /api/v1/platform/api-keys/`. A per-user monthly quota "+ "applies; `monthly_usage` and `monthly_limit` report it on every response.\n\n"+ "A source that is down or unconfigured costs the request its extra recall, not its "+ "results: the remaining sources are ranked and returned as usual.\n\n"+ "**Query parameter:** `q` - the search query string (required)."+)++OPENAPI_EXTRA = {+ "parameters": [+ {+ "name": "q",+ "in": "query",+ "required": True,+ "schema": {"type": "string", "example": "rust async runtimes"},+ }+ ]+}+++async def _gather_external(query: str):+ """Staan's and Wikipedia's results, fetched concurrently.++ Both are synchronous, network-bound and independent, so they run on threads and the+ request waits once rather than twice. Each returns [] on failure by itself.+ """+ return await asyncio.gather(+ asyncio.to_thread(get_staan_results, query),+ asyncio.to_thread(get_wiki_results, query),+ )+++def init_router(ranker) -> None:+ @router.get(+ "",+ response=SearchResponse,+ # Handled manually in the view so both an API key and a JWT work under an async+ # view - the same reason Super Search does it this way.+ auth=None,+ summary="Combined Search (Mwmbl + Staan + Wikipedia)",+ description=DESCRIPTION,+ openapi_extra=OPENAPI_EXTRA,+ )+ async def combined_search(request, q: str):+ user = await authenticate_user(request)++ if not await sync_to_async(check_rate_limit)(user.id):+ raise HttpError(429, "Rate limit exceeded: maximum 5 requests per second.")++ monthly_limit = settings.COMBINED_SEARCH_MONTHLY_LIMIT+ # Increment first, then check: this makes the quota check atomic under concurrent+ # requests (a check-then-increment would let racing requests both pass). Refund the+ # increment if the caller is over the limit.+ monthly_usage = await sync_to_async(increment_monthly_combined_search)(user.id)+ if monthly_usage > monthly_limit:+ await sync_to_async(decrement_monthly_combined_search)(user.id)+ raise HttpError(+ 429,+ f"Combined Search monthly quota exceeded: {monthly_limit} requests per month "+ f"and you have used {monthly_usage - 1}.",+ )++ staan_results, wiki_results = await _gather_external(q)+ results = await asyncio.to_thread(+ ranker.search,+ q,+ staan_results + wiki_results,+ False, # use_external_search=False+ )++ formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]+ return SearchResponse(+ query=q,+ number_of_results=len(formatted),+ results=formatted,+ monthly_usage=monthly_usage,+ monthly_limit=monthly_limit,+ )diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex e954695..6b3d97b 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -270,7 +270,19 @@ def fix_document_state(result: Document):fixed_state = DocumentState(result.state)except ValueError:fixed_state = None- fixed_document = Document(result.title, result.url, result.extract, result.score, result.term, fixed_state)+ # `source` is carried across: it is what tells the formatter a result came from+ # Wikipedia or Staan rather than our index, and rebuilding without it silently+ # relabelled every external result as "mwmbl". user_ids and last_crawled are still+ # dropped on purpose - the formatted path has no business exposing them.+ fixed_document = Document(+ result.title,+ result.url,+ result.extract,+ result.score,+ result.term,+ fixed_state,+ source=result.source,+ )return fixed_documentdiff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.pynew file mode 100644index 0000000..5e4718a--- /dev/null+++ b/mwmbl/tinysearchengine/staan.py@@ -0,0 +1,114 @@+"""Staan web-search results, cached alongside Wikipedia's.++Staan is a general web-search API. It gives us broad recall over pages our own crawl has+never reached, which is the whole reason Combined Search exists.++Deliberately shaped like mwmbl.tinysearchengine.rank.get_wiki_results rather than like the+Super Search adapters: results come back as Documents through+mwmbl.indexer.external_cache, so they are cached, deduplicated and ranked by exactly the+same machinery as every other candidate. The cache file is already namespaced by+DocumentSource, so Staan entries and Wikipedia entries share a page without seeing each+other - see the external_cache module docstring.++Nothing here writes to the search index: these are somebody else's results, and indexing+them is a separate decision with its own quality question.+"""++from logging import getLogger++import requests+from django.conf import settings++from mwmbl.indexer.external_cache import get_cached_external_results, store_external_results+from mwmbl.tinysearchengine.indexer import Document, DocumentSource+from mwmbl.tinysearchengine.rank import MAX_QUERY_CHARS++logger = getLogger(__name__)+++# How many results we keep, and therefore what a cache entry holds. Fixed rather than taken+# from the caller for the same reason NUM_WIKI_RESULTS is (see rank.py): one entry has to+# mean the same thing to every caller, and a pool of a size the model was never fitted to+# is what that constant exists to prevent. The Staan API is not asked for a limit - the+# documented parameters are the query and the market - so this is a cut we make ourselves.+NUM_STAAN_RESULTS = 10++# The score a Staan result carries into the LTR model, top result first, counting down.+#+# `score` becomes the `item_score` feature, so this is a trained-on quantity: the constant+# used to build the training data must be the constant used here. It is the only channel+# through which Staan's own ranking can reach a model that has no source feature. Anchored+# at the top and a function of rank alone, exactly as WIKI_TOP_SCORE is, and on the same+# scale so the two providers' priors are comparable. Changing it means retraining.+STAAN_TOP_SCORE = 6.0+++def staan_score(rank: int) -> float:+ """The `score` feature for a Staan result at 0-based `rank`. See STAAN_TOP_SCORE."""+ return STAAN_TOP_SCORE - rank+++def _documents(payload: dict, query: str) -> list[Document]:+ results = payload["web"]["results"][:NUM_STAAN_RESULTS]+ documents = []+ for rank, result in enumerate(results):+ url = result.get("url")+ if not url:+ continue+ documents.append(+ Document(+ title=result.get("title", ""),+ url=url,+ extract=result.get("snippet") or result.get("description", ""),+ score=staan_score(rank),+ term=query,+ source=DocumentSource.STAAN,+ )+ )+ return documents+++def get_staan_results(query: str, max_results: int = NUM_STAAN_RESULTS) -> list[Document]:+ """Staan's results for a query, from the external cache where possible.++ Never raises: a provider that is down, slow or unconfigured costs the caller its extra+ recall, never its search.+ """+ query = query[:MAX_QUERY_CHARS]++ # Ahead of the API-key check deliberately: a cache hit costs Staan nothing and needs no+ # credentials, so a deployment that has since dropped the key still serves what it has.+ cached = get_cached_external_results(DocumentSource.STAAN, query)+ if cached is not None:+ # The cache stores the rank Staan gave, not a score - scoring is this function's to+ # do, and doing it here is what makes an entry mean the same thing to every caller.+ staan_results = cached[:max_results]+ for rank, document in enumerate(staan_results):+ document.score = staan_score(rank)+ return staan_results++ if not settings.STAAN_SEARCH_API_KEY:+ logger.warning("STAAN_SEARCH_API_KEY is not configured")+ return []++ try:+ response = requests.get(+ settings.STAAN_SEARCH_URL,+ params={"q": query, "market": settings.STAAN_MARKET},+ headers={"Authorization": f"Bearer {settings.STAAN_SEARCH_API_KEY}"},+ timeout=settings.STAAN_TIMEOUT_SECONDS,+ )+ response.raise_for_status()+ documents = _documents(response.json(), query)+ except Exception as e:+ # Never log the exception or the response body: both embed the request URL, and the+ # request URL embeds the user's (private) query. rank.py makes the same trade for+ # Wikipedia - the type is enough to tell a timeout from a 401.+ logger.warning("Failed to fetch Staan results: %s", type(e).__name__)+ return []++ # Only a well-formed answer is stored, including one with no results in it. Every path+ # above returns [] without storing, so a transient failure is never remembered as+ # "Staan has nothing for this".+ store_external_results(DocumentSource.STAAN, query, documents)+ return documents[:max_results]diff --git a/mwmbl/urls.py b/mwmbl/urls.pyindex 4240ed9..8dca731 100644--- a/mwmbl/urls.py+++ b/mwmbl/urls.py@@ -24,7 +24,7 @@ import mwmbl.crawler.app as crawlerfrom mwmbl.admin_views import blacklist_status_viewfrom mwmbl.api import api as v1_apifrom mwmbl.api import register_routers, v2_api-from mwmbl.search_setup import ranker+from mwmbl.search_setup import combined_ranker, rankerfrom mwmbl.tinysearchengine import searchfrom mwmbl.views import (CurationDetailView,@@ -45,7 +45,7 @@ from mwmbl.views import (# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.# This must be called before urlpatterns is evaluated.-register_routers(ranker=ranker)+register_routers(ranker=ranker, combined_ranker=combined_ranker)def trigger_error(request):diff --git a/test/test_combined_search.py b/test/test_combined_search.pynew file mode 100644index 0000000..11df5fa--- /dev/null+++ b/test/test_combined_search.py@@ -0,0 +1,294 @@+"""Integration tests for the Combined Search endpoint (/api/v2/combined-search/).++The endpoint's own job is auth, quota, fanning out to three sources concurrently and+handing the union to one ranker. The ranking itself belongs to LTRRanker and is tested+there, so these tests stub the ranker and the two external providers: what is checked here+is that each source reaches the pool, that provenance survives as far as the wire, that a+provider being down costs recall rather than the request, and that the quota is enforced+atomically.+"""++import pytest+from allauth.account.models import EmailAddress+from django.contrib.auth import get_user_model+from django.core.cache import cache+from django.test import Client, override_settings+from ninja_jwt.tokens import RefreshToken++import mwmbl.tinysearchengine.combined_search as combined_search+from mwmbl.models import ApiKey, generate_api_key+from mwmbl.quota import _combined_search_monthly_key+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState++User = get_user_model()++URL = "/api/v2/combined-search/"++INDEX_RESULT = Document("Tokio internals", "https://blog.example.com/tokio", "A crawled page about tokio.")+STAAN_RESULT = Document(+ "Tokio", "https://tokio.rs/", "An asynchronous runtime for Rust.", 6.0, source=DocumentSource.STAAN+)+WIKI_RESULT = Document(+ "Tokio (software)",+ "https://en.wikipedia.org/wiki/Tokio",+ "A Rust runtime.",+ 6.0,+ state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA,+)+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------+++def user(db):+ u = User.objects.create_user(username="csuser", email="[email protected]", password="x")+ EmailAddress.objects.create(user=u, email="[email protected]", verified=True, primary=True)+ return u+++def access_token(user):+ return str(RefreshToken.for_user(user).access_token)+++def api_key(user):+ raw, hashed = generate_api_key()+ key = ApiKey.objects.create(user=user, key=hashed, name="cs", scopes=[ApiKey.Scope.SEARCH])+ key.raw_key = raw+ return key+++def client(db):+ return Client()+++def fresh_quota(api_key):+ cache.delete(_combined_search_monthly_key(api_key.user.id))+ yield+ cache.delete(_combined_search_monthly_key(api_key.user.id))+++def stub_sources(monkeypatch):+ """Stub the two external providers and the ranker, recording what the ranker was given.++ The stub ranker returns the pool unchanged, so the response is the pool: that is what+ lets these tests assert on which sources reached it.+ """+ calls = {}++ def configure(staan=(STAAN_RESULT,), wiki=(WIKI_RESULT,), index=(INDEX_RESULT,)):+ def fake_staan(query, *args, **kwargs):+ calls["staan_query"] = query+ if isinstance(staan, Exception):+ raise staan+ return list(staan)++ def fake_wiki(query, *args, **kwargs):+ calls["wiki_query"] = query+ return list(wiki)++ def fake_search(query, additional_results, use_external_search=True):+ calls["additional_results"] = additional_results+ calls["use_external_search"] = use_external_search+ return list(index) + list(additional_results)++ monkeypatch.setattr(combined_search, "get_staan_results", fake_staan)+ monkeypatch.setattr(combined_search, "get_wiki_results", fake_wiki)+ # The router closed over the ranker at registration time, so the ranker instance+ # itself is what has to be patched, not the name in search_setup.+ from mwmbl.search_setup import combined_ranker++ monkeypatch.setattr(combined_ranker, "search", fake_search)+ return calls++ return configure+++def _get(client, api_key, query="tokio"):+ return client.get(f"{URL}?q={query}", HTTP_X_API_KEY=api_key.raw_key)+++# ---------------------------------------------------------------------------+# Auth & quota+# ---------------------------------------------------------------------------+++def test_combined_search_requires_auth(client):+ assert client.get(f"{URL}?q=tokio").status_code == 401+++def test_combined_search_with_api_key(client, api_key, fresh_quota, stub_sources):+ stub_sources()++ response = _get(client, api_key)++ assert response.status_code == 200+ assert response.json()["query"] == "tokio"+++def test_combined_search_with_jwt(client, user, access_token, fresh_quota, stub_sources):+ stub_sources()++ response = client.get(f"{URL}?q=tokio", HTTP_AUTHORIZATION=f"Bearer {access_token}")++ assert response.status_code == 200+++@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)+def test_the_response_reports_the_quota(client, api_key, fresh_quota, stub_sources):+ stub_sources()++ body = _get(client, api_key).json()++ assert body["monthly_usage"] == 1+ assert body["monthly_limit"] == 10+++@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)+def test_combined_search_quota_enforced(client, api_key, fresh_quota, stub_sources):+ stub_sources()+ cache.set(_combined_search_monthly_key(api_key.user.id), 10, timeout=3600)++ assert _get(client, api_key).status_code == 429+++@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)+def test_a_rejected_request_refunds_its_increment(client, api_key, fresh_quota, stub_sources):+ """The counter is incremented before it is checked, so that concurrent requests cannot+ both pass. A rejected request must give that increment back, or being over the limit+ once would push the counter up forever."""+ stub_sources()+ key = _combined_search_monthly_key(api_key.user.id)+ cache.set(key, 10, timeout=3600)++ _get(client, api_key)++ assert cache.get(key) == 10+++@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)+def test_the_quota_counter_is_its_own(client, api_key, fresh_quota, stub_sources):+ """Combined Search must not spend the standard-search or Super Search allowance."""+ from mwmbl.quota import get_monthly_count, get_monthly_super_search_count++ stub_sources()+ _get(client, api_key)++ assert get_monthly_count(api_key.user.id) == 0+ assert get_monthly_super_search_count(api_key.user.id) == 0+++# ---------------------------------------------------------------------------+# The pool+# ---------------------------------------------------------------------------+++def test_all_three_sources_reach_the_response(client, api_key, fresh_quota, stub_sources):+ stub_sources()++ results = _get(client, api_key).json()["results"]++ assert [result["url"] for result in results] == [+ INDEX_RESULT.url,+ STAAN_RESULT.url,+ WIKI_RESULT.url,+ ]+++def test_each_result_names_the_provider_it_came_from(client, api_key, fresh_quota, stub_sources):+ stub_sources()++ results = _get(client, api_key).json()["results"]++ assert [result["engine"] for result in results] == ["mwmbl", "staan", "wikipedia"]+++def test_the_external_results_are_passed_in_as_additional_results(client, api_key, fresh_quota, stub_sources):+ """Staan and Wikipedia go in through Ranker.get_results' additional_results hook, which+ is what gets them blacklist-filtered and ranked alongside the index candidates."""+ calls = stub_sources()++ _get(client, api_key)++ assert [document.url for document in calls["additional_results"]] == [STAAN_RESULT.url, WIKI_RESULT.url]+++def test_the_ranker_is_told_not_to_fetch_wikipedia_again(client, api_key, fresh_quota, stub_sources):+ """The endpoint fetches Wikipedia itself so that call runs concurrently with Staan's.+ Leaving external search on would fetch it a second time, serially."""+ calls = stub_sources()++ _get(client, api_key)++ assert calls["use_external_search"] is False+++def test_the_query_reaches_every_source(client, api_key, fresh_quota, stub_sources):+ calls = stub_sources()++ _get(client, api_key, query="rust")++ assert calls["staan_query"] == "rust"+ assert calls["wiki_query"] == "rust"+++def test_the_result_count_matches_the_results(client, api_key, fresh_quota, stub_sources):+ stub_sources()++ body = _get(client, api_key).json()++ assert body["number_of_results"] == len(body["results"])+++# ---------------------------------------------------------------------------+# Degradation+# ---------------------------------------------------------------------------+++def test_staan_returning_nothing_still_serves_the_other_sources(client, api_key, fresh_quota, stub_sources):+ stub_sources(staan=())++ results = _get(client, api_key).json()["results"]++ assert [result["url"] for result in results] == [INDEX_RESULT.url, WIKI_RESULT.url]+++def test_wikipedia_returning_nothing_still_serves_the_other_sources(client, api_key, fresh_quota, stub_sources):+ stub_sources(wiki=())++ results = _get(client, api_key).json()["results"]++ assert [result["url"] for result in results] == [INDEX_RESULT.url, STAAN_RESULT.url]+++def test_an_empty_pool_is_an_empty_response_not_an_error(client, api_key, fresh_quota, stub_sources):+ stub_sources(staan=(), wiki=(), index=())++ body = _get(client, api_key).json()++ assert body["results"] == []+ assert body["number_of_results"] == 0diff --git a/test/test_external_cache.py b/test/test_external_cache.pyindex 4e2dbea..801ac4b 100644--- a/test/test_external_cache.py+++ b/test/test_external_cache.py@@ -319,6 +319,20 @@ def test_a_stale_entry_is_not_a_cache_hit(cache_index):assert stale is None+def test_a_provider_with_its_own_ttl_expires_on_it(cache_index):+ """Staan names its own TTL, far shorter than Wikipedia's, because a general web index+ turns over far faster than an encyclopedia. A single shared TTL would serve six-month-old+ SERP results."""+ now = int(time.time())+ store_external_results(DocumentSource.STAAN, "python", [STAAN_RESULT], now=now)+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)++ a_fortnight_later = now + 14 * 24 * 60 * 60++ assert get_cached_external_results(DocumentSource.STAAN, "python", now=a_fortnight_later) is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=a_fortnight_later) is not None++def test_an_untimestamped_document_is_stale():"""A document with no last_crawled predates the cache or came from somewhere else;treating it as fresh would pin it forever."""diff --git a/test/test_format.py b/test/test_format.pyindex aed7498..46d9729 100644--- a/test/test_format.py+++ b/test/test_format.py@@ -1,5 +1,5 @@-from mwmbl.format import format_result-from mwmbl.tinysearchengine.indexer import Document+from mwmbl.format import DOCUMENT_PROVIDERS, format_result, format_result_v2, get_result_source+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentStatedef test_format_result():@@ -14,3 +14,32 @@ def test_format_result():{"value": "Insist in ", "is_bold": False},{"value": "Bananas", "is_bold": True},]+++def test_every_document_source_can_be_named():+ """DocumentSource is append-only and the wire name of each member is looked up directly,+ so a new provider must be added to DOCUMENT_PROVIDERS at the same time - otherwise the+ first result it returns raises inside a response."""+ assert set(DOCUMENT_PROVIDERS) == set(DocumentSource)+++def test_an_external_provider_names_itself():+ """Without this a Staan result reports as "mwmbl": it has no DocumentState of its own,+ because it never enters the index."""+ result = Document("Tokio", "https://tokio.rs/", "An async runtime.", source=DocumentSource.STAAN)++ assert get_result_source(result) == "staan"+ assert format_result(result, "tokio")["source"] == "staan"+ assert format_result_v2(result, 1, "tokio")["engine"] == "staan"+++def test_a_document_with_no_source_still_answers_from_its_state():+ result = Document("Python", "https://en.wikipedia.org/wiki/Python", "", state=DocumentState.FROM_WIKI)++ assert get_result_source(result) == "wikipedia"+++def test_an_index_document_is_mwmbl():+ result = Document("Something", "https://something.com", "")++ assert get_result_source(result) == "mwmbl"diff --git a/test/test_openapi.py b/test/test_openapi.pyindex a1f5708..c295418 100644--- a/test/test_openapi.py+++ b/test/test_openapi.py@@ -188,3 +188,39 @@ def test_schema_examples_in_openapi():assert "example" in vote_stats_properties["downvotes"]assert isinstance(vote_stats_properties["upvotes"]["example"], int)assert isinstance(vote_stats_properties["downvotes"]["example"], int)+++def test_v2_openapi_spec_generation():+ """The v2 spec generates and lists every v2 endpoint.++ The v2 API had no spec test at all, so a router that failed to register - or one whose+ response schema could not be built - showed up only as a missing endpoint at runtime.+ """+ response = Client().get("/api/v2/openapi.json")+ assert response.status_code == 200++ paths = response.json()["paths"]++ assert set(paths) == {+ "/api/v2/search/",+ "/api/v2/search/complete",+ "/api/v2/search/raw",+ "/api/v2/super-search/",+ "/api/v2/combined-search/",+ }+++def test_combined_search_is_documented():+ """The endpoint is authenticated and quota-limited, so its spec has to say so and has to+ declare the one parameter a caller needs."""+ spec = Client().get("/api/v2/openapi.json").json()+ operation = spec["paths"]["/api/v2/combined-search/"]["get"]++ assert [(p["name"], p["in"], p["required"]) for p in operation["parameters"]] == [("q", "query", True)]+ assert operation["responses"]["200"]["content"]["application/json"]["schema"] == {+ "$ref": "#/components/schemas/SearchResponse"+ }+ for expected in ("X-API-Key", "monthly_limit", "staan", "wikipedia"):+ assert expected in operation["description"]diff --git a/test/test_staan.py b/test/test_staan.pynew file mode 100644index 0000000..042574b--- /dev/null+++ b/test/test_staan.py@@ -0,0 +1,239 @@+"""Tests for the Staan search adapter (mwmbl.tinysearchengine.staan).++Staan is shaped like get_wiki_results rather than like a Super Search adapter: it returns+Documents through the external results cache, so these tests cover the same three things the+Wikipedia path is covered for - the parse, the cache round trip, and that a provider being+unavailable costs the caller recall rather than an exception.+"""++import time+from unittest.mock import MagicMock, patch++import pytest+from django.test import override_settings++from mwmbl.indexer.external_cache import (+ external_cache_path,+ get_cached_external_results,+ store_external_results,+)+from mwmbl.tinysearchengine import staan+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentSource, TinyIndex+from mwmbl.tinysearchengine.staan import NUM_STAAN_RESULTS, get_staan_results, staan_score++NUM_PAGES = 8++API_RESPONSE = {+ "web": {+ "results": [+ {+ "url": "https://tokio.rs/",+ "title": "Tokio",+ "snippet": "An asynchronous runtime for Rust.",+ },+ {+ "url": "https://docs.rs/tokio",+ "title": "tokio - Rust",+ "description": "API documentation for the tokio crate.",+ },+ {"title": "No URL here", "snippet": "dropped"},+ ]+ }+}+++def cache_index(tmp_path):+ """A real, empty external results cache index that the module-level helpers pick up."""+ with override_settings(+ DATA_PATH=str(tmp_path),+ EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",+ EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES,+ EXTERNAL_CACHE_ENABLED=True,+ STAAN_SEARCH_API_KEY="test-key",+ ):+ TinyIndex.create(+ item_factory=Document, index_path=str(external_cache_path()), num_pages=NUM_PAGES, page_size=PAGE_SIZE+ )+ yield str(external_cache_path())+++def _patched_staan(response):+ """Stand in for the Staan API, returning `response` (or raising, if it is an error)."""+ mock = patch.object(staan.requests, "get")+ started = mock.start()+ if isinstance(response, Exception):+ started.side_effect = response+ else:+ started.return_value = MagicMock(**{"json.return_value": response})+ return mock, started+++# ---------------------------------------------------------------------------+# Parsing+# ---------------------------------------------------------------------------+++def test_results_come_back_as_documents_attributed_to_staan(cache_index):+ mock, requested = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert [document.url for document in results] == ["https://tokio.rs/", "https://docs.rs/tokio"]+ assert [document.title for document in results] == ["Tokio", "tokio - Rust"]+ assert {document.source for document in results} == {DocumentSource.STAAN}+ assert requested.call_args.kwargs["params"] == {"q": "rust async", "market": "en-us"}+ assert requested.call_args.kwargs["headers"] == {"Authorization": "Bearer test-key"}+++def test_the_description_field_stands_in_for_a_missing_snippet(cache_index):+ mock, _ = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert results[0].extract == "An asynchronous runtime for Rust."+ assert results[1].extract == "API documentation for the tokio crate."+++def test_results_are_scored_by_their_rank(cache_index):+ """`score` becomes the item_score feature, so it is the only channel through which+ Staan's own ranking reaches a model that has no source feature."""+ mock, _ = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert [document.score for document in results] == [staan_score(0), staan_score(1)]+++def test_a_result_with_no_url_is_dropped(cache_index):+ mock, _ = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert all(document.url for document in results)+ assert len(results) == 2+++def test_no_more_than_num_staan_results_are_kept(cache_index):+ many = {"web": {"results": [{"url": f"https://example.com/{i}", "title": str(i)} for i in range(50)]}}+ mock, _ = _patched_staan(many)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert len(results) == NUM_STAAN_RESULTS+++# ---------------------------------------------------------------------------+# Caching+# ---------------------------------------------------------------------------+++def test_a_fetch_is_stored_in_the_external_cache(cache_index):+ mock, _ = _patched_staan(API_RESPONSE)+ try:+ get_staan_results("rust async")+ finally:+ mock.stop()++ cached = get_cached_external_results(DocumentSource.STAAN, "rust async")++ assert [document.url for document in cached] == ["https://tokio.rs/", "https://docs.rs/tokio"]+++def test_a_cache_hit_does_not_call_the_api(cache_index):+ store_external_results(+ DocumentSource.STAAN,+ "rust async",+ [Document("Tokio", "https://tokio.rs/", "An asynchronous runtime.", source=DocumentSource.STAAN)],+ now=int(time.time()),+ )++ mock, requested = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert requested.call_count == 0+ assert [document.url for document in results] == ["https://tokio.rs/"]+++def test_a_cache_hit_is_rescored_here_not_by_the_cache(cache_index):+ """The cache stores the rank the provider gave, never a score: turning one into the+ other needs this module's scale."""+ store_external_results(+ DocumentSource.STAAN,+ "rust async",+ [+ Document("Tokio", "https://tokio.rs/", "", source=DocumentSource.STAAN),+ Document("docs.rs", "https://docs.rs/tokio", "", source=DocumentSource.STAAN),+ ],+ now=int(time.time()),+ )++ mock, _ = _patched_staan(API_RESPONSE)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert [document.score for document in results] == [staan_score(0), staan_score(1)]+++def test_a_query_staan_has_nothing_for_is_remembered(cache_index):+ """Otherwise every empty query is re-fetched forever."""+ mock, requested = _patched_staan({"web": {"results": []}})+ try:+ assert get_staan_results("nonsense query") == []+ assert get_staan_results("nonsense query") == []+ finally:+ mock.stop()++ assert requested.call_count == 1+++# ---------------------------------------------------------------------------+# Failure+# ---------------------------------------------------------------------------+++def test_a_missing_api_key_costs_recall_not_the_search(cache_index):+ with override_settings(STAAN_SEARCH_API_KEY=""):+ assert get_staan_results("rust async") == []+++def test_a_request_failure_costs_recall_not_the_search(cache_index):+ mock, _ = _patched_staan(Exception("boom"))+ try:+ assert get_staan_results("rust async") == []+ finally:+ mock.stop()+++def test_a_failure_is_not_remembered_as_an_empty_result(cache_index):+ """A transient failure must never be cached as "Staan has nothing for this"."""+ mock, _ = _patched_staan(Exception("boom"))+ try:+ get_staan_results("rust async")+ finally:+ mock.stop()++ assert get_cached_external_results(DocumentSource.STAAN, "rust async") is None+++def test_an_unparseable_response_costs_recall_not_the_search(cache_index):+ mock, _ = _patched_staan({"unexpected": "shape"})+ try:+ assert get_staan_results("rust async") == []+ finally:+ mock.stop()