Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Add a Combined Search endpoint over Mwmbl, Staan and Wikipedia

Daoud Clarke97bb2f83105d
diff --git a/mwmbl/api.py b/mwmbl/api.py
index 63f9143..951bc56 100644
--- a/mwmbl/api.py
+++ b/mwmbl/api.py
@@ -10,7 +10,9 @@ Sub-routers (v1):
/api/v1/platform/ — user accounts, domain submissions, voting
Sub-routers (v2):
- /api/v2/search/ — full-text search with optional API-key auth and quota info
+ /api/v2/search/ — full-text search with optional API-key auth and quota info
+ /api/v2/super-search/ — multi-source streaming search (SSE), authenticated
+ /api/v2/combined-search/ — Mwmbl + Staan + Wikipedia in one ranked response, authenticated
JWT token endpoints (from NinjaJWTDefaultController) are registered on v1,
typically at /api/v1/platform/token/pair, /api/v1/platform/token/refresh, etc.
@@ -131,14 +133,15 @@ def v2_invalid_request_handler(request, exc: InvalidRequest):
# Routers are imported after the api instances are created to avoid circular imports,
# and after init functions have been called from urls.py.
-def register_routers(ranker):
+def register_routers(ranker, combined_ranker):
"""
Initialise and register all sub-routers on the v1 and v2 APIs.
This is called from urls.py after Django app setup is complete, so that
- dependencies (ranker) are available.
+ dependencies (the rankers) are available.
"""
import mwmbl.crawler.app as crawler_module
+ import mwmbl.tinysearchengine.combined_search as combined_search_module
import mwmbl.tinysearchengine.search as search_module
import mwmbl.tinysearchengine.super_search as super_search_module
from mwmbl.platform.api import router as platform_router
@@ -147,6 +150,7 @@ def register_routers(ranker):
search_module.init_router(ranker)
search_module.init_v2_router(ranker)
super_search_module.init_router()
+ combined_search_module.init_router(combined_ranker)
crawler_module.init_router()
api.add_router("/search/", search_module.router, tags=["Search"])
@@ -155,3 +159,4 @@ def register_routers(ranker):
v2_api.add_router("/search/", search_module.v2_router, tags=["Search"])
v2_api.add_router("/super-search/", super_search_module.router, tags=["Super Search"])
+ v2_api.add_router("/combined-search/", combined_search_module.router, tags=["Combined Search"])
diff --git a/mwmbl/format.py b/mwmbl/format.py
index ce34372..bcc2d8b 100644
--- a/mwmbl/format.py
+++ b/mwmbl/format.py
@@ -1,6 +1,6 @@
import re
-from mwmbl.tinysearchengine.indexer import DocumentState
+from mwmbl.tinysearchengine.indexer import DocumentSource, DocumentState
from mwmbl.tokenizer import clean_unicode, tokenize
DOCUMENT_SOURCES = {
@@ -13,6 +13,14 @@ DOCUMENT_SOURCES = {
DocumentState.FROM_WIKI_APPROVED: "wikipedia",
}
+# What each external provider is called on the wire. DocumentSource is append-only and every
+# member must appear here, or a result from a new provider cannot be named - test_format
+# asserts the map is complete rather than leaving that to a KeyError in a response.
+DOCUMENT_PROVIDERS = {
+ DocumentSource.WIKIPEDIA: "wikipedia",
+ DocumentSource.STAAN: "staan",
+}
+
HIGHLIGHT_STOPWORDS = {
# Articles & Determiners
@@ -113,6 +121,19 @@ def get_document_source(state: DocumentState):
return DOCUMENT_SOURCES.get(state, "mwmbl")
+def get_result_source(result):
+ """Where a result came from, for the API's `source` / `engine` field.
+
+ DocumentSource names the external provider a document was fetched from, so it is the more
+ specific answer and wins when set; DocumentState answers for everything that came out of
+ our own index. Without this a Staan result would report as "mwmbl": it has no
+ DocumentState of its own, because it never enters the index.
+ """
+ if result.source is not None:
+ return DOCUMENT_PROVIDERS[result.source]
+ return get_document_source(result.state)
+
+
def format_result_with_pattern(pattern, result):
formatted_result = {}
for content_type, content_raw in [("title", result.title), ("extract", result.extract)]:
@@ -128,7 +149,7 @@ def format_result_with_pattern(pattern, result):
content_result.append({"value": content[start:end], "is_bold": is_bold})
formatted_result[content_type] = content_result
formatted_result["url"] = result.url
- formatted_result["source"] = get_document_source(result.state)
+ formatted_result["source"] = get_result_source(result)
return formatted_result
@@ -187,6 +208,6 @@ def format_result_v2(result, position: int, query: str) -> dict:
"title_highlights": _extract_highlights(v1["title"]),
"content": content,
"content_highlights": _extract_highlights(v1["extract"]),
- "engine": get_document_source(result.state),
+ "engine": get_result_source(result),
"score": 1.0 / position,
}
diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.py
index 8af54b4..ffd5828 100644
--- a/mwmbl/indexer/external_cache.py
+++ b/mwmbl/indexer/external_cache.py
@@ -138,19 +138,26 @@ def external_cache_term(query: str) -> str:
return digest.hexdigest()[:16]
-def is_fresh(document: Document, now: int) -> bool:
- """Whether a cached document is still within its TTL. Untimestamped means stale.
+def _ttl_seconds(document: Document) -> int:
+ """How long this entry stays fresh.
+
+ An empty-result sentinel expires sooner than a real entry, and a provider whose results
+ turn over faster than an encyclopedia's gets its own TTL. document.source is the only
+ thing that identifies a provider here - this runs over a page's other entries too, which
+ belong to other queries and other providers, so the term says nothing about them.
- This runs over a page's other entries too, which belong to other queries and other
- providers, so it cannot recover the source from the term - document.source is the only
- thing that identifies them, and it is where a per-source TTL would key off.
+ Built per call rather than at import: a setting read at import time is a setting that
+ override_settings cannot move, and the tests do move it.
"""
- ttl = (
- settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS
- if document.url == EXTERNAL_CACHE_EMPTY_URL
- else settings.EXTERNAL_CACHE_TTL_SECONDS
- )
- return document.last_crawled is not None and now - document.last_crawled < ttl
+ if document.url == EXTERNAL_CACHE_EMPTY_URL:
+ return settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS
+ ttl_by_source = {DocumentSource.STAAN: settings.EXTERNAL_CACHE_STAAN_TTL_SECONDS}
+ return ttl_by_source.get(document.source, settings.EXTERNAL_CACHE_TTL_SECONDS)
+
+
+def is_fresh(document: Document, now: int) -> bool:
+ """Whether a cached document is still within its TTL. Untimestamped means stale."""
+ return document.last_crawled is not None and now - document.last_crawled < _ttl_seconds(document)
def get_cached_external_results(
diff --git a/mwmbl/quota.py b/mwmbl/quota.py
index ef66072..a46f297 100644
--- a/mwmbl/quota.py
+++ b/mwmbl/quota.py
@@ -34,6 +34,13 @@ def _super_search_monthly_key(user_id: int, year: int | None = None, month: int
return f"super_search:monthly:{user_id}:{y}:{m:02d}"
+def _combined_search_monthly_key(user_id: int, year: int | None = None, month: int | None = None) -> str:
+ now = datetime.now(timezone.utc)
+ y = year if year is not None else now.year
+ m = month if month is not None else now.month
+ return f"combined_search:monthly:{user_id}:{y}:{m:02d}"
+
+
def _rate_key(user_id: int) -> str:
return f"search:rate:{user_id}"
@@ -110,6 +117,33 @@ def decrement_monthly_super_search(user_id: int) -> None:
pass
+def get_monthly_combined_search_count(user_id: int) -> int:
+ """Return the current monthly combined-search request count for a user (0 if not set)."""
+ return cache.get(_combined_search_monthly_key(user_id), default=0)
+
+
+def increment_monthly_combined_search(user_id: int) -> int:
+ """Increment the monthly combined-search counter and return the new value."""
+ key = _combined_search_monthly_key(user_id)
+ if cache.add(key, 1, timeout=MONTHLY_TTL):
+ return 1
+ return cache.incr(key)
+
+
+def decrement_monthly_combined_search(user_id: int) -> None:
+ """Refund one combined-search increment (e.g. when the request is rejected over-limit).
+
+ Never drops below 0. No-op if the counter is missing.
+ """
+ key = _combined_search_monthly_key(user_id)
+ try:
+ if cache.get(key, default=0) > 0:
+ cache.decr(key)
+ except ValueError:
+ # decr() raises if the key vanished between the get and the decr; ignore.
+ pass
+
+
# ---------------------------------------------------------------------------
# Key scanning (used by background jobs only — requires django-redis)
# ---------------------------------------------------------------------------
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 812818c..0cdbe23 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -25,3 +25,17 @@ ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
# passed 3 here while mwmbl.rankeval.ltr.dataset trained on 5, so the model was ranking a
# pool a size it had never been fitted to.
ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True))
+
+# Combined Search gets its own model and its own ranker, so retraining on the pooled
+# Mwmbl + Staan + Wikipedia candidate set never moves standard search's ranking. Until
+# model-combined.xgb has been trained the endpoint serves the deployed model, which is a
+# working answer rather than a missing endpoint - the artifact ships in Stage 4.
+combined_model_path = Path(settings.COMBINED_MODEL_PATH)
+if not combined_model_path.exists():
+ combined_model_path = Path(settings.RUST_MODEL_PATH)
+combined_ltr_model = RustXGBPipeline.from_model_path(str(combined_model_path))
+
+# include_wiki=False on purpose: the endpoint fetches Wikipedia itself so that call can run
+# concurrently with Staan's, and passes both in as additional_results. Leaving it True would
+# fetch Wikipedia a second time, serially, inside get_results.
+combined_ranker = MMRRanker(LTRRanker(tiny_index, completer, combined_ltr_model, include_wiki=False))
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index b511bdd..9752484 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -285,6 +285,21 @@ CURRENT_AGREEMENT_VERSIONS = {
"TERMS_OF_SERVICE_API": "v2026-04-A",
}
+# Combined Search - the v2 endpoint that pools the Mwmbl index, Staan and Wikipedia and
+# ranks the union with its own LTR model. Gated behind login at a flat monthly limit, like
+# Super Search, which it is meant to replace.
+COMBINED_SEARCH_MONTHLY_LIMIT = 100
+# Its own model artifact, so Combined Search can be retrained on the pooled candidate set
+# without moving standard search's ranking. Falls back to RUST_MODEL_PATH until trained.
+COMBINED_MODEL_PATH = Path(__file__).parent / "resources" / "model-combined.xgb"
+
+# Staan, the external web-search provider Combined Search pools in. Results are cached in
+# the external results index - see mwmbl.tinysearchengine.staan.
+STAAN_SEARCH_API_KEY = os.environ.get("STAAN_SEARCH_API_KEY", "")
+STAAN_SEARCH_URL = os.environ.get("STAAN_SEARCH_URL", "https://api.staan.ai/v2/search/web")
+STAAN_MARKET = os.environ.get("STAAN_MARKET", "en-us")
+STAAN_TIMEOUT_SECONDS = 5
+
# Super Search
SUPER_SEARCH_MONTHLY_LIMIT = 100
SUPER_SEARCH_TOP_K = 10 # promote sources in top-K seen so far for crawling
@@ -345,10 +360,15 @@ EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower(
# Counting search traffic (mwmbl.traffic). Off in the test settings: it is the only thing in
# the request path that reaches for Redis, and the CI test job runs none.
SEARCH_TRAFFIC_COUNTING = os.environ.get("SEARCH_TRAFFIC_COUNTING", "true").lower() != "false"
-# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six
-# months; a provider whose results turn over faster wants its own, and the place to add that
-# is here, keyed by source.
+# The default TTL, for any provider that does not name its own below. Wikipedia articles
+# move rarely, which is what justifies six months.
EXTERNAL_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks
+# A provider that turns over faster gets its own, read by external_cache.is_fresh. A general
+# web-search index moves far faster than an encyclopedia, so six months of Staan results
+# would be six months of stale answers. One scalar per provider rather than a dict keyed by
+# DocumentSource: the settings module must not import the indexer, which would pull the Rust
+# extension in at settings-import time.
+EXTERNAL_CACHE_STAAN_TTL_SECONDS = 7 * 24 * 60 * 60 # 1 week
# A query a provider has nothing for is worth remembering too, or it is re-fetched forever.
# Shorter, because a result appearing is a likelier change than an existing one moving.
EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.py
new file mode 100644
index 0000000..00aac26
--- /dev/null
+++ b/mwmbl/tinysearchengine/combined_search.py
@@ -0,0 +1,135 @@
+"""Combined Search - one request over the Mwmbl index, Staan and Wikipedia.
+
+Pools three sources, ranks the union with a model of its own, and answers with the same
+SearXNG-shaped JSON as /api/v2/search/. Authentication is required and a flat monthly quota
+applies, exactly as for Super Search, which this endpoint is meant to replace.
+
+There is deliberately no streaming and no crawling here. Super Search streams because it
+crawls promoted pages and follows their outbound links, which takes seconds; all three
+sources here answer in well under a second, and two of the three are cached, so there is
+nothing to stream and a plain response is what a client actually wants.
+
+The ranking core is three calls, because Ranker.get_results already pools
+``pages + additional_results + external_search_items`` and blacklist-filters the lot. Staan
+and Wikipedia go in as additional_results, fetched concurrently; the index side and the
+ranking are the ranker's own job.
+
+Nothing found here is written back to the search index. Super Search indexes what it finds;
+whether third-party SERP results belong in our index is a separate question with its own
+answer, and this endpoint does not pre-empt it.
+"""
+
+import asyncio
+from logging import getLogger
+
+from asgiref.sync import sync_to_async
+from django.conf import settings
+from ninja import Router
+from ninja.errors import HttpError
+
+from mwmbl.format import format_result_v2
+from mwmbl.quota import (
+ check_rate_limit,
+ decrement_monthly_combined_search,
+ increment_monthly_combined_search,
+)
+from mwmbl.search_auth import authenticate_user
+from mwmbl.tinysearchengine.rank import get_wiki_results
+from mwmbl.tinysearchengine.search import SearchResponse
+from mwmbl.tinysearchengine.staan import get_staan_results
+
+logger = getLogger(__name__)
+
+router = Router(tags=["Combined Search"])
+
+
+DESCRIPTION = (
+ "Search the Mwmbl index, Staan and Wikipedia in one request and return the ranked "
+ "union.\n\n"
+ "Every candidate - crawled by Mwmbl, returned by Staan, or fetched from Wikipedia - is "
+ "scored by one learning-to-rank model trained on the pooled candidate set, then "
+ "diversified so a single domain cannot take the whole page. The response is the same "
+ "SearXNG-compatible shape as `/api/v2/search/`.\n\n"
+ "The `engine` field names the provider a result came from:\n"
+ "- `mwmbl` - organically crawled by the Mwmbl crawler\n"
+ "- `staan` - returned by the Staan web-search API\n"
+ "- `wikipedia` - fetched from Wikipedia\n"
+ "- `google`, `user` - originally suggested via Google, or submitted by a user\n\n"
+ "Authentication is required: a search-scoped API key in `X-API-Key`, or a JWT bearer "
+ "token. Obtain a key via `POST /api/v1/platform/api-keys/`. A per-user monthly quota "
+ "applies; `monthly_usage` and `monthly_limit` report it on every response.\n\n"
+ "A source that is down or unconfigured costs the request its extra recall, not its "
+ "results: the remaining sources are ranked and returned as usual.\n\n"
+ "**Query parameter:** `q` - the search query string (required)."
+)
+
+OPENAPI_EXTRA = {
+ "parameters": [
+ {
+ "name": "q",
+ "in": "query",
+ "required": True,
+ "schema": {"type": "string", "example": "rust async runtimes"},
+ }
+ ]
+}
+
+
+async def _gather_external(query: str):
+ """Staan's and Wikipedia's results, fetched concurrently.
+
+ Both are synchronous, network-bound and independent, so they run on threads and the
+ request waits once rather than twice. Each returns [] on failure by itself.
+ """
+ return await asyncio.gather(
+ asyncio.to_thread(get_staan_results, query),
+ asyncio.to_thread(get_wiki_results, query),
+ )
+
+
+def init_router(ranker) -> None:
+ @router.get(
+ "",
+ response=SearchResponse,
+ # Handled manually in the view so both an API key and a JWT work under an async
+ # view - the same reason Super Search does it this way.
+ auth=None,
+ summary="Combined Search (Mwmbl + Staan + Wikipedia)",
+ description=DESCRIPTION,
+ openapi_extra=OPENAPI_EXTRA,
+ )
+ async def combined_search(request, q: str):
+ user = await authenticate_user(request)
+
+ if not await sync_to_async(check_rate_limit)(user.id):
+ raise HttpError(429, "Rate limit exceeded: maximum 5 requests per second.")
+
+ monthly_limit = settings.COMBINED_SEARCH_MONTHLY_LIMIT
+ # Increment first, then check: this makes the quota check atomic under concurrent
+ # requests (a check-then-increment would let racing requests both pass). Refund the
+ # increment if the caller is over the limit.
+ monthly_usage = await sync_to_async(increment_monthly_combined_search)(user.id)
+ if monthly_usage > monthly_limit:
+ await sync_to_async(decrement_monthly_combined_search)(user.id)
+ raise HttpError(
+ 429,
+ f"Combined Search monthly quota exceeded: {monthly_limit} requests per month "
+ f"and you have used {monthly_usage - 1}.",
+ )
+
+ staan_results, wiki_results = await _gather_external(q)
+ results = await asyncio.to_thread(
+ ranker.search,
+ q,
+ staan_results + wiki_results,
+ False, # use_external_search=False
+ )
+
+ formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]
+ return SearchResponse(
+ query=q,
+ number_of_results=len(formatted),
+ results=formatted,
+ monthly_usage=monthly_usage,
+ monthly_limit=monthly_limit,
+ )
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index e954695..6b3d97b 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -270,7 +270,19 @@ def fix_document_state(result: Document):
fixed_state = DocumentState(result.state)
except ValueError:
fixed_state = None
- fixed_document = Document(result.title, result.url, result.extract, result.score, result.term, fixed_state)
+ # `source` is carried across: it is what tells the formatter a result came from
+ # Wikipedia or Staan rather than our index, and rebuilding without it silently
+ # relabelled every external result as "mwmbl". user_ids and last_crawled are still
+ # dropped on purpose - the formatted path has no business exposing them.
+ fixed_document = Document(
+ result.title,
+ result.url,
+ result.extract,
+ result.score,
+ result.term,
+ fixed_state,
+ source=result.source,
+ )
return fixed_document
diff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.py
new file mode 100644
index 0000000..5e4718a
--- /dev/null
+++ b/mwmbl/tinysearchengine/staan.py
@@ -0,0 +1,114 @@
+"""Staan web-search results, cached alongside Wikipedia's.
+
+Staan is a general web-search API. It gives us broad recall over pages our own crawl has
+never reached, which is the whole reason Combined Search exists.
+
+Deliberately shaped like mwmbl.tinysearchengine.rank.get_wiki_results rather than like the
+Super Search adapters: results come back as Documents through
+mwmbl.indexer.external_cache, so they are cached, deduplicated and ranked by exactly the
+same machinery as every other candidate. The cache file is already namespaced by
+DocumentSource, so Staan entries and Wikipedia entries share a page without seeing each
+other - see the external_cache module docstring.
+
+Nothing here writes to the search index: these are somebody else's results, and indexing
+them is a separate decision with its own quality question.
+"""
+
+from logging import getLogger
+
+import requests
+from django.conf import settings
+
+from mwmbl.indexer.external_cache import get_cached_external_results, store_external_results
+from mwmbl.tinysearchengine.indexer import Document, DocumentSource
+from mwmbl.tinysearchengine.rank import MAX_QUERY_CHARS
+
+logger = getLogger(__name__)
+
+
+# How many results we keep, and therefore what a cache entry holds. Fixed rather than taken
+# from the caller for the same reason NUM_WIKI_RESULTS is (see rank.py): one entry has to
+# mean the same thing to every caller, and a pool of a size the model was never fitted to
+# is what that constant exists to prevent. The Staan API is not asked for a limit - the
+# documented parameters are the query and the market - so this is a cut we make ourselves.
+NUM_STAAN_RESULTS = 10
+
+# The score a Staan result carries into the LTR model, top result first, counting down.
+#
+# `score` becomes the `item_score` feature, so this is a trained-on quantity: the constant
+# used to build the training data must be the constant used here. It is the only channel
+# through which Staan's own ranking can reach a model that has no source feature. Anchored
+# at the top and a function of rank alone, exactly as WIKI_TOP_SCORE is, and on the same
+# scale so the two providers' priors are comparable. Changing it means retraining.
+STAAN_TOP_SCORE = 6.0
+
+
+def staan_score(rank: int) -> float:
+ """The `score` feature for a Staan result at 0-based `rank`. See STAAN_TOP_SCORE."""
+ return STAAN_TOP_SCORE - rank
+
+
+def _documents(payload: dict, query: str) -> list[Document]:
+ results = payload["web"]["results"][:NUM_STAAN_RESULTS]
+ documents = []
+ for rank, result in enumerate(results):
+ url = result.get("url")
+ if not url:
+ continue
+ documents.append(
+ Document(
+ title=result.get("title", ""),
+ url=url,
+ extract=result.get("snippet") or result.get("description", ""),
+ score=staan_score(rank),
+ term=query,
+ source=DocumentSource.STAAN,
+ )
+ )
+ return documents
+
+
+def get_staan_results(query: str, max_results: int = NUM_STAAN_RESULTS) -> list[Document]:
+ """Staan's results for a query, from the external cache where possible.
+
+ Never raises: a provider that is down, slow or unconfigured costs the caller its extra
+ recall, never its search.
+ """
+ query = query[:MAX_QUERY_CHARS]
+
+ # Ahead of the API-key check deliberately: a cache hit costs Staan nothing and needs no
+ # credentials, so a deployment that has since dropped the key still serves what it has.
+ cached = get_cached_external_results(DocumentSource.STAAN, query)
+ if cached is not None:
+ # The cache stores the rank Staan gave, not a score - scoring is this function's to
+ # do, and doing it here is what makes an entry mean the same thing to every caller.
+ staan_results = cached[:max_results]
+ for rank, document in enumerate(staan_results):
+ document.score = staan_score(rank)
+ return staan_results
+
+ if not settings.STAAN_SEARCH_API_KEY:
+ logger.warning("STAAN_SEARCH_API_KEY is not configured")
+ return []
+
+ try:
+ response = requests.get(
+ settings.STAAN_SEARCH_URL,
+ params={"q": query, "market": settings.STAAN_MARKET},
+ headers={"Authorization": f"Bearer {settings.STAAN_SEARCH_API_KEY}"},
+ timeout=settings.STAAN_TIMEOUT_SECONDS,
+ )
+ response.raise_for_status()
+ documents = _documents(response.json(), query)
+ except Exception as e:
+ # Never log the exception or the response body: both embed the request URL, and the
+ # request URL embeds the user's (private) query. rank.py makes the same trade for
+ # Wikipedia - the type is enough to tell a timeout from a 401.
+ logger.warning("Failed to fetch Staan results: %s", type(e).__name__)
+ return []
+
+ # Only a well-formed answer is stored, including one with no results in it. Every path
+ # above returns [] without storing, so a transient failure is never remembered as
+ # "Staan has nothing for this".
+ store_external_results(DocumentSource.STAAN, query, documents)
+ return documents[:max_results]
diff --git a/mwmbl/urls.py b/mwmbl/urls.py
index 4240ed9..8dca731 100644
--- a/mwmbl/urls.py
+++ b/mwmbl/urls.py
@@ -24,7 +24,7 @@ import mwmbl.crawler.app as crawler
from mwmbl.admin_views import blacklist_status_view
from mwmbl.api import api as v1_api
from mwmbl.api import register_routers, v2_api
-from mwmbl.search_setup import ranker
+from mwmbl.search_setup import combined_ranker, ranker
from mwmbl.tinysearchengine import search
from mwmbl.views import (
CurationDetailView,
@@ -45,7 +45,7 @@ from mwmbl.views import (
# Initialise the unified v1 API by registering all sub-routers with their runtime dependencies.
# This must be called before urlpatterns is evaluated.
-register_routers(ranker=ranker)
+register_routers(ranker=ranker, combined_ranker=combined_ranker)
def trigger_error(request):
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
new file mode 100644
index 0000000..11df5fa
--- /dev/null
+++ b/test/test_combined_search.py
@@ -0,0 +1,294 @@
+"""Integration tests for the Combined Search endpoint (/api/v2/combined-search/).
+
+The endpoint's own job is auth, quota, fanning out to three sources concurrently and
+handing the union to one ranker. The ranking itself belongs to LTRRanker and is tested
+there, so these tests stub the ranker and the two external providers: what is checked here
+is that each source reaches the pool, that provenance survives as far as the wire, that a
+provider being down costs recall rather than the request, and that the quota is enforced
+atomically.
+"""
+
+import pytest
+from allauth.account.models import EmailAddress
+from django.contrib.auth import get_user_model
+from django.core.cache import cache
+from django.test import Client, override_settings
+from ninja_jwt.tokens import RefreshToken
+
+import mwmbl.tinysearchengine.combined_search as combined_search
+from mwmbl.models import ApiKey, generate_api_key
+from mwmbl.quota import _combined_search_monthly_key
+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState
+
+User = get_user_model()
+
+URL = "/api/v2/combined-search/"
+
+INDEX_RESULT = Document("Tokio internals", "https://blog.example.com/tokio", "A crawled page about tokio.")
+STAAN_RESULT = Document(
+ "Tokio", "https://tokio.rs/", "An asynchronous runtime for Rust.", 6.0, source=DocumentSource.STAAN
+)
+WIKI_RESULT = Document(
+ "Tokio (software)",
+ "https://en.wikipedia.org/wiki/Tokio",
+ "A Rust runtime.",
+ 6.0,
+ state=DocumentState.FROM_WIKI,
+ source=DocumentSource.WIKIPEDIA,
+)
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+
+def user(db):
+ u = User.objects.create_user(username="csuser", email="[email protected]", password="x")
+ EmailAddress.objects.create(user=u, email="[email protected]", verified=True, primary=True)
+ return u
+
+
+def access_token(user):
+ return str(RefreshToken.for_user(user).access_token)
+
+
+def api_key(user):
+ raw, hashed = generate_api_key()
+ key = ApiKey.objects.create(user=user, key=hashed, name="cs", scopes=[ApiKey.Scope.SEARCH])
+ key.raw_key = raw
+ return key
+
+
+def client(db):
+ return Client()
+
+
+def fresh_quota(api_key):
+ cache.delete(_combined_search_monthly_key(api_key.user.id))
+ yield
+ cache.delete(_combined_search_monthly_key(api_key.user.id))
+
+
+def stub_sources(monkeypatch):
+ """Stub the two external providers and the ranker, recording what the ranker was given.
+
+ The stub ranker returns the pool unchanged, so the response is the pool: that is what
+ lets these tests assert on which sources reached it.
+ """
+ calls = {}
+
+ def configure(staan=(STAAN_RESULT,), wiki=(WIKI_RESULT,), index=(INDEX_RESULT,)):
+ def fake_staan(query, *args, **kwargs):
+ calls["staan_query"] = query
+ if isinstance(staan, Exception):
+ raise staan
+ return list(staan)
+
+ def fake_wiki(query, *args, **kwargs):
+ calls["wiki_query"] = query
+ return list(wiki)
+
+ def fake_search(query, additional_results, use_external_search=True):
+ calls["additional_results"] = additional_results
+ calls["use_external_search"] = use_external_search
+ return list(index) + list(additional_results)
+
+ monkeypatch.setattr(combined_search, "get_staan_results", fake_staan)
+ monkeypatch.setattr(combined_search, "get_wiki_results", fake_wiki)
+ # The router closed over the ranker at registration time, so the ranker instance
+ # itself is what has to be patched, not the name in search_setup.
+ from mwmbl.search_setup import combined_ranker
+
+ monkeypatch.setattr(combined_ranker, "search", fake_search)
+ return calls
+
+ return configure
+
+
+def _get(client, api_key, query="tokio"):
+ return client.get(f"{URL}?q={query}", HTTP_X_API_KEY=api_key.raw_key)
+
+
+# ---------------------------------------------------------------------------
+# Auth & quota
+# ---------------------------------------------------------------------------
+
+
+def test_combined_search_requires_auth(client):
+ assert client.get(f"{URL}?q=tokio").status_code == 401
+
+
+def test_combined_search_with_api_key(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+
+ response = _get(client, api_key)
+
+ assert response.status_code == 200
+ assert response.json()["query"] == "tokio"
+
+
+def test_combined_search_with_jwt(client, user, access_token, fresh_quota, stub_sources):
+ stub_sources()
+
+ response = client.get(f"{URL}?q=tokio", HTTP_AUTHORIZATION=f"Bearer {access_token}")
+
+ assert response.status_code == 200
+
+
+@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)
+def test_the_response_reports_the_quota(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+
+ body = _get(client, api_key).json()
+
+ assert body["monthly_usage"] == 1
+ assert body["monthly_limit"] == 10
+
+
+@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)
+def test_combined_search_quota_enforced(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+ cache.set(_combined_search_monthly_key(api_key.user.id), 10, timeout=3600)
+
+ assert _get(client, api_key).status_code == 429
+
+
+@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)
+def test_a_rejected_request_refunds_its_increment(client, api_key, fresh_quota, stub_sources):
+ """The counter is incremented before it is checked, so that concurrent requests cannot
+ both pass. A rejected request must give that increment back, or being over the limit
+ once would push the counter up forever."""
+ stub_sources()
+ key = _combined_search_monthly_key(api_key.user.id)
+ cache.set(key, 10, timeout=3600)
+
+ _get(client, api_key)
+
+ assert cache.get(key) == 10
+
+
+@override_settings(COMBINED_SEARCH_MONTHLY_LIMIT=10)
+def test_the_quota_counter_is_its_own(client, api_key, fresh_quota, stub_sources):
+ """Combined Search must not spend the standard-search or Super Search allowance."""
+ from mwmbl.quota import get_monthly_count, get_monthly_super_search_count
+
+ stub_sources()
+ _get(client, api_key)
+
+ assert get_monthly_count(api_key.user.id) == 0
+ assert get_monthly_super_search_count(api_key.user.id) == 0
+
+
+# ---------------------------------------------------------------------------
+# The pool
+# ---------------------------------------------------------------------------
+
+
+def test_all_three_sources_reach_the_response(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+
+ results = _get(client, api_key).json()["results"]
+
+ assert [result["url"] for result in results] == [
+ INDEX_RESULT.url,
+ STAAN_RESULT.url,
+ WIKI_RESULT.url,
+ ]
+
+
+def test_each_result_names_the_provider_it_came_from(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+
+ results = _get(client, api_key).json()["results"]
+
+ assert [result["engine"] for result in results] == ["mwmbl", "staan", "wikipedia"]
+
+
+def test_the_external_results_are_passed_in_as_additional_results(client, api_key, fresh_quota, stub_sources):
+ """Staan and Wikipedia go in through Ranker.get_results' additional_results hook, which
+ is what gets them blacklist-filtered and ranked alongside the index candidates."""
+ calls = stub_sources()
+
+ _get(client, api_key)
+
+ assert [document.url for document in calls["additional_results"]] == [STAAN_RESULT.url, WIKI_RESULT.url]
+
+
+def test_the_ranker_is_told_not_to_fetch_wikipedia_again(client, api_key, fresh_quota, stub_sources):
+ """The endpoint fetches Wikipedia itself so that call runs concurrently with Staan's.
+ Leaving external search on would fetch it a second time, serially."""
+ calls = stub_sources()
+
+ _get(client, api_key)
+
+ assert calls["use_external_search"] is False
+
+
+def test_the_query_reaches_every_source(client, api_key, fresh_quota, stub_sources):
+ calls = stub_sources()
+
+ _get(client, api_key, query="rust")
+
+ assert calls["staan_query"] == "rust"
+ assert calls["wiki_query"] == "rust"
+
+
+def test_the_result_count_matches_the_results(client, api_key, fresh_quota, stub_sources):
+ stub_sources()
+
+ body = _get(client, api_key).json()
+
+ assert body["number_of_results"] == len(body["results"])
+
+
+# ---------------------------------------------------------------------------
+# Degradation
+# ---------------------------------------------------------------------------
+
+
+def test_staan_returning_nothing_still_serves_the_other_sources(client, api_key, fresh_quota, stub_sources):
+ stub_sources(staan=())
+
+ results = _get(client, api_key).json()["results"]
+
+ assert [result["url"] for result in results] == [INDEX_RESULT.url, WIKI_RESULT.url]
+
+
+def test_wikipedia_returning_nothing_still_serves_the_other_sources(client, api_key, fresh_quota, stub_sources):
+ stub_sources(wiki=())
+
+ results = _get(client, api_key).json()["results"]
+
+ assert [result["url"] for result in results] == [INDEX_RESULT.url, STAAN_RESULT.url]
+
+
+def test_an_empty_pool_is_an_empty_response_not_an_error(client, api_key, fresh_quota, stub_sources):
+ stub_sources(staan=(), wiki=(), index=())
+
+ body = _get(client, api_key).json()
+
+ assert body["results"] == []
+ assert body["number_of_results"] == 0
diff --git a/test/test_external_cache.py b/test/test_external_cache.py
index 4e2dbea..801ac4b 100644
--- a/test/test_external_cache.py
+++ b/test/test_external_cache.py
@@ -319,6 +319,20 @@ def test_a_stale_entry_is_not_a_cache_hit(cache_index):
assert stale is None
+def test_a_provider_with_its_own_ttl_expires_on_it(cache_index):
+ """Staan names its own TTL, far shorter than Wikipedia's, because a general web index
+ turns over far faster than an encyclopedia. A single shared TTL would serve six-month-old
+ SERP results."""
+ now = int(time.time())
+ store_external_results(DocumentSource.STAAN, "python", [STAAN_RESULT], now=now)
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)
+
+ a_fortnight_later = now + 14 * 24 * 60 * 60
+
+ assert get_cached_external_results(DocumentSource.STAAN, "python", now=a_fortnight_later) is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=a_fortnight_later) is not None
+
+
def test_an_untimestamped_document_is_stale():
"""A document with no last_crawled predates the cache or came from somewhere else;
treating it as fresh would pin it forever."""
diff --git a/test/test_format.py b/test/test_format.py
index aed7498..46d9729 100644
--- a/test/test_format.py
+++ b/test/test_format.py
@@ -1,5 +1,5 @@
-from mwmbl.format import format_result
-from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.format import DOCUMENT_PROVIDERS, format_result, format_result_v2, get_result_source
+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, DocumentState
def test_format_result():
@@ -14,3 +14,32 @@ def test_format_result():
{"value": "Insist in ", "is_bold": False},
{"value": "Bananas", "is_bold": True},
]
+
+
+def test_every_document_source_can_be_named():
+ """DocumentSource is append-only and the wire name of each member is looked up directly,
+ so a new provider must be added to DOCUMENT_PROVIDERS at the same time - otherwise the
+ first result it returns raises inside a response."""
+ assert set(DOCUMENT_PROVIDERS) == set(DocumentSource)
+
+
+def test_an_external_provider_names_itself():
+ """Without this a Staan result reports as "mwmbl": it has no DocumentState of its own,
+ because it never enters the index."""
+ result = Document("Tokio", "https://tokio.rs/", "An async runtime.", source=DocumentSource.STAAN)
+
+ assert get_result_source(result) == "staan"
+ assert format_result(result, "tokio")["source"] == "staan"
+ assert format_result_v2(result, 1, "tokio")["engine"] == "staan"
+
+
+def test_a_document_with_no_source_still_answers_from_its_state():
+ result = Document("Python", "https://en.wikipedia.org/wiki/Python", "", state=DocumentState.FROM_WIKI)
+
+ assert get_result_source(result) == "wikipedia"
+
+
+def test_an_index_document_is_mwmbl():
+ result = Document("Something", "https://something.com", "")
+
+ assert get_result_source(result) == "mwmbl"
diff --git a/test/test_openapi.py b/test/test_openapi.py
index a1f5708..c295418 100644
--- a/test/test_openapi.py
+++ b/test/test_openapi.py
@@ -188,3 +188,39 @@ def test_schema_examples_in_openapi():
assert "example" in vote_stats_properties["downvotes"]
assert isinstance(vote_stats_properties["upvotes"]["example"], int)
assert isinstance(vote_stats_properties["downvotes"]["example"], int)
+
+
+def test_v2_openapi_spec_generation():
+ """The v2 spec generates and lists every v2 endpoint.
+
+ The v2 API had no spec test at all, so a router that failed to register - or one whose
+ response schema could not be built - showed up only as a missing endpoint at runtime.
+ """
+ response = Client().get("/api/v2/openapi.json")
+ assert response.status_code == 200
+
+ paths = response.json()["paths"]
+
+ assert set(paths) == {
+ "/api/v2/search/",
+ "/api/v2/search/complete",
+ "/api/v2/search/raw",
+ "/api/v2/super-search/",
+ "/api/v2/combined-search/",
+ }
+
+
+def test_combined_search_is_documented():
+ """The endpoint is authenticated and quota-limited, so its spec has to say so and has to
+ declare the one parameter a caller needs."""
+ spec = Client().get("/api/v2/openapi.json").json()
+ operation = spec["paths"]["/api/v2/combined-search/"]["get"]
+
+ assert [(p["name"], p["in"], p["required"]) for p in operation["parameters"]] == [("q", "query", True)]
+ assert operation["responses"]["200"]["content"]["application/json"]["schema"] == {
+ "$ref": "#/components/schemas/SearchResponse"
+ }
+ for expected in ("X-API-Key", "monthly_limit", "staan", "wikipedia"):
+ assert expected in operation["description"]
diff --git a/test/test_staan.py b/test/test_staan.py
new file mode 100644
index 0000000..042574b
--- /dev/null
+++ b/test/test_staan.py
@@ -0,0 +1,239 @@
+"""Tests for the Staan search adapter (mwmbl.tinysearchengine.staan).
+
+Staan is shaped like get_wiki_results rather than like a Super Search adapter: it returns
+Documents through the external results cache, so these tests cover the same three things the
+Wikipedia path is covered for - the parse, the cache round trip, and that a provider being
+unavailable costs the caller recall rather than an exception.
+"""
+
+import time
+from unittest.mock import MagicMock, patch
+
+import pytest
+from django.test import override_settings
+
+from mwmbl.indexer.external_cache import (
+ external_cache_path,
+ get_cached_external_results,
+ store_external_results,
+)
+from mwmbl.tinysearchengine import staan
+from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentSource, TinyIndex
+from mwmbl.tinysearchengine.staan import NUM_STAAN_RESULTS, get_staan_results, staan_score
+
+NUM_PAGES = 8
+
+API_RESPONSE = {
+ "web": {
+ "results": [
+ {
+ "url": "https://tokio.rs/",
+ "title": "Tokio",
+ "snippet": "An asynchronous runtime for Rust.",
+ },
+ {
+ "url": "https://docs.rs/tokio",
+ "title": "tokio - Rust",
+ "description": "API documentation for the tokio crate.",
+ },
+ {"title": "No URL here", "snippet": "dropped"},
+ ]
+ }
+}
+
+
+def cache_index(tmp_path):
+ """A real, empty external results cache index that the module-level helpers pick up."""
+ with override_settings(
+ DATA_PATH=str(tmp_path),
+ EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",
+ EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES,
+ EXTERNAL_CACHE_ENABLED=True,
+ STAAN_SEARCH_API_KEY="test-key",
+ ):
+ TinyIndex.create(
+ item_factory=Document, index_path=str(external_cache_path()), num_pages=NUM_PAGES, page_size=PAGE_SIZE
+ )
+ yield str(external_cache_path())
+
+
+def _patched_staan(response):
+ """Stand in for the Staan API, returning `response` (or raising, if it is an error)."""
+ mock = patch.object(staan.requests, "get")
+ started = mock.start()
+ if isinstance(response, Exception):
+ started.side_effect = response
+ else:
+ started.return_value = MagicMock(**{"json.return_value": response})
+ return mock, started
+
+
+# ---------------------------------------------------------------------------
+# Parsing
+# ---------------------------------------------------------------------------
+
+
+def test_results_come_back_as_documents_attributed_to_staan(cache_index):
+ mock, requested = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert [document.url for document in results] == ["https://tokio.rs/", "https://docs.rs/tokio"]
+ assert [document.title for document in results] == ["Tokio", "tokio - Rust"]
+ assert {document.source for document in results} == {DocumentSource.STAAN}
+ assert requested.call_args.kwargs["params"] == {"q": "rust async", "market": "en-us"}
+ assert requested.call_args.kwargs["headers"] == {"Authorization": "Bearer test-key"}
+
+
+def test_the_description_field_stands_in_for_a_missing_snippet(cache_index):
+ mock, _ = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert results[0].extract == "An asynchronous runtime for Rust."
+ assert results[1].extract == "API documentation for the tokio crate."
+
+
+def test_results_are_scored_by_their_rank(cache_index):
+ """`score` becomes the item_score feature, so it is the only channel through which
+ Staan's own ranking reaches a model that has no source feature."""
+ mock, _ = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert [document.score for document in results] == [staan_score(0), staan_score(1)]
+
+
+def test_a_result_with_no_url_is_dropped(cache_index):
+ mock, _ = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert all(document.url for document in results)
+ assert len(results) == 2
+
+
+def test_no_more_than_num_staan_results_are_kept(cache_index):
+ many = {"web": {"results": [{"url": f"https://example.com/{i}", "title": str(i)} for i in range(50)]}}
+ mock, _ = _patched_staan(many)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert len(results) == NUM_STAAN_RESULTS
+
+
+# ---------------------------------------------------------------------------
+# Caching
+# ---------------------------------------------------------------------------
+
+
+def test_a_fetch_is_stored_in_the_external_cache(cache_index):
+ mock, _ = _patched_staan(API_RESPONSE)
+ try:
+ get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ cached = get_cached_external_results(DocumentSource.STAAN, "rust async")
+
+ assert [document.url for document in cached] == ["https://tokio.rs/", "https://docs.rs/tokio"]
+
+
+def test_a_cache_hit_does_not_call_the_api(cache_index):
+ store_external_results(
+ DocumentSource.STAAN,
+ "rust async",
+ [Document("Tokio", "https://tokio.rs/", "An asynchronous runtime.", source=DocumentSource.STAAN)],
+ now=int(time.time()),
+ )
+
+ mock, requested = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert requested.call_count == 0
+ assert [document.url for document in results] == ["https://tokio.rs/"]
+
+
+def test_a_cache_hit_is_rescored_here_not_by_the_cache(cache_index):
+ """The cache stores the rank the provider gave, never a score: turning one into the
+ other needs this module's scale."""
+ store_external_results(
+ DocumentSource.STAAN,
+ "rust async",
+ [
+ Document("Tokio", "https://tokio.rs/", "", source=DocumentSource.STAAN),
+ Document("docs.rs", "https://docs.rs/tokio", "", source=DocumentSource.STAAN),
+ ],
+ now=int(time.time()),
+ )
+
+ mock, _ = _patched_staan(API_RESPONSE)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert [document.score for document in results] == [staan_score(0), staan_score(1)]
+
+
+def test_a_query_staan_has_nothing_for_is_remembered(cache_index):
+ """Otherwise every empty query is re-fetched forever."""
+ mock, requested = _patched_staan({"web": {"results": []}})
+ try:
+ assert get_staan_results("nonsense query") == []
+ assert get_staan_results("nonsense query") == []
+ finally:
+ mock.stop()
+
+ assert requested.call_count == 1
+
+
+# ---------------------------------------------------------------------------
+# Failure
+# ---------------------------------------------------------------------------
+
+
+def test_a_missing_api_key_costs_recall_not_the_search(cache_index):
+ with override_settings(STAAN_SEARCH_API_KEY=""):
+ assert get_staan_results("rust async") == []
+
+
+def test_a_request_failure_costs_recall_not_the_search(cache_index):
+ mock, _ = _patched_staan(Exception("boom"))
+ try:
+ assert get_staan_results("rust async") == []
+ finally:
+ mock.stop()
+
+
+def test_a_failure_is_not_remembered_as_an_empty_result(cache_index):
+ """A transient failure must never be cached as "Staan has nothing for this"."""
+ mock, _ = _patched_staan(Exception("boom"))
+ try:
+ get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert get_cached_external_results(DocumentSource.STAAN, "rust async") is None
+
+
+def test_an_unparseable_response_costs_recall_not_the_search(cache_index):
+ mock, _ = _patched_staan({"unexpected": "shape"})
+ try:
+ assert get_staan_results("rust async") == []
+ finally:
+ mock.stop()