An open source, non-profit web search engine
Search the index while Staan is in flight, and reuse Staan connections
Daoud Clarkecfdcb6deee59
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.pyindex 5f6c81b..98d7879 100644--- a/mwmbl/tinysearchengine/combined_search.py+++ b/mwmbl/tinysearchengine/combined_search.py@@ -9,9 +9,9 @@ crawls promoted pages and follows their outbound links, which takes seconds; Staonly network call here, and it is cached, so there is nothing to stream and a plain responseis what a client actually wants.-The ranking core is two calls, because Ranker.get_results already pools-``pages + additional_results`` and blacklist-filters the lot. Staan goes in as-additional_results; the index side and the ranking are the ranker's own job.+The ranking core is three calls. Ranker.retrieve looks the query up in the index while+Staan is fetched, so the two overlap. Ranker.search_retrieved then pools the index pages+with Staan's results as additional_results, blacklist-filters the lot and ranks it.There is no separate Wikipedia fetch: Staan already returns Wikipedia pages when they arerelevant, and evaluation found the extra fetch roughly neutral on quality (see@@ -107,14 +107,13 @@ def init_router(ranker) -> None:f"and you have used {monthly_usage - 1}.",)- # get_staan_results returns [] on failure by itself.- staan_results = await asyncio.to_thread(get_staan_results, q)- results = await asyncio.to_thread(- ranker.search,- q,- staan_results,- False, # use_external_search=False+ # The index lookup doesn't need Staan's answer, so it runs while Staan is in flight+ # rather than after it. get_staan_results returns [] on failure by itself.+ retrieval, staan_results = await asyncio.gather(+ asyncio.to_thread(ranker.retrieve, q),+ asyncio.to_thread(get_staan_results, q),)+ results = await asyncio.to_thread(ranker.search_retrieved, retrieval, staan_results)formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]return SearchResponse(diff --git a/mwmbl/tinysearchengine/mmr_rank.py b/mwmbl/tinysearchengine/mmr_rank.pyindex 0ccdcf3..4147b7c 100644--- a/mwmbl/tinysearchengine/mmr_rank.py+++ b/mwmbl/tinysearchengine/mmr_rank.py@@ -13,7 +13,7 @@ from collections import Counterfrom urllib.parse import urlparsefrom mwmbl.tinysearchengine.indexer import Document-from mwmbl.tinysearchengine.rank import Ranker+from mwmbl.tinysearchengine.rank import Ranker, Retrievalfrom mwmbl.tokenizer import tokenize# MMR tuning parameters.@@ -92,6 +92,12 @@ class MMRRanker:def search(self, s: str, additional_results: list[Document], use_external_search: bool = True) -> list[Document]:return mmr_rerank(self.ranker.search(s, additional_results, use_external_search))+ def retrieve(self, q: str) -> Retrieval:+ return self.ranker.retrieve(q)++ def search_retrieved(self, retrieval: Retrieval, additional_results: list[Document]) -> list[Document]:+ return mmr_rerank(self.ranker.search_retrieved(retrieval, additional_results))+def complete(self, q: str):return self.ranker.complete(q)diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 6b3d97b..8bbcf89 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -7,6 +7,7 @@ import timeimport urllibfrom abc import abstractmethodfrom collections import defaultdict+from dataclasses import dataclassfrom logging import getLoggerfrom operator import itemgetterfrom pathlib import Path@@ -296,6 +297,17 @@ def remove_curate_state(state: DocumentState):return state+@dataclass+class Retrieval:+ """What Ranker.retrieve found in the index for a query, ready to be ranked."""++ terms: list[str]+ completions: list[str]+ is_complete: bool+ curated_items: list[Document]+ pages: list[Document]++class Ranker:def __init__(self, tiny_index: TinyIndex, completer: Completer):self.tiny_index = tiny_index@@ -309,8 +321,20 @@ class Ranker:# use_external_search=False is how the search-as-you-type path avoids a Wikipedia# call per keystroke; get_results has taken the flag all along, but only complete()# could reach it. See mwmbl.views.- results, terms, _ = self.get_results(s, additional_results, use_external_search)+ results, _, _ = self.get_results(s, additional_results, use_external_search)+ return self._drop_repeated_urls(results)++ def search_retrieved(self, retrieval: Retrieval, additional_results: list[Document]) -> list[Document]:+ """search() over an index retrieval already made with retrieve().++ This is how a caller overlaps the index lookup with a slow external fetch: retrieve+ and fetch concurrently, then rank the two together here. Never fetches Wikipedia.+ """+ results = self._rank(retrieval, additional_results, external_search_items=[])+ return self._drop_repeated_urls(results)+ @staticmethod+ def _drop_repeated_urls(results: list[Document]) -> list[Document]:ranked_results = []seen_urls = set()for result in results:@@ -346,6 +370,13 @@ class Ranker:def get_results(self, q: str, additional_results: list[Document], use_external_search: bool = True):logger.info(f"Get results with {len(additional_results)} additional results")+ retrieval = self.retrieve(q)+ external_search_items = self.external_search(q) if use_external_search else []+ results = self._rank(retrieval, additional_results, external_search_items)+ return results, retrieval.terms, retrieval.completions++ def retrieve(self, q: str) -> Retrieval:+ """The index lookup for a query: everything get_results needs before ranking."""terms = tokenize(q)is_complete = q.endswith(" ")@@ -386,14 +417,19 @@ class Ranker:if items is not None:pages += items- external_search_items = self.external_search(q) if use_external_search else []- candidates = pages + additional_results + external_search_items- candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)+ return Retrieval(terms, completions, is_complete, curated_items, pages)++ def _rank(+ self, retrieval: Retrieval, additional_results: list[Document], external_search_items: list[Document]+ ) -> list[Document]:+ candidates = retrieval.pages + additional_results + external_search_items+ candidates, curated_items = self._remove_blacklisted(+ candidates, retrieval.curated_items, index_items=retrieval.pages+ )- ordered_results = self.order_results(terms, candidates, is_complete)+ ordered_results = self.order_results(retrieval.terms, candidates, retrieval.is_complete)deduplicated_results = deduplicate(curated_items + ordered_results, set())- state_fixed = [fix_document_state(result) for result in deduplicated_results]- return state_fixed, terms, completions+ return [fix_document_state(result) for result in deduplicated_results]@staticmethoddef _remove_blacklisted(diff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.pyindex 904e2d0..2b7d9ce 100644--- a/mwmbl/tinysearchengine/staan.py+++ b/mwmbl/tinysearchengine/staan.py@@ -42,6 +42,11 @@ NUM_STAAN_RESULTS = 10# scale so the two providers' priors are comparable. Changing it means retraining.STAAN_TOP_SCORE = 6.0+# One session for the process, so its pooled connections are reused: a fresh connection per+# request pays a TCP and TLS handshake to Staan every time, and Staan is on Combined+# Search's critical path.+session = requests.Session()+def staan_score(rank: int) -> float:"""The `score` feature for a Staan result at 0-based `rank`. See STAAN_TOP_SCORE."""@@ -93,7 +98,7 @@ def get_staan_results(query: str, max_results: int = NUM_STAAN_RESULTS) -> list[return []try:- response = requests.get(+ response = session.get(settings.STAAN_SEARCH_URL,params={"q": query, "market": settings.STAAN_MARKET},headers={"Authorization": f"Bearer {settings.STAAN_SEARCH_API_KEY}"},diff --git a/test/test_combined_search.py b/test/test_combined_search.pyindex 006c38c..bc9cad5 100644--- a/test/test_combined_search.py+++ b/test/test_combined_search.py@@ -1,13 +1,15 @@"""Integration tests for the Combined Search endpoint (/api/v2/combined-search/).-The endpoint's own job is auth, quota, fetching Staan and handing it to one ranker alongside-the index. The ranking itself belongs to CombinedLTRRanker and is tested there, so these+The endpoint's own job is auth, quota, fetching Staan while the index is searched, and+handing both to one ranker. The ranking itself belongs to CombinedLTRRanker and is tested there, so thesetests stub the ranker and Staan: what is checked hereis that each source reaches the pool, that provenance survives as far as the wire, that aprovider being down costs recall rather than the request, and that the quota is enforcedatomically."""+import threading+import pytestfrom allauth.account.models import EmailAddressfrom django.contrib.auth import get_user_model@@ -90,17 +92,21 @@ def stub_sources(monkeypatch):raise staanreturn list(staan)- def fake_search(query, additional_results, use_external_search=True):+ def fake_retrieve(query):+ calls["retrieve_query"] = query+ return list(index)++ def fake_search_retrieved(retrieval, additional_results):calls["additional_results"] = additional_results- calls["use_external_search"] = use_external_search- return list(index) + list(additional_results)+ return retrieval + list(additional_results)monkeypatch.setattr(combined_search, "get_staan_results", fake_staan)# The router closed over the ranker at registration time, so the ranker instance# itself is what has to be patched, not the name in search_setup.from mwmbl.search_setup import combined_ranker- monkeypatch.setattr(combined_ranker, "search", fake_search)+ monkeypatch.setattr(combined_ranker, "retrieve", fake_retrieve)+ monkeypatch.setattr(combined_ranker, "search_retrieved", fake_search_retrieved)return callsreturn configure@@ -222,14 +228,33 @@ def test_the_external_results_are_passed_in_as_additional_results(client, api_ke@pytest.mark.django_db-def test_the_ranker_is_told_not_to_fetch_wikipedia(client, api_key, fresh_quota, stub_sources):- """External search is standard search's Wikipedia fetch, which Combined Search drops:- Staan already returns Wikipedia pages when they are relevant."""- calls = stub_sources()+def test_the_index_is_searched_while_staan_is_in_flight(client, api_key, fresh_quota, stub_sources, monkeypatch):+ """Staan is the slow call, so the index lookup must overlap it, not wait for it. Each+ stub waits for the other to start: run one after the other, the first would time out."""+ stub_sources()+ staan_started = threading.Event()+ retrieve_started = threading.Event()+ from mwmbl.search_setup import combined_ranker- _get(client, api_key)+ fake_retrieve = combined_ranker.retrieve+ fake_staan = combined_search.get_staan_results++ def overlapping_retrieve(query):+ retrieve_started.set()+ assert staan_started.wait(timeout=5)+ return fake_retrieve(query)- assert calls["use_external_search"] is False+ def overlapping_staan(query, *args, **kwargs):+ staan_started.set()+ assert retrieve_started.wait(timeout=5)+ return fake_staan(query, *args, **kwargs)++ monkeypatch.setattr(combined_ranker, "retrieve", overlapping_retrieve)+ monkeypatch.setattr(combined_search, "get_staan_results", overlapping_staan)++ results = _get(client, api_key).json()["results"]++ assert [result["url"] for result in results] == [INDEX_RESULT.url, STAAN_RESULT.url]@pytest.mark.django_db@@ -239,6 +264,7 @@ def test_the_query_reaches_staan(client, api_key, fresh_quota, stub_sources):_get(client, api_key, query="rust")assert calls["staan_query"] == "rust"+ assert calls["retrieve_query"] == "rust"@pytest.mark.django_dbdiff --git a/test/test_rank.py b/test/test_rank.pyindex 0788341..be14d81 100644--- a/test/test_rank.py+++ b/test/test_rank.py@@ -64,6 +64,26 @@ def test_search_still_triggers_external_search():assert ranker.external_search_calls == ["some query"]+def test_search_retrieved_ranks_like_search_without_external_search():+ """Combined Search retrieves from the index while Staan is in flight, then ranks the two+ together. That split must rank exactly as a single search() call would."""+ index_pages = [+ Document("Bananas", "https://a.com/", "Bananas and apples", 1.0),+ Document("Apples", "https://b.com/", "Only apples", 2.0),+ Document("Bananas again", "https://a.com/", "A repeat of the first URL", 0.5),+ ]+ additional = [Document("Banana bread", "https://c.com/", "Bananas, baked", 3.0)]+ ranker = _TrackingRanker()+ ranker.tiny_index.retrieve.return_value = index_pages++ retrieved = ranker.search_retrieved(ranker.retrieve("bananas "), additional)+ searched = ranker.search("bananas ", additional, use_external_search=False)++ assert [d.url for d in retrieved] == [d.url for d in searched]+ assert sorted(d.url for d in retrieved) == ["https://a.com/", "https://c.com/"]+ assert ranker.external_search_calls == []++def _make_retry_error(status_code: int) -> RetryError:reason = ResponseError(f"too many {status_code} error responses")max_retry_error = MaxRetryError(diff --git a/test/test_staan.py b/test/test_staan.pyindex 7bcf052..a6fb7d3 100644--- a/test/test_staan.py+++ b/test/test_staan.py@@ -60,7 +60,7 @@ def cache_index(tmp_path):def _patched_staan(response):"""Stand in for the Staan API, returning `response` (or raising, if it is an error)."""- mock = patch.object(staan.requests, "get")+ mock = patch.object(staan.session, "get")started = mock.start()if isinstance(response, Exception):started.side_effect = response