Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Search the index while Staan is in flight, and reuse Staan connections

Daoud Clarkecfdcb6deee59
diff --git a/mwmbl/tinysearchengine/combined_search.py b/mwmbl/tinysearchengine/combined_search.py
index 5f6c81b..98d7879 100644
--- a/mwmbl/tinysearchengine/combined_search.py
+++ b/mwmbl/tinysearchengine/combined_search.py
@@ -9,9 +9,9 @@ crawls promoted pages and follows their outbound links, which takes seconds; Sta
only network call here, and it is cached, so there is nothing to stream and a plain response
is what a client actually wants.
-The ranking core is two calls, because Ranker.get_results already pools
-``pages + additional_results`` and blacklist-filters the lot. Staan goes in as
-additional_results; the index side and the ranking are the ranker's own job.
+The ranking core is three calls. Ranker.retrieve looks the query up in the index while
+Staan is fetched, so the two overlap. Ranker.search_retrieved then pools the index pages
+with Staan's results as additional_results, blacklist-filters the lot and ranks it.
There is no separate Wikipedia fetch: Staan already returns Wikipedia pages when they are
relevant, and evaluation found the extra fetch roughly neutral on quality (see
@@ -107,14 +107,13 @@ def init_router(ranker) -> None:
f"and you have used {monthly_usage - 1}.",
)
- # get_staan_results returns [] on failure by itself.
- staan_results = await asyncio.to_thread(get_staan_results, q)
- results = await asyncio.to_thread(
- ranker.search,
- q,
- staan_results,
- False, # use_external_search=False
+ # The index lookup doesn't need Staan's answer, so it runs while Staan is in flight
+ # rather than after it. get_staan_results returns [] on failure by itself.
+ retrieval, staan_results = await asyncio.gather(
+ asyncio.to_thread(ranker.retrieve, q),
+ asyncio.to_thread(get_staan_results, q),
)
+ results = await asyncio.to_thread(ranker.search_retrieved, retrieval, staan_results)
formatted = [format_result_v2(result, i + 1, q) for i, result in enumerate(results)]
return SearchResponse(
diff --git a/mwmbl/tinysearchengine/mmr_rank.py b/mwmbl/tinysearchengine/mmr_rank.py
index 0ccdcf3..4147b7c 100644
--- a/mwmbl/tinysearchengine/mmr_rank.py
+++ b/mwmbl/tinysearchengine/mmr_rank.py
@@ -13,7 +13,7 @@ from collections import Counter
from urllib.parse import urlparse
from mwmbl.tinysearchengine.indexer import Document
-from mwmbl.tinysearchengine.rank import Ranker
+from mwmbl.tinysearchengine.rank import Ranker, Retrieval
from mwmbl.tokenizer import tokenize
# MMR tuning parameters.
@@ -92,6 +92,12 @@ class MMRRanker:
def search(self, s: str, additional_results: list[Document], use_external_search: bool = True) -> list[Document]:
return mmr_rerank(self.ranker.search(s, additional_results, use_external_search))
+ def retrieve(self, q: str) -> Retrieval:
+ return self.ranker.retrieve(q)
+
+ def search_retrieved(self, retrieval: Retrieval, additional_results: list[Document]) -> list[Document]:
+ return mmr_rerank(self.ranker.search_retrieved(retrieval, additional_results))
+
def complete(self, q: str):
return self.ranker.complete(q)
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 6b3d97b..8bbcf89 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -7,6 +7,7 @@ import time
import urllib
from abc import abstractmethod
from collections import defaultdict
+from dataclasses import dataclass
from logging import getLogger
from operator import itemgetter
from pathlib import Path
@@ -296,6 +297,17 @@ def remove_curate_state(state: DocumentState):
return state
+@dataclass
+class Retrieval:
+ """What Ranker.retrieve found in the index for a query, ready to be ranked."""
+
+ terms: list[str]
+ completions: list[str]
+ is_complete: bool
+ curated_items: list[Document]
+ pages: list[Document]
+
+
class Ranker:
def __init__(self, tiny_index: TinyIndex, completer: Completer):
self.tiny_index = tiny_index
@@ -309,8 +321,20 @@ class Ranker:
# use_external_search=False is how the search-as-you-type path avoids a Wikipedia
# call per keystroke; get_results has taken the flag all along, but only complete()
# could reach it. See mwmbl.views.
- results, terms, _ = self.get_results(s, additional_results, use_external_search)
+ results, _, _ = self.get_results(s, additional_results, use_external_search)
+ return self._drop_repeated_urls(results)
+
+ def search_retrieved(self, retrieval: Retrieval, additional_results: list[Document]) -> list[Document]:
+ """search() over an index retrieval already made with retrieve().
+
+ This is how a caller overlaps the index lookup with a slow external fetch: retrieve
+ and fetch concurrently, then rank the two together here. Never fetches Wikipedia.
+ """
+ results = self._rank(retrieval, additional_results, external_search_items=[])
+ return self._drop_repeated_urls(results)
+ @staticmethod
+ def _drop_repeated_urls(results: list[Document]) -> list[Document]:
ranked_results = []
seen_urls = set()
for result in results:
@@ -346,6 +370,13 @@ class Ranker:
def get_results(self, q: str, additional_results: list[Document], use_external_search: bool = True):
logger.info(f"Get results with {len(additional_results)} additional results")
+ retrieval = self.retrieve(q)
+ external_search_items = self.external_search(q) if use_external_search else []
+ results = self._rank(retrieval, additional_results, external_search_items)
+ return results, retrieval.terms, retrieval.completions
+
+ def retrieve(self, q: str) -> Retrieval:
+ """The index lookup for a query: everything get_results needs before ranking."""
terms = tokenize(q)
is_complete = q.endswith(" ")
@@ -386,14 +417,19 @@ class Ranker:
if items is not None:
pages += items
- external_search_items = self.external_search(q) if use_external_search else []
- candidates = pages + additional_results + external_search_items
- candidates, curated_items = self._remove_blacklisted(candidates, curated_items, index_items=pages)
+ return Retrieval(terms, completions, is_complete, curated_items, pages)
+
+ def _rank(
+ self, retrieval: Retrieval, additional_results: list[Document], external_search_items: list[Document]
+ ) -> list[Document]:
+ candidates = retrieval.pages + additional_results + external_search_items
+ candidates, curated_items = self._remove_blacklisted(
+ candidates, retrieval.curated_items, index_items=retrieval.pages
+ )
- ordered_results = self.order_results(terms, candidates, is_complete)
+ ordered_results = self.order_results(retrieval.terms, candidates, retrieval.is_complete)
deduplicated_results = deduplicate(curated_items + ordered_results, set())
- state_fixed = [fix_document_state(result) for result in deduplicated_results]
- return state_fixed, terms, completions
+ return [fix_document_state(result) for result in deduplicated_results]
@staticmethod
def _remove_blacklisted(
diff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.py
index 904e2d0..2b7d9ce 100644
--- a/mwmbl/tinysearchengine/staan.py
+++ b/mwmbl/tinysearchengine/staan.py
@@ -42,6 +42,11 @@ NUM_STAAN_RESULTS = 10
# scale so the two providers' priors are comparable. Changing it means retraining.
STAAN_TOP_SCORE = 6.0
+# One session for the process, so its pooled connections are reused: a fresh connection per
+# request pays a TCP and TLS handshake to Staan every time, and Staan is on Combined
+# Search's critical path.
+session = requests.Session()
+
def staan_score(rank: int) -> float:
"""The `score` feature for a Staan result at 0-based `rank`. See STAAN_TOP_SCORE."""
@@ -93,7 +98,7 @@ def get_staan_results(query: str, max_results: int = NUM_STAAN_RESULTS) -> list[
return []
try:
- response = requests.get(
+ response = session.get(
settings.STAAN_SEARCH_URL,
params={"q": query, "market": settings.STAAN_MARKET},
headers={"Authorization": f"Bearer {settings.STAAN_SEARCH_API_KEY}"},
diff --git a/test/test_combined_search.py b/test/test_combined_search.py
index 006c38c..bc9cad5 100644
--- a/test/test_combined_search.py
+++ b/test/test_combined_search.py
@@ -1,13 +1,15 @@
"""Integration tests for the Combined Search endpoint (/api/v2/combined-search/).
-The endpoint's own job is auth, quota, fetching Staan and handing it to one ranker alongside
-the index. The ranking itself belongs to CombinedLTRRanker and is tested there, so these
+The endpoint's own job is auth, quota, fetching Staan while the index is searched, and
+handing both to one ranker. The ranking itself belongs to CombinedLTRRanker and is tested there, so these
tests stub the ranker and Staan: what is checked here
is that each source reaches the pool, that provenance survives as far as the wire, that a
provider being down costs recall rather than the request, and that the quota is enforced
atomically.
"""
+import threading
+
import pytest
from allauth.account.models import EmailAddress
from django.contrib.auth import get_user_model
@@ -90,17 +92,21 @@ def stub_sources(monkeypatch):
raise staan
return list(staan)
- def fake_search(query, additional_results, use_external_search=True):
+ def fake_retrieve(query):
+ calls["retrieve_query"] = query
+ return list(index)
+
+ def fake_search_retrieved(retrieval, additional_results):
calls["additional_results"] = additional_results
- calls["use_external_search"] = use_external_search
- return list(index) + list(additional_results)
+ return retrieval + list(additional_results)
monkeypatch.setattr(combined_search, "get_staan_results", fake_staan)
# The router closed over the ranker at registration time, so the ranker instance
# itself is what has to be patched, not the name in search_setup.
from mwmbl.search_setup import combined_ranker
- monkeypatch.setattr(combined_ranker, "search", fake_search)
+ monkeypatch.setattr(combined_ranker, "retrieve", fake_retrieve)
+ monkeypatch.setattr(combined_ranker, "search_retrieved", fake_search_retrieved)
return calls
return configure
@@ -222,14 +228,33 @@ def test_the_external_results_are_passed_in_as_additional_results(client, api_ke
@pytest.mark.django_db
-def test_the_ranker_is_told_not_to_fetch_wikipedia(client, api_key, fresh_quota, stub_sources):
- """External search is standard search's Wikipedia fetch, which Combined Search drops:
- Staan already returns Wikipedia pages when they are relevant."""
- calls = stub_sources()
+def test_the_index_is_searched_while_staan_is_in_flight(client, api_key, fresh_quota, stub_sources, monkeypatch):
+ """Staan is the slow call, so the index lookup must overlap it, not wait for it. Each
+ stub waits for the other to start: run one after the other, the first would time out."""
+ stub_sources()
+ staan_started = threading.Event()
+ retrieve_started = threading.Event()
+ from mwmbl.search_setup import combined_ranker
- _get(client, api_key)
+ fake_retrieve = combined_ranker.retrieve
+ fake_staan = combined_search.get_staan_results
+
+ def overlapping_retrieve(query):
+ retrieve_started.set()
+ assert staan_started.wait(timeout=5)
+ return fake_retrieve(query)
- assert calls["use_external_search"] is False
+ def overlapping_staan(query, *args, **kwargs):
+ staan_started.set()
+ assert retrieve_started.wait(timeout=5)
+ return fake_staan(query, *args, **kwargs)
+
+ monkeypatch.setattr(combined_ranker, "retrieve", overlapping_retrieve)
+ monkeypatch.setattr(combined_search, "get_staan_results", overlapping_staan)
+
+ results = _get(client, api_key).json()["results"]
+
+ assert [result["url"] for result in results] == [INDEX_RESULT.url, STAAN_RESULT.url]
@pytest.mark.django_db
@@ -239,6 +264,7 @@ def test_the_query_reaches_staan(client, api_key, fresh_quota, stub_sources):
_get(client, api_key, query="rust")
assert calls["staan_query"] == "rust"
+ assert calls["retrieve_query"] == "rust"
@pytest.mark.django_db
diff --git a/test/test_rank.py b/test/test_rank.py
index 0788341..be14d81 100644
--- a/test/test_rank.py
+++ b/test/test_rank.py
@@ -64,6 +64,26 @@ def test_search_still_triggers_external_search():
assert ranker.external_search_calls == ["some query"]
+def test_search_retrieved_ranks_like_search_without_external_search():
+ """Combined Search retrieves from the index while Staan is in flight, then ranks the two
+ together. That split must rank exactly as a single search() call would."""
+ index_pages = [
+ Document("Bananas", "https://a.com/", "Bananas and apples", 1.0),
+ Document("Apples", "https://b.com/", "Only apples", 2.0),
+ Document("Bananas again", "https://a.com/", "A repeat of the first URL", 0.5),
+ ]
+ additional = [Document("Banana bread", "https://c.com/", "Bananas, baked", 3.0)]
+ ranker = _TrackingRanker()
+ ranker.tiny_index.retrieve.return_value = index_pages
+
+ retrieved = ranker.search_retrieved(ranker.retrieve("bananas "), additional)
+ searched = ranker.search("bananas ", additional, use_external_search=False)
+
+ assert [d.url for d in retrieved] == [d.url for d in searched]
+ assert sorted(d.url for d in retrieved) == ["https://a.com/", "https://c.com/"]
+ assert ranker.external_search_calls == []
+
+
def _make_retry_error(status_code: int) -> RetryError:
reason = ResponseError(f"too many {status_code} error responses")
max_retry_error = MaxRetryError(
diff --git a/test/test_staan.py b/test/test_staan.py
index 7bcf052..a6fb7d3 100644
--- a/test/test_staan.py
+++ b/test/test_staan.py
@@ -60,7 +60,7 @@ def cache_index(tmp_path):
def _patched_staan(response):
"""Stand in for the Staan API, returning `response` (or raising, if it is an error)."""
- mock = patch.object(staan.requests, "get")
+ mock = patch.object(staan.session, "get")
started = mock.start()
if isinstance(response, Exception):
started.side_effect = response