An open source, non-profit web search engine
Rank Staan results over the ones the cache keeps
Daoud Clarkec0d909f9ffbb
diff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.pyindex 5e4718a..904e2d0 100644--- a/mwmbl/tinysearchengine/staan.py+++ b/mwmbl/tinysearchengine/staan.py@@ -49,16 +49,17 @@ def staan_score(rank: int) -> float:def _documents(payload: dict, query: str) -> list[Document]:- results = payload["web"]["results"][:NUM_STAAN_RESULTS]+ # Drop what the external cache would drop (no url or no title) *before* ranking, so the+ # rank a result is scored by here is the rank it is stored at and rescored by on a cache+ # hit. Ranking over the raw list meant a first fetch and every later one disagreed on+ # `score` - a trained-on feature - whenever an unusable result came ahead of a usable one.+ results = [result for result in payload["web"]["results"] if result.get("url") and result.get("title")]documents = []- for rank, result in enumerate(results):- url = result.get("url")- if not url:- continue+ for rank, result in enumerate(results[:NUM_STAAN_RESULTS]):documents.append(Document(- title=result.get("title", ""),- url=url,+ title=result["title"],+ url=result["url"],extract=result.get("snippet") or result.get("description", ""),score=staan_score(rank),term=query,diff --git a/test/test_staan.py b/test/test_staan.pyindex 042574b..7bcf052 100644--- a/test/test_staan.py+++ b/test/test_staan.py@@ -122,6 +122,51 @@ def test_a_result_with_no_url_is_dropped(cache_index):assert len(results) == 2+def test_a_result_with_no_title_is_dropped(cache_index):+ """The cache drops untitled results, so keeping one here would serve it once only."""+ untitled = {"web": {"results": [{"url": "https://untitled.example/"}, *API_RESPONSE["web"]["results"]]}}+ mock, _ = _patched_staan(untitled)+ try:+ results = get_staan_results("rust async")+ finally:+ mock.stop()++ assert [document.url for document in results] == ["https://tokio.rs/", "https://docs.rs/tokio"]+++def test_a_fetch_and_a_cache_hit_score_alike(cache_index):+ """`score` is a trained-on feature: the first request for a query and every later one+ must agree on it, even when unusable results come ahead of usable ones."""+ unusable_first = {+ "web": {+ "results": [+ {"title": "No URL here"},+ {"url": "https://untitled.example/"},+ *API_RESPONSE["web"]["results"],+ ]+ }+ }+ mock, requested = _patched_staan(unusable_first)+ try:+ fetched = get_staan_results("rust async")+ cached = get_staan_results("rust async")+ finally:+ mock.stop()++ assert requested.call_count == 1+ assert [(d.url, d.score) for d in fetched] == [(d.url, d.score) for d in cached]+ assert [d.score for d in fetched] == [staan_score(0), staan_score(1)]+++def test_a_response_with_no_titled_results_is_empty_on_first_fetch_too(cache_index):+ mock, _ = _patched_staan({"web": {"results": [{"url": "https://untitled.example/"}]}})+ try:+ assert get_staan_results("rust async") == []+ assert get_staan_results("rust async") == []+ finally:+ mock.stop()++def test_no_more_than_num_staan_results_are_kept(cache_index):many = {"web": {"results": [{"url": f"https://example.com/{i}", "title": str(i)} for i in range(50)]}}mock, _ = _patched_staan(many)