Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Rank Staan results over the ones the cache keeps

Daoud Clarkec0d909f9ffbb
diff --git a/mwmbl/tinysearchengine/staan.py b/mwmbl/tinysearchengine/staan.py
index 5e4718a..904e2d0 100644
--- a/mwmbl/tinysearchengine/staan.py
+++ b/mwmbl/tinysearchengine/staan.py
@@ -49,16 +49,17 @@ def staan_score(rank: int) -> float:
def _documents(payload: dict, query: str) -> list[Document]:
- results = payload["web"]["results"][:NUM_STAAN_RESULTS]
+ # Drop what the external cache would drop (no url or no title) *before* ranking, so the
+ # rank a result is scored by here is the rank it is stored at and rescored by on a cache
+ # hit. Ranking over the raw list meant a first fetch and every later one disagreed on
+ # `score` - a trained-on feature - whenever an unusable result came ahead of a usable one.
+ results = [result for result in payload["web"]["results"] if result.get("url") and result.get("title")]
documents = []
- for rank, result in enumerate(results):
- url = result.get("url")
- if not url:
- continue
+ for rank, result in enumerate(results[:NUM_STAAN_RESULTS]):
documents.append(
Document(
- title=result.get("title", ""),
- url=url,
+ title=result["title"],
+ url=result["url"],
extract=result.get("snippet") or result.get("description", ""),
score=staan_score(rank),
term=query,
diff --git a/test/test_staan.py b/test/test_staan.py
index 042574b..7bcf052 100644
--- a/test/test_staan.py
+++ b/test/test_staan.py
@@ -122,6 +122,51 @@ def test_a_result_with_no_url_is_dropped(cache_index):
assert len(results) == 2
+def test_a_result_with_no_title_is_dropped(cache_index):
+ """The cache drops untitled results, so keeping one here would serve it once only."""
+ untitled = {"web": {"results": [{"url": "https://untitled.example/"}, *API_RESPONSE["web"]["results"]]}}
+ mock, _ = _patched_staan(untitled)
+ try:
+ results = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert [document.url for document in results] == ["https://tokio.rs/", "https://docs.rs/tokio"]
+
+
+def test_a_fetch_and_a_cache_hit_score_alike(cache_index):
+ """`score` is a trained-on feature: the first request for a query and every later one
+ must agree on it, even when unusable results come ahead of usable ones."""
+ unusable_first = {
+ "web": {
+ "results": [
+ {"title": "No URL here"},
+ {"url": "https://untitled.example/"},
+ *API_RESPONSE["web"]["results"],
+ ]
+ }
+ }
+ mock, requested = _patched_staan(unusable_first)
+ try:
+ fetched = get_staan_results("rust async")
+ cached = get_staan_results("rust async")
+ finally:
+ mock.stop()
+
+ assert requested.call_count == 1
+ assert [(d.url, d.score) for d in fetched] == [(d.url, d.score) for d in cached]
+ assert [d.score for d in fetched] == [staan_score(0), staan_score(1)]
+
+
+def test_a_response_with_no_titled_results_is_empty_on_first_fetch_too(cache_index):
+ mock, _ = _patched_staan({"web": {"results": [{"url": "https://untitled.example/"}]}})
+ try:
+ assert get_staan_results("rust async") == []
+ assert get_staan_results("rust async") == []
+ finally:
+ mock.stop()
+
+
def test_no_more_than_num_staan_results_are_kept(cache_index):
many = {"web": {"results": [{"url": f"https://example.com/{i}", "title": str(i)} for i in range(50)]}}
mock, _ = _patched_staan(many)