An open source, non-profit web search engine
Ranking: retry Wikipedia search API on 429 rate-limiting
Daoud Clarke397797fe4ed7
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 33ad103..81a58d9 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -12,6 +12,7 @@ from pathlib import Pathfrom urllib.parse import urlparseimport numpy as np+from requests.adapters import HTTPAdapter, Retryfrom mwmbl.format import get_query_regexfrom mwmbl.hn_top_domains_filtered import DOMAINS@@ -354,6 +355,14 @@ class HeuristicRanker(Ranker):WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"+# Wikipedia rate-limits (HTTP 429, with a non-JSON body) bursts of search-API+# calls — e.g. an evaluation run that queries it once per gold query. Retry with+# backoff, honouring any Retry-After, so a transient 429 doesn't silently drop a+# query's wiki results. Live search never bursts, so this adds no latency in+# normal operation. Successful responses are cached for weeks (see request_cache).+WIKI_RETRY = Retry(total=4, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504),+ allowed_methods=frozenset({"GET"}), respect_retry_after_header=True)+def get_wiki_url(title: str):return WIKI_URL_FORMAT.format(title=title.replace(" ", "_"))@@ -373,6 +382,7 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:}try:with request_cache(timedelta(weeks=10)) as session:+ session.mount("https://en.wikipedia.org", HTTPAdapter(max_retries=WIKI_RETRY))response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query), headers=headers, timeout=5)wiki_response = response.json()except Exception: