An open source, non-profit web search engine
Cache the rank rather than the score, and serve the pool the model was trained on
Daoud Clarke919d223fcf6d
diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex 624a847..c4e2939 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -23,24 +23,58 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):index_path = Path(settings.DATA_PATH) / index_nametry:existing_index = TinyIndex(item_factory=Document, index_path=index_path)- print("======================================")- print(f"Found existing index at {index_path}")- print("======================================")- if existing_index.page_size != PAGE_SIZE or existing_index.num_pages != num_pages:- message = (f"Existing index page sizes ({existing_index.page_size}) or number of pages "- f"({existing_index.num_pages}) do not match")- if not rebuild_on_mismatch:- raise ValueError(message)- print(f"{message} - rebuilding {index_path}")- index_path.unlink()- TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=num_pages,- page_size=PAGE_SIZE)except FileNotFoundError:print("======================================")print("Index not found - creating a new index")print("======================================")- TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=num_pages,- page_size=PAGE_SIZE)+ _create_index_file(index_path, num_pages)+ return+ except ValueError as e:+ # A file that is there but does not parse as an index: TinyIndexMetadata.from_bytes+ # rejects it before there is a page size to compare, so the size check below never+ # sees it. For a disposable index that is the same situation as a size mismatch -+ # unusable contents, cheap to rebuild - and crashing the boot on it would be the+ # thing rebuild_on_mismatch exists to avoid.+ if not rebuild_on_mismatch:+ raise+ print(f"{index_path} is not a readable index ({e}) - rebuilding")+ _replace_index_file(index_path, num_pages)+ return++ print("======================================")+ print(f"Found existing index at {index_path}")+ print("======================================")+ if existing_index.page_size == PAGE_SIZE and existing_index.num_pages == num_pages:+ return++ message = (f"Existing index page sizes ({existing_index.page_size}) or number of pages "+ f"({existing_index.num_pages}) do not match")+ if not rebuild_on_mismatch:+ raise ValueError(message)+ print(f"{message} - rebuilding {index_path}")+ _replace_index_file(index_path, num_pages)+++def _create_index_file(index_path, num_pages):+ from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE+ TinyIndex.create(item_factory=Document, index_path=index_path, num_pages=num_pages,+ page_size=PAGE_SIZE)+++def _replace_index_file(index_path, num_pages):+ """Build the replacement alongside the old file and rename it into place.++ unlink() and then create() is not one step. It leaves a window with no file there at+ all, and on a deploy where the outgoing and incoming containers share /data the+ outgoing one's open handles go on writing to the unlinked inode while the new file is+ written beside it - two live indexes under one name, one of which nobody will ever read+ again. A rename is atomic: every reader has either the whole old file or the whole new+ one. Same directory, so it is a rename and not a copy.+ """+ temp_path = index_path.parent / f"{index_path.name}.rebuild"+ temp_path.unlink(missing_ok=True)+ _create_index_file(temp_path, num_pages)+ temp_path.replace(index_path)def create_index_db():diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.pyindex 1e88086..5f05d2e 100644--- a/mwmbl/indexer/external_cache.py+++ b/mwmbl/indexer/external_cache.py@@ -19,12 +19,14 @@ lets this module keep a negative-cache sentinel - see EXTERNAL_CACHE_EMPTY_URL.Entries are keyed by a keyed hash of the query alone and carry their source as an int inthe document itself - a DocumentSource, exactly as the search index carries DocumentState.+What an entry stores in place of a score is the *rank* the provider gave it, because a+score belongs to whoever is ranking - see _entries_to_store.Every provider's results for one query therefore land on one page: the second provider'slookup reads a page the first one just warmed, rather than faulting in a second 4 KB pagefrom a 15 GB file. So the file says which provider each entry came from, and nothing about-which query. Be-straight about what that does and does not do: it removes the query *text*, not the fact-that somebody searched in that area, since an entry still holds the results that came back.+which query. Be straight about what that does and does not do: it removes the query *text*,+not the fact that somebody searched in that area, since an entry still holds the results+that came back.That is strictly less than the disk cache gave away."""import hashlib@@ -37,19 +39,20 @@ from typing import Optionalfrom django.conf import settingsfrom mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndex-from mwmbl.tokenizer import tokenizelogger = getLogger(__name__)-# Terms under this prefix are cache entries rather than words anybody searched for.-EXTERNAL_CACHE_TERM_PREFIX = "#ext-"-# A query a provider returned nothing for is stored as a single document under this URL, so# that "we asked and there was nothing" stays distinguishable from "we never asked".# Without it those queries are re-fetched forever. It is filtered out on read and never# leaves this module.-EXTERNAL_CACHE_EMPTY_URL = "mwmbl:external-cache-empty"+#+# The empty string, because there will be a lot of these and an entry that holds no result+# should cost as close to nothing as an entry can. Nothing real can be mistaken for it:+# _entries_to_store drops any document without a url, so an empty url only ever means the+# sentinel.+EXTERNAL_CACHE_EMPTY_URL = ""def external_cache_path() -> Path:@@ -75,9 +78,16 @@ def get_cache_index() -> TinyIndex:if _cache_index is None or _cache_index_path != path:if _cache_index is not None:_cache_index.__exit__(None, None, None)- _cache_index = TinyIndex(item_factory=Document, index_path=path)- _cache_index.__enter__()- _cache_index_path = path+ _cache_index, _cache_index_path = None, None+ # Both globals are assigned only once __enter__ has returned. Assigning the index+ # first and the path after would, if __enter__ raised (the file removed between+ # __init__ and open, EMFILE, a failed mmap), leave a half-open index behind that+ # every later call would take the path-mismatch branch on and call __exit__ on -+ # an AttributeError on its None mmap, swallowed as a cache miss, for the life of+ # the worker. A failed open has to leave this exactly as it found it.+ index = TinyIndex(item_factory=Document, index_path=path)+ index.__enter__()+ _cache_index, _cache_index_path = index, pathreturn _cache_index@@ -95,9 +105,23 @@ def external_cache_term(query: str) -> str:queries. Keyed, the mapping is not reproducible without the key. Rotating SECRET_KEYinvalidates the cache, which is harmless - it is a cache.- The query is normalised through tokenize() first, so "Python" and " python " share an- entry. The disk cache keyed on the request URL built from the raw string, so they did- not.+ Normalised on whitespace and case, so "Python" and " python " share an entry. The disk+ cache keyed on the request URL built from the raw string, so they did not.++ Deliberately not tokenize(): the document tokenizer is not a total function on queries.+ It drops the last two tokens from anything ending in "…" and returns [] for a query+ that is all whitespace, so "python asyncio…" and "python tutorial…" would both key on+ "python" and serve each other's results, and every whitespace-only query would share+ one entry. A key collision here is silent - one query is answered with another's+ results - so the normalisation has to be one that cannot merge two distinct queries.+ errors="ignore" on the encode keeps it total over strings the codec cannot represent+ (a lone surrogate), which is what clean_unicode() gave us inside tokenize: this runs+ outside the try in get_cached_external_results, so raising here would cost a search its+ results rather than a cache lookup.++ Bare, with no prefix on it: the file holds nothing but cache entries, so there is no+ real term for a namespace to keep it apart from. The prefix was needed when this lived+ in the search index, and every byte of it was stored on every entry.64 bits of digest is enough. Two distinct queries colliding here would serve onequery's results for the other, so it is worth being explicit: at 30M live queries the@@ -107,9 +131,10 @@ def external_cache_term(query: str) -> str:are page collisions, which are a different thing entirely; see EXTERNAL_CACHE_NUM_PAGESin settings_prod."""- normalised = " ".join(tokenize(query))- digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8"), hashlib.sha256)- return f"{EXTERNAL_CACHE_TERM_PREFIX}{digest.hexdigest()[:16]}"+ normalised = " ".join(query.split()).casefold()+ digest = hmac.new(settings.SECRET_KEY.encode("utf8"),+ normalised.encode("utf8", errors="ignore"), hashlib.sha256)+ return digest.hexdigest()[:16]def is_fresh(document: Document, now: int) -> bool:@@ -157,14 +182,17 @@ def get_cached_external_results(source: DocumentSource, query: str,if not entries:return None- # Handed back with term set to the query, and the provider's own state and source- # preserved, exactly as a live fetch builds them. The hashed term is how the entry is- # stored, not something the ranker should ever see.- results = [Document(title=document.title, url=document.url, extract=document.extract,- score=document.score, term=query, state=document.state,- source=document.source)- for document in entries if document.url != EXTERNAL_CACHE_EMPTY_URL]- return sorted(results, key=lambda document: -(document.score or 0.0))+ # Handed back in the provider's own rank order, with term set to the query and the+ # provider's state and source preserved, exactly as a live fetch builds them. The+ # hashed term is how the entry is stored, not something the ranker should ever see.+ #+ # score comes back None. What is stored is the rank (see _entries_to_store), and a rank+ # is not a score: turning one into the other needs the caller's scale, which the cache+ # does not know and must not guess. Callers score by position - get_wiki_results does.+ ranked = sorted(entries, key=lambda document: document.score or 0.0)+ return [Document(title=document.title, url=document.url, extract=document.extract,+ score=None, term=query, state=document.state, source=document.source)+ for document in ranked if document.url != EXTERNAL_CACHE_EMPTY_URL]def store_external_results(source: DocumentSource, query: str, documents: list[Document],@@ -217,6 +245,15 @@ def _entries_to_store(source: DocumentSource, term: str, documents: list[DocumenEvery entry carries its source, the sentinel included - an entry has to say whichprovider it came from without the key, and is_fresh has nothing else to go on. `state`is carried through from whatever the provider produced rather than stamped here.++ The score slot holds the document's *rank* in what the provider returned, not the score+ it was handed in with. A caller's score is on the caller's scale - get_wiki_results+ derives one from max_wiki_results - so storing it means an entry written by a caller+ asking for five results answers a caller asking for three with the wrong numbers, and+ that number is a feature of the LTR model. A rank is the provider's own statement about+ the result and means the same thing to every caller. Storing it explicitly rather than+ relying on list order also keeps the round trip from depending on the page preserving+ the order entries were written in."""usable = [document for document in documents if document.url and document.title]if not usable:@@ -224,6 +261,6 @@ def _entries_to_store(source: DocumentSource, term: str, documents: list[Documenlast_crawled=now, source=source)]return [Document(title=document.title, url=document.url, extract=document.extract,- score=document.score, term=term, state=document.state, last_crawled=now,+ score=float(rank), term=term, state=document.state, last_crawled=now,source=source)- for document in usable]+ for rank, document in enumerate(usable)]diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex 20aa783..21ce5f4 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -36,7 +36,7 @@ def run():ranker = MMRRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))# ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3) # MMR off# ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())- # ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter(), max_wiki_results=3)+ # ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())model = MwmblRankingModel(ranker)evaluate(model, use_test=True)diff --git a/mwmbl/rankeval/ltr/dataset.py b/mwmbl/rankeval/ltr/dataset.pyindex e2d980a..91ffdbf 100644--- a/mwmbl/rankeval/ltr/dataset.py+++ b/mwmbl/rankeval/ltr/dataset.py@@ -27,7 +27,6 @@ def get_dataset(completer):completer,return_none_if_no_mwmbl_results=True,score_threshold=float('-inf'),- max_wiki_results=5,)gold_standard = pd.read_csv(RANKINGS_DATASET_TRAIN_PATH, index_col=0)dataset = []diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 96bc2a0..eebfd11 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -38,6 +38,9 @@ tiny_index.__enter__()ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))# Diversity is applied by the wrapping MMRRanker, which demotes (rather than drops)# same-domain / near-duplicate results. Unwrap to disable diversity.-ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True, num_wiki_results=3))+# num_wiki_results is left at its default on purpose - see NUM_WIKI_RESULTS. Serving+# passed 3 here while mwmbl.rankeval.ltr.dataset trained on 5, so the model was ranking a+# pool a size it had never been fitted to.+ranker = MMRRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True))batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex 38d2e04..1cb1f14 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -36,20 +36,25 @@ EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'# by hash: the queries per page are Poisson-distributed, not evenly spread, so the pages# that run over start truncating while empty ones sit unused.#-# A 4 KB page holds ~13 cached queries - 37 documents at 3 results a query - measured by-# packing the 11,693 real Wikipedia responses left in the old devdata/request_cache until-# the page overflowed. That is in line with the live index, which averages 35.4 documents a-# page, and it is worth stating how it was measured because a first pass using synthetic-# random-vocabulary text got 8, 60% low: zstd exploits the redundancy in real prose and-# there is a lot of it. Real extracts are ~154 characters.+# A 4 KB page holds ~37 documents, measured by packing the 11,693 real Wikipedia responses+# left in the old devdata/request_cache until the page overflowed. That is in line with the+# live index, which averages 35.4 documents a page, and it is worth stating how it was+# measured because a first pass using synthetic random-vocabulary text got 8, 60% low: zstd+# exploits the redundancy in real prose and there is a lot of it. Real extracts are ~154+# characters.#-# At an average load of L queries per page the fraction still resident is E[min(K, 13)] / L+# An entry holds WIKI_FETCH_LIMIT documents a query, so at the current 5 that is ~7 cached+# queries a page. It was 3 a query (~13 queries a page) when this was first sized, and the+# capacity below is the price of serving the pool size the LTR model was trained on - see+# NUM_WIKI_RESULTS - rather than a choice made here.+#+# At an average load of L queries per page the fraction still resident is E[min(K, 7)] / L# for K ~ Poisson(L), which gives, for one provider:#-# L = 8.2 -> 99% retained -> 30.9M distinct queries-# L = 10.9 -> 95% retained -> 40.8M-# L = 12.7 -> 90% retained -> 47.6M-# L = 13 -> 89% retained -> 48.8M (the naive number)+# L = 3.4 -> 99% retained -> 12.7M distinct queries+# L = 5.0 -> 95% retained -> 18.6M+# L = 6.1 -> 90% retained -> 22.9M+# L = 7 -> 85% retained -> 26.3M (the naive number)## A query cached from several providers holds one entry per provider, so divide by the# number of providers: with two, ~15M queries at 99% and ~24M at 90%. Longer extracts fitdiff --git a/mwmbl/templates/_search_bar.html b/mwmbl/templates/_search_bar.htmlindex 279b70a..2b027a8 100644--- a/mwmbl/templates/_search_bar.html+++ b/mwmbl/templates/_search_bar.html@@ -33,11 +33,20 @@It lives here in the header rather than inside .main because .main's contents arereplaced on every keystroke, which would destroy and re-arm this element each time.- hx-sync on both keeps a slow response from landing on top of a newer fast one. #}+ hx-sync on both keeps a slow response from landing on top of a newer fast one.++ The trigger is `input`, not `keyup`: keyup fires for arrow keys, Shift, Ctrl and+ Escape too, so every one of those scheduled a full search 800ms later against a+ query that had not changed - a wasted search, and a swap of .main that can land on+ top of a curate the user made in the meantime. `input` fires only when the value+ actually changes. The `changed` modifier is not the fix here: htmx keeps lastValue+ on the element being listened to, which for both triggers is the same+ .search-bar-input, so the 100ms trigger above would consume the change and this one+ would never fire at all. #}<spanclass="external-search"hx-get="/app/home/?external=1"- hx-trigger="keyup delay:800ms from:.search-bar-input"+ hx-trigger="input delay:800ms from:.search-bar-input"hx-target=".main"hx-include=".search-bar-input"hx-sync="closest form:replace"diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex cdebeee..a4b089c 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -21,6 +21,10 @@ METADATA_SIZE = 4096PAGE_SIZE = 4096+# Where `state` sits in Document.as_tuple(). get_page needs it to blank an unreadable state+# without disturbing the values around it.+STATE_INDEX = 5+logger = getLogger(__name__)@@ -386,8 +390,15 @@ class TinyIndex(Generic[T]):items.append(self.item_factory(*item))except ValueError as e:logger.error(f"Invalid item in index page {i}, fixing state to None: {e}. Item: {item}")- # The state value is the last element if present; strip it and retry with state=None- fixed_item = item[:-1] if len(item) > 3 else item+ # state is at index 5 of Document.as_tuple's layout, and it is not the last+ # element: source follows it on external cache entries. Blank it where it+ # is rather than truncating - truncating would take source off a cache+ # entry, leave the unknown state value in place, and lose the document to+ # the same ValueError on the retry.+ if len(item) <= STATE_INDEX:+ logger.error(f"Could not recover item in index page {i}, skipping. Item: {item}")+ continue+ fixed_item = list(item[:STATE_INDEX]) + [None] + list(item[STATE_INDEX + 1:])try:items.append(self.item_factory(*fixed_item))except Exception as e2:diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.pyindex cca1bc0..49acbc3 100644--- a/mwmbl/tinysearchengine/ltr_rank.py+++ b/mwmbl/tinysearchengine/ltr_rank.py@@ -10,7 +10,7 @@ from sklearn.base import BaseEstimatorfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex-from mwmbl.tinysearchengine.rank import Ranker, get_wiki_results+from mwmbl.tinysearchengine.rank import NUM_WIKI_RESULTS, Ranker, get_wiki_resultsclass LTRRanker(Ranker):@@ -37,7 +37,9 @@ class LTRRanker(Ranker):include_wiki : boolWhether to include Wikipedia results via external search.num_wiki_results : int- Maximum number of Wikipedia results to include.+ Maximum number of Wikipedia results to include. Leave it alone unless you are+ retraining: the LTR dataset is built with the same default, and a serving pool of a+ different size to the trained one is what NUM_WIKI_RESULTS exists to prevent."""def __init__(@@ -46,7 +48,7 @@ class LTRRanker(Ranker):completer: Completer,model,include_wiki: bool = True,- num_wiki_results: int = 5,+ num_wiki_results: int = NUM_WIKI_RESULTS,):super().__init__(tiny_index, completer)self.model = modeldiff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex d36955b..f6eedd1 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -426,9 +426,33 @@ class HeuristicRanker(Ranker):return filtered_results-WIKI_SEARCH_API_URL = "https://en.wikipedia.org/w/api.php?action=query&list=search&srsearch={query}&format=json"+WIKI_SEARCH_API_URL = ("https://en.wikipedia.org/w/api.php?action=query&list=search"+ "&srsearch={query}&srlimit={limit}&format=json")WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"+# How many wiki results a caller gets unless it says otherwise, and the number the LTR+# model was trained against - mwmbl.rankeval.ltr.dataset builds its training data through+# HeuristicAndWikiRanker with this default. Serving used to pass 3 here while training used+# 5, so the model was scoring production pools it had never seen the shape of. One constant,+# used everywhere, is what keeps them from drifting apart again.+NUM_WIKI_RESULTS = 5++# What we ask Wikipedia for, and therefore what a cache entry holds. Fixed rather than the+# caller's max_wiki_results so that one entry serves every caller: a cache warmed by a+# caller wanting three must not be a short answer to a caller wanting five. Left unset the+# API returns 10 by default, so this narrows the response as well.+WIKI_FETCH_LIMIT = NUM_WIKI_RESULTS++# The score a wiki result carries into the LTR model, top result first, counting down.+#+# A function of rank alone, deliberately. It used to be max_wiki_results + 1 - i, so the+# same Wikipedia result scored 6 for a caller asking for five and 4 for a caller asking for+# three - and `score` is a feature the model reads, so the number a caller passed for how+# *many* results it wanted silently moved the ranking of the ones it got. Anchored at the+# top instead, rank 0 is 6.0 for everybody, which is what the deployed model was trained+# on (max_wiki_results=5 there). Changing it changes a trained-on feature: retrain.+WIKI_TOP_SCORE = 6.0+# Wikipedia rate-limits (HTTP 429, with a non-JSON body) bursts of search-API# calls — e.g. an evaluation run that queries it once per gold query. Retry with# backoff, honouring any Retry-After, so a transient 429 doesn't silently drop a@@ -481,7 +505,12 @@ def clean_html(s: str):return html.unescape(HTML_TAG_REGEX.sub('', s))-def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:+def wiki_score(rank: int) -> float:+ """The `score` feature for a wiki result at 0-based `rank`. See WIKI_TOP_SCORE."""+ return WIKI_TOP_SCORE - rank+++def get_wiki_results(s: str, max_wiki_results: int = NUM_WIKI_RESULTS) -> list[Document]:# MAX_QUERY_CHARS was only ever applied in HeuristicAndWikiRanker, which is the eval# ranker - the production LTRRanker path sent the untruncated query straight out.query = s[:MAX_QUERY_CHARS]@@ -490,7 +519,13 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:# should still be served while we are backing off from a rate limit.cached = get_cached_external_results(DocumentSource.WIKIPEDIA, query)if cached is not None:- return cached[:max_wiki_results]+ # The cache stores the rank Wikipedia gave, not a score - scoring is this function's+ # to do, and doing it here rather than at write time is what makes an entry mean the+ # same thing to every caller. The documents are ours; nothing else holds them.+ wiki_results = cached[:max_wiki_results]+ for rank, document in enumerate(wiki_results):+ document.score = wiki_score(rank)+ return wiki_resultsif _wiki_circuit_open():return []@@ -502,7 +537,9 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:try:with requests.Session() as session:session.mount("https://en.wikipedia.org", HTTPAdapter(max_retries=WIKI_RETRY))- response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query), headers=headers, timeout=5)+ response = session.get(+ WIKI_SEARCH_API_URL.format(query=escaped_query, limit=WIKI_FETCH_LIMIT),+ headers=headers, timeout=5)wiki_response = response.json()except RetryError as e:if _is_wiki_rate_limited(e):@@ -527,15 +564,19 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:return []wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- max_wiki_results + 1 - i, query, state=DocumentState.FROM_WIKI,+ wiki_score(rank), query, state=DocumentState.FROM_WIKI,source=DocumentSource.WIKIPEDIA)- for i, result in enumerate(wiki_response['query']['search'][:max_wiki_results])]+ for rank, result in enumerate(wiki_response['query']['search'][:WIKI_FETCH_LIMIT])]# Only a well-formed answer is stored, including one with no results in it. Every path# above returns [] without storing, so a transient failure is never remembered as# "Wikipedia has nothing for this".+ #+ # The whole response is stored and the caller's cut taken afterwards, so that a caller+ # wanting fewer than WIKI_FETCH_LIMIT does not leave a short entry behind for the next+ # caller to be silently short-changed by.store_external_results(DocumentSource.WIKIPEDIA, query, wiki_results)- return wiki_results+ return wiki_results[:max_wiki_results]class HeuristicAndWikiRanker(HeuristicRanker):@@ -545,7 +586,7 @@ class HeuristicAndWikiRanker(HeuristicRanker):completer: Completer,return_none_if_no_mwmbl_results: bool = False,score_threshold: float = 0.0,- max_wiki_results: int = 5+ max_wiki_results: int = NUM_WIKI_RESULTS):super().__init__(tiny_index, completer, score_threshold)self.return_none_if_no_mwmbl_results = return_none_if_no_mwmbl_resultsdiff --git a/scripts/llm_relabel_pass2_collect.py b/scripts/llm_relabel_pass2_collect.pyindex 11cee79..a13cc0a 100644--- a/scripts/llm_relabel_pass2_collect.py+++ b/scripts/llm_relabel_pass2_collect.py@@ -56,7 +56,6 @@ ALWAYS_ON = {name for name in SOURCES if get_meta(name).always_on}_std_ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter(),return_none_if_no_mwmbl_results=True, score_threshold=float("-inf"),- max_wiki_results=5,)diff --git a/test/test_external_cache.py b/test/test_external_cache.pyindex 4bb68dd..34859be 100644--- a/test/test_external_cache.py+++ b/test/test_external_cache.py@@ -10,6 +10,7 @@ import jsonimport randomimport stringimport time+from pathlib import Pathfrom unittest.mock import MagicMock, patchimport pytest@@ -18,7 +19,6 @@ from django.test import override_settingsfrom mwmbl.apps import create_indexfrom mwmbl.indexer.external_cache import (EXTERNAL_CACHE_EMPTY_URL,- EXTERNAL_CACHE_TERM_PREFIX,external_cache_path,external_cache_term,get_cached_external_results,@@ -28,12 +28,14 @@ from mwmbl.indexer.external_cache import (from mwmbl.tinysearchengine import rankfrom mwmbl.tinysearchengine.indexer import (PAGE_SIZE, Document, DocumentSource, DocumentState,TinyIndex)-from mwmbl.tinysearchengine.rank import Ranker, get_wiki_results+from mwmbl.tinysearchengine.rank import (WIKI_FETCH_LIMIT, Ranker, get_wiki_results,+ wiki_score)NUM_PAGES = 8# state and source are set because get_wiki_results sets them, and the cache preserves what-# the provider produced rather than stamping its own.+# the provider produced rather than stamping its own. The scores are not preserved - what+# an entry stores is the rank - and the tests below rely on that.PYTHON = Document("Python (programming language)", "https://en.wikipedia.org/wiki/Python","A high-level programming language.", 3.0, state=DocumentState.FROM_WIKI,source=DocumentSource.WIKIPEDIA)@@ -84,11 +86,25 @@ def test_cache_term_is_stable_and_normalised():def test_cache_term_does_not_contain_the_query():term = external_cache_term("something private")- assert term.startswith(EXTERNAL_CACHE_TERM_PREFIX)assert "something" not in termassert "private" not in term+def test_distinct_queries_do_not_share_a_term():+ """A term collision is silent - one query is answered with another's results - so the+ normalisation must not merge two queries. tokenize() did: it drops the last two tokens+ from anything ending in an ellipsis, and returns [] for whitespace."""+ assert external_cache_term("python asyncio…") != external_cache_term("python tutorial…")+ assert external_cache_term("python asyncio…") != external_cache_term("python")+ assert external_cache_term(" ") != external_cache_term("\t\n x")+++def test_the_term_is_derived_from_anything_that_is_a_string():+ """It runs outside the try in get_cached_external_results, so raising here costs a+ search its results rather than a cache lookup."""+ assert external_cache_term("\udce9 lone surrogate")++def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():"""A bare digest of a one-word query is brute-forceable from a wordlist; keyed onSECRET_KEY the term->query mapping is not reproducible without the key."""@@ -109,11 +125,31 @@ def test_results_come_back_from_the_index(cache_index):cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")- assert [document.url for document in cached] == [PYTHON.url, MONTY.url] # descending score- assert [document.score for document in cached] == [3.0, 2.0]+ assert [document.url for document in cached] == [PYTHON.url, MONTY.url]assert all(document.state == DocumentState.FROM_WIKI for document in cached)+def test_the_providers_order_is_what_comes_back(cache_index):+ """The cache stores the rank the provider gave, not the score it was handed. Storing+ the score would make an entry mean different things to callers asking for different+ numbers of results - see wiki_score."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [MONTY, PYTHON])++ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")++ assert [document.url for document in cached] == [MONTY.url, PYTHON.url]+++def test_a_cached_result_carries_no_score(cache_index):+ """Scoring is the caller's: a rank is what the provider said, a score is on a scale the+ cache does not know."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])++ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")++ assert [document.score for document in cached] == [None, None]++def test_a_hit_is_indistinguishable_from_a_live_fetch(cache_index):"""The hashed term is a storage detail. Handing it to the ranker would make a cachedresult differ from the identical one fetched live."""@@ -348,6 +384,72 @@ def test_a_cache_hit_does_not_call_wikipedia(cache_index):assert session.get.call_count == 0assert [document.url for document in results] == [PYTHON.url]+ assert [document.score for document in results] == [wiki_score(0)]+++def test_wikipedia_is_asked_for_a_fixed_number_of_results(cache_index):+ """Not the caller's max: what comes back is what the entry holds, and an entry warmed+ by a caller wanting three must not be a short answer for a caller wanting five."""+ mock, session = _patched_wikipedia(_wiki_api_response("Python"))+ try:+ get_wiki_results("python", 2)+ finally:+ mock.stop()++ assert f"srlimit={WIKI_FETCH_LIMIT}" in session.get.call_args[0][0]+++def test_a_short_request_does_not_leave_a_short_entry(cache_index):+ """The whole response is stored and the caller's cut taken afterwards. Storing only+ what the first caller wanted would silently short-change every later one for the length+ of the TTL - a cache miss is visible, a truncated hit is not."""+ titles = [f"Title {i}" for i in range(WIKI_FETCH_LIMIT)]+ mock, _ = _patched_wikipedia(_wiki_api_response(*titles))+ try:+ assert len(get_wiki_results("python", 2)) == 2+ finally:+ mock.stop()++ mock, session = _patched_wikipedia(_wiki_api_response(*titles))+ try:+ results = get_wiki_results("python", WIKI_FETCH_LIMIT)+ finally:+ mock.stop()++ assert session.get.call_count == 0+ assert len(results) == WIKI_FETCH_LIMIT+++def test_a_results_score_does_not_depend_on_how_many_were_asked_for(cache_index):+ """`score` is a feature of the LTR model. When it was max_wiki_results + 1 - i the same+ Wikipedia result scored 6 for a caller wanting five and 4 for a caller wanting three,+ so how many results a caller asked for moved the ranking of the ones it got."""+ titles = [f"Title {i}" for i in range(WIKI_FETCH_LIMIT)]+ mock, _ = _patched_wikipedia(_wiki_api_response(*titles))+ try:+ three = get_wiki_results("python", 3)+ five = get_wiki_results("python", 5)+ finally:+ mock.stop()++ assert [document.score for document in three] == [wiki_score(0), wiki_score(1), wiki_score(2)]+ assert [document.score for document in five[:3]] == [document.score for document in three]+++def test_a_cache_hit_scores_the_same_as_the_live_fetch_that_filled_it(cache_index):+ """The scores are derived on the way out rather than stored, so this is the assertion+ that keeps the derivation honest."""+ titles = [f"Title {i}" for i in range(WIKI_FETCH_LIMIT)]+ mock, _ = _patched_wikipedia(_wiki_api_response(*titles))+ try:+ live = get_wiki_results("python", 3)+ finally:+ mock.stop()++ cached = get_wiki_results("python", 3)++ assert [(document.url, document.score) for document in cached] == \+ [(document.url, document.score) for document in live]def test_a_hit_is_served_while_the_circuit_breaker_is_open(cache_index):@@ -540,3 +642,51 @@ def test_a_resized_search_index_still_refuses_to_start(tmp_path):with override_settings(DATA_PATH=str(tmp_path)), pytest.raises(ValueError):create_index("index.tinysearch", 16)+++def test_an_unreadable_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_path):+ """A file that does not parse as an index at all is rejected before there is a page+ size to compare, so it never reaches the size check. For a disposable index it is the+ same situation: unusable contents, cheap to rebuild."""+ path = tmp_path / "external-cache.tinysearch"+ path.write_bytes(b"not an index" + bytes(PAGE_SIZE))++ with override_settings(DATA_PATH=str(tmp_path)):+ create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)++ with TinyIndex(Document, str(path), 'r') as index:+ assert index.num_pages == 16+++def test_an_unreadable_search_index_still_refuses_to_start(tmp_path):+ path = tmp_path / "index.tinysearch"+ path.write_bytes(b"not an index" + bytes(PAGE_SIZE))++ with override_settings(DATA_PATH=str(tmp_path)), pytest.raises(ValueError):+ create_index("index.tinysearch", 16)+++def test_a_rebuild_leaves_no_window_with_the_index_missing(tmp_path):+ """The replacement is built alongside and renamed over the old file. unlink() then+ create() leaves a gap where the path does not exist, and on a deploy sharing /data the+ outgoing container goes on writing to the unlinked inode."""+ path = tmp_path / "external-cache.tinysearch"+ TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=4, page_size=PAGE_SIZE)+ original_inode = path.stat().st_ino+ seen = []++ real_create = TinyIndex.create++ def watched_create(*args, **kwargs):+ seen.append((path.exists(), kwargs.get("index_path", args[1] if len(args) > 1 else None)))+ return real_create(*args, **kwargs)++ with override_settings(DATA_PATH=str(tmp_path)), \+ patch.object(TinyIndex, "create", side_effect=watched_create):+ create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)++ assert seen, "nothing was rebuilt, so this would pass for the wrong reason"+ assert all(existed for existed, _ in seen), "the old index was removed before the new one existed"+ assert all(Path(str(target)) != path for _, target in seen), "the new index was written over the old one in place"+ assert path.stat().st_ino != original_inode+ assert not list(tmp_path.glob("*.rebuild"))diff --git a/test/test_indexer.py b/test/test_indexer.pyindex ad85626..97fcb3e 100644--- a/test/test_indexer.py+++ b/test/test_indexer.py@@ -3,8 +3,8 @@ from tempfile import TemporaryDirectoryfrom zstandard import ZstdCompressor-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, _binary_search_fitting_size, \- _trim_items_to_page, _pad_to_page_size, _get_page_data+from mwmbl.tinysearchengine.indexer import STATE_INDEX, TinyIndex, Document, DocumentSource, \+ DocumentState, _binary_search_fitting_size, _trim_items_to_page, _pad_to_page_size, _get_page_datadef test_create_index():@@ -157,3 +157,34 @@ def test_document_backward_compat_old_six_element_tuple():doc = Document(*old_tuple)assert doc.user_ids is Noneassert doc.last_crawled is None+++def test_state_index_matches_the_tuple_layout():+ """get_page blanks the state by position, so the constant has to track as_tuple()."""+ document = Document("Title", "https://example.com", "Extract", 1.0, "term",+ DocumentState.FROM_WIKI, None, 123, DocumentSource.WIKIPEDIA)++ assert document.as_tuple()[STATE_INDEX] == DocumentState.FROM_WIKI.value+++def test_a_document_with_an_unreadable_state_keeps_its_other_values():+ """State values are written to disk, so a page can hold one this build does not know.+ Blanking it where it sits is what keeps the document: it is not the last value in the+ tuple - source follows it on external cache entries - so stripping the tail would take+ source off instead, leave the bad state in place, and drop the document on the retry."""+ with TemporaryDirectory() as temp_dir:+ index_path = str(Path(temp_dir) / 'temp-index.tinysearch')+ TinyIndex.create(Document, index_path, num_pages=2, page_size=4096)+ # title, url, extract, score, term, state, user_ids, last_crawled, source+ item = ["Title", "https://example.com", "Extract", 1.0, "term", 99, None, 123,+ DocumentSource.WIKIPEDIA.value]+ with TinyIndex(Document, index_path, 'w') as index:+ index._write_page([item], 0)++ with TinyIndex(Document, index_path, 'r') as index:+ documents = index.get_page(0)++ assert len(documents) == 1+ assert documents[0].state is None+ assert documents[0].source == DocumentSource.WIKIPEDIA+ assert documents[0].last_crawled == 123diff --git a/test/test_rank.py b/test/test_rank.pyindex 2e4b73c..928a094 100644--- a/test/test_rank.py+++ b/test/test_rank.py@@ -1,3 +1,5 @@+import re+from pathlib import Pathfrom unittest.mock import MagicMock, patchimport pytest@@ -7,7 +9,9 @@ from urllib3.exceptions import MaxRetryError, ResponseErrorfrom mwmbl.tinysearchengine import rankfrom mwmbl.tinysearchengine.indexer import Document-from mwmbl.tinysearchengine.rank import HeuristicRanker, get_wiki_results+from mwmbl.tinysearchengine.ltr_rank import LTRRanker+from mwmbl.tinysearchengine.rank import (NUM_WIKI_RESULTS, HeuristicAndWikiRanker,+ HeuristicRanker, get_wiki_results, wiki_score)def test_order_result():@@ -127,3 +131,41 @@ def test_is_wiki_rate_limited_detects_429_reason():def test_is_wiki_rate_limited_ignores_non_429_reason():assert not rank._is_wiki_rate_limited(_make_retry_error(503))+++# ---------------------------------------------------------------------------+# Keeping the wiki results the model is served the ones it was trained on+# ---------------------------------------------------------------------------++def test_the_wiki_score_depends_on_rank_alone():+ """`score` is a feature of the LTR model. It used to be max_wiki_results + 1 - i, so a+ caller asking for a different number of results moved a feature value and with it the+ ranking - training built its pools with five, serving asked for three."""+ assert wiki_score(0) == 6.0+ assert [wiki_score(rank) for rank in range(3)] == [6.0, 5.0, 4.0]+++def test_every_ranker_takes_its_wiki_result_count_from_one_place():+ """The count the dataset is built with and the count production ranks with have to be+ the same, and what keeps them the same is that there is only one number."""+ assert LTRRanker.__init__.__defaults__[-1] == NUM_WIKI_RESULTS+ assert HeuristicAndWikiRanker.__init__.__defaults__[-1] == NUM_WIKI_RESULTS+ assert get_wiki_results.__defaults__[-1] == NUM_WIKI_RESULTS+++def test_no_call_site_passes_its_own_wiki_result_count():+ """How they came apart in the first place: mwmbl/search_setup.py passed 3 while+ mwmbl/rankeval/ltr/dataset.py trained on 5, and nothing connected the two literals. A+ call site that needs a different number needs a retrained model, so it should have to+ change NUM_WIKI_RESULTS to get one."""+ root = Path(__file__).parent.parent+ literal_count = re.compile(r"(?:num|max)_wiki_results(?:\s*:\s*int)?\s*=\s*\d")+ offenders = [+ f"{path.relative_to(root)}:{lineno}: {line.strip()}"+ for path in sorted([*root.glob("mwmbl/**/*.py"), *root.glob("scripts/**/*.py")])+ for lineno, line in enumerate(path.read_text().splitlines(), 1)+ # Code only: the comments around here talk about the counts they are explaining.+ if literal_count.search(line.split("#", 1)[0])+ ]++ assert offenders == []