An open source, non-profit web search engine
Generalise the Wikipedia cache to any external provider, and size it for real pages
Daoud Clarke8850373d05eb
diff --git a/.gitignore b/.gitignoreindex 4ea090d..d8fcb86 100644--- a/.gitignore+++ b/.gitignore@@ -166,8 +166,8 @@ devdata/rankeval-2024-06/remote-datasets/devdata/*.bloomdevdata/*.sqlitedevdata/request_cache/-# Generated by apps.ready(); holds cached Wikipedia responses, not source.-devdata/wiki-cache.tinysearch+# Generated by apps.ready(); holds cached external provider responses, not source.+devdata/external-cache.tinysearchdevdata/rankeval-*/front-end/node_modules/diff --git a/app.json b/app.jsonindex 5142598..9af476d 100644--- a/app.json+++ b/app.json@@ -11,7 +11,7 @@"path": "/api/v1/search/?s=rydnik","attempts": 1,"interval": 600,- "grace": 60,+ "grace": 300,"timeout": 10}]diff --git a/mwmbl/apps.py b/mwmbl/apps.pyindex 87c51f8..624a847 100644--- a/mwmbl/apps.py+++ b/mwmbl/apps.py@@ -14,8 +14,9 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):rebuild_on_mismatch is for indexes whose contents are disposable. The search index isnot one of them - a size that disagrees with settings there means somebody changedNUM_PAGES against a 400 GB file holding the only copy of the crawl, and refusing to- start is the right answer. The Wikipedia cache is the opposite case: resizing it should- be a config change, not a startup crash, and the cost of rebuilding is re-fetching.+ start is the right answer. The external results cache is the opposite case: resizing it+ should be a config change, not a startup crash, and the cost of rebuilding is+ re-fetching."""# Imports here to avoid AppRegistryNotReady exceptionfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE@@ -55,8 +56,10 @@ class MwmblConfig(AppConfig):def ready(self):create_index(settings.INDEX_NAME, settings.NUM_PAGES)# Note this writes num_pages * 4 KB page by page rather than sparsely, so the first- # boot after a size change pays for the whole file up front.- create_index(settings.WIKI_CACHE_INDEX_NAME, settings.WIKI_CACHE_NUM_PAGES,+ # boot after a size change pays for the whole file up front - ~17s for the 15 GB+ # production cache at the ~0.9 GB/s measured locally, which is what the healthcheck+ # grace in app.json has to cover.+ create_index(settings.EXTERNAL_CACHE_INDEX_NAME, settings.EXTERNAL_CACHE_NUM_PAGES,rebuild_on_mismatch=True)if settings.HAS_DATABASE:create_index_db()diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.pyindex 0ce2c49..f1ccdf1 100644--- a/mwmbl/indexer/blacklist_providers.py+++ b/mwmbl/indexer/blacklist_providers.py@@ -152,7 +152,7 @@ class RemoteListBlacklistProvider(BlacklistProvider):# *every* user of request_cache breaks - and it shares that volume with the index# itself. (get_wiki_results() used to be the worst of those, retrying a live fetch# 4 times per uncached query on the search path; it now has a cache of its own, see- # mwmbl.indexer.wiki_cache.) A big periodic download has no business in a shared+ # mwmbl.indexer.external_cache.) A big periodic download has no business in a shared# small-response cache on the index volume.response = requests.get(self.url, timeout=60)response.raise_for_status()diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.pynew file mode 100644index 0000000..1e88086--- /dev/null+++ b/mwmbl/indexer/external_cache.py@@ -0,0 +1,229 @@+"""Results from external search providers, cached in a TinyIndex of their own.++Wikipedia is the only provider using this today, but nothing here is Wikipedia-specific:+every entry is namespaced by its source, so a second provider (Staan, say) caches alongside+it in the same file without the two ever seeing each other's entries.++get_wiki_results() used to cache through mwmbl.utils.request_cache, a requests-cache+*filesystem* session rooted at REQUEST_CACHE_PATH - the same volume as the 400 GB index -+with no LRU, no size cap and nothing ever calling delete(expired=True), so it grew without+bound. Because the stored JSON holds the whole request, it also wrote the raw user query+to disk, where it sat for the length of the expiry.++This is a *separate* index file, not the search index. Caching into the search index was+tried in #357 and measured in #359: a cache entry and a real document compete for the same+4 KB page, and on a dense index storing one evicts about as much as it adds (-0.0054 NDCG+overall, -0.1927 on the queries where it fired). A file of its own also means a cache entry+can never surface in /raw or in the candidate pool for somebody else's query, which is what+lets this module keep a negative-cache sentinel - see EXTERNAL_CACHE_EMPTY_URL.++Entries are keyed by a keyed hash of the query alone and carry their source as an int in+the document itself - a DocumentSource, exactly as the search index carries DocumentState.+Every provider's results for one query therefore land on one page: the second provider's+lookup reads a page the first one just warmed, rather than faulting in a second 4 KB page+from a 15 GB file. So the file says which provider each entry came from, and nothing about+which query. Be+straight about what that does and does not do: it removes the query *text*, not the fact+that somebody searched in that area, since an entry still holds the results that came back.+That is strictly less than the disk cache gave away.+"""+import hashlib+import hmac+import time+from logging import getLogger+from pathlib import Path+from typing import Optional++from django.conf import settings++from mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndex+from mwmbl.tokenizer import tokenize++logger = getLogger(__name__)+++# Terms under this prefix are cache entries rather than words anybody searched for.+EXTERNAL_CACHE_TERM_PREFIX = "#ext-"++# A query a provider returned nothing for is stored as a single document under this URL, so+# that "we asked and there was nothing" stays distinguishable from "we never asked".+# Without it those queries are re-fetched forever. It is filtered out on read and never+# leaves this module.+EXTERNAL_CACHE_EMPTY_URL = "mwmbl:external-cache-empty"+++def external_cache_path() -> Path:+ return Path(settings.DATA_PATH) / settings.EXTERNAL_CACHE_INDEX_NAME+++_cache_index: Optional[TinyIndex] = None+_cache_index_path: Optional[Path] = None+++def get_cache_index() -> TinyIndex:+ """The read handle for this process, opened once and left open.++ Same lifetime as the main index in search_setup - mmap'd read-only for the life of the+ worker. Writers open their own short-lived 'w' handle instead, as every other writer in+ the codebase does; see store_external_results.++ Keyed on the path so that override_settings in the tests gets a fresh handle rather+ than whichever index the first test to run happened to open.+ """+ global _cache_index, _cache_index_path+ path = external_cache_path()+ if _cache_index is None or _cache_index_path != path:+ if _cache_index is not None:+ _cache_index.__exit__(None, None, None)+ _cache_index = TinyIndex(item_factory=Document, index_path=path)+ _cache_index.__enter__()+ _cache_index_path = path+ return _cache_index+++def external_cache_term(query: str) -> str:+ """The term a query's cached results are stored under, for every provider alike.++ Deliberately not keyed on the source. All of a query's providers then share one page, so+ asking a second provider costs no extra page fault - it reads the page the first lookup+ just brought in. Which provider an entry belongs to is on the entry, as a DocumentSource+ int, and that is what the readers filter on. It is also what is_fresh would key a+ per-source TTL off, since a page's other entries tell it nothing through the term.++ Keyed on SECRET_KEY rather than a bare digest: a plain hash of a one- or two-word query+ is trivially brute-forced from a wordlist, so a dump of the file would hand over the+ queries. Keyed, the mapping is not reproducible without the key. Rotating SECRET_KEY+ invalidates the cache, which is harmless - it is a cache.++ The query is normalised through tokenize() first, so "Python" and " python " share an+ entry. The disk cache keyed on the request URL built from the raw string, so they did+ not.++ 64 bits of digest is enough. Two distinct queries colliding here would serve one+ query's results for the other, so it is worth being explicit: at 30M live queries the+ birthday probability of *any* such pair existing is 2e-5. Widening the term is not free+ either - it is stored on every document, and 32 hex characters measured ~9% fewer+ entries per page - so the short term is the better trade. The collisions that do matter+ are page collisions, which are a different thing entirely; see EXTERNAL_CACHE_NUM_PAGES+ in settings_prod.+ """+ normalised = " ".join(tokenize(query))+ digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8"), hashlib.sha256)+ return f"{EXTERNAL_CACHE_TERM_PREFIX}{digest.hexdigest()[:16]}"+++def is_fresh(document: Document, now: int) -> bool:+ """Whether a cached document is still within its TTL. Untimestamped means stale.++ This runs over a page's other entries too, which belong to other queries and other+ providers, so it cannot recover the source from the term - document.source is the only+ thing that identifies them, and it is where a per-source TTL would key off.+ """+ ttl = (settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS if document.url == EXTERNAL_CACHE_EMPTY_URL+ else settings.EXTERNAL_CACHE_TTL_SECONDS)+ return document.last_crawled is not None and now - document.last_crawled < ttl+++def get_cached_external_results(source: DocumentSource, query: str,+ now: Optional[int] = None) -> Optional[list[Document]]:+ """A query's cached results from one provider.++ None means we have nothing for this (source, query) and should ask the provider. A list+ - possibly empty - means we already asked; an empty one is a query the provider had+ nothing for.+ """+ if not settings.EXTERNAL_CACHE_ENABLED:+ return None++ now = int(time.time()) if now is None else now+ term = external_cache_term(query)+ try:+ stored = get_cache_index().retrieve(term)+ except FileNotFoundError:+ logger.warning("No external results cache index at %s", external_cache_path())+ return None+ except Exception:+ # A cache we cannot read is a cache miss, never a failed search. retrieve() already+ # absorbs an unreadable page; this covers the file itself going away underneath us.+ logger.exception("Could not read the external results cache index")+ return None++ # retrieve() hands back the whole page, which is shared with every other query that+ # hashes to it, and it also lets through documents with no term at all. The term narrows+ # that to this query - across all providers, since the term does not name one - and the+ # source picks out the provider being asked about.+ entries = [document for document in stored+ if document.term == term and document.source == source and is_fresh(document, now)]+ if not entries:+ return None++ # Handed back with term set to the query, and the provider's own state and source+ # preserved, exactly as a live fetch builds them. The hashed term is how the entry is+ # stored, not something the ranker should ever see.+ results = [Document(title=document.title, url=document.url, extract=document.extract,+ score=document.score, term=query, state=document.state,+ source=document.source)+ for document in entries if document.url != EXTERNAL_CACHE_EMPTY_URL]+ return sorted(results, key=lambda document: -(document.score or 0.0))+++def store_external_results(source: DocumentSource, query: str, documents: list[Document],+ now: Optional[int] = None) -> None:+ """Store one provider's results for a query, replacing any existing entry for it.++ Never raises. A cache write that fails costs a re-fetch later; it must not cost a+ search its results.+ """+ if not settings.EXTERNAL_CACHE_ENABLED:+ return++ now = int(time.time()) if now is None else now+ term = external_cache_term(query)+ entries = _entries_to_store(source, term, documents, now)++ try:+ with TinyIndex(item_factory=Document, index_path=external_cache_path(), mode='w') as index:+ page_index = index.get_key_page_index(term)+ with index.page(page_index) as page:+ # Newest first. store() drops the tail that does not fit, so ordering by+ # age is what makes eviction fall on the oldest entries on the page.+ # Emphatically not the ranker's ordering, which would evict by relevance.+ # Dropping stale entries here is also the whole of expiry: a page under+ # write pressure cleans itself, and a page nobody rewrites is not under+ # pressure, so there is no sweep to schedule. Entries from other sources+ # are ordinary neighbours here - they compete for the page like any other.+ # Replaced entries are this query's *for this provider only*. Matching on+ # the term alone would be wrong now that providers share it: storing one+ # provider's results would silently drop every other provider's results for+ # the same query.+ kept = sorted(+ (document for document in page.documents+ if not (document.term == term and document.source == source)+ and is_fresh(document, now)),+ key=lambda document: -(document.last_crawled or 0))+ stored = page.store(entries + kept)+ if stored < len(entries):+ logger.warning(+ "External cache page %d held only %d of %d entries for one query",+ page_index, stored, len(entries))+ except Exception:+ logger.exception("Could not store %s results in the external cache index", source.name)+++def _entries_to_store(source: DocumentSource, term: str, documents: list[Document],+ now: int) -> list[Document]:+ """The documents to write for this query, or the empty-result sentinel if there are none.++ Every entry carries its source, the sentinel included - an entry has to say which+ provider it came from without the key, and is_fresh has nothing else to go on. `state`+ is carried through from whatever the provider produced rather than stamped here.+ """+ usable = [document for document in documents if document.url and document.title]+ if not usable:+ return [Document(title="", url=EXTERNAL_CACHE_EMPTY_URL, extract="", score=0.0, term=term,+ last_crawled=now, source=source)]++ return [Document(title=document.title, url=document.url, extract=document.extract,+ score=document.score, term=term, state=document.state, last_crawled=now,+ source=source)+ for document in usable]diff --git a/mwmbl/indexer/wiki_cache.py b/mwmbl/indexer/wiki_cache.pydeleted file mode 100644index c61f856..0000000--- a/mwmbl/indexer/wiki_cache.py+++ /dev/null@@ -1,181 +0,0 @@-"""Wikipedia search results, cached in a TinyIndex of their own.--get_wiki_results() used to cache through mwmbl.utils.request_cache, a requests-cache-*filesystem* session rooted at REQUEST_CACHE_PATH - the same volume as the 400 GB index --with no LRU, no size cap and nothing ever calling delete(expired=True), so it grew without-bound. Because the stored JSON holds the whole request, it also wrote the raw user query-to disk, where it sat for the length of the expiry.--This is a *separate* index file, not the search index. Caching into the search index was-tried in #357 and measured in #359: a cache entry and a real document compete for the same-4 KB page, and on a dense index storing one evicts about as much as it adds (-0.0054 NDCG-overall, -0.1927 on the queries where it fired). A file of its own also means a cache entry-can never surface in /raw or in the candidate pool for somebody else's query, which is what-lets this module keep a negative-cache sentinel - see WIKI_CACHE_EMPTY_URL.--Entries are keyed by a hash of the query, so the file holds no query text. Be straight-about what that does and does not do: it removes the query *text*, not the fact that-somebody searched in that area, since an entry still holds the articles that came back.-That is strictly less than the disk cache gave away.-"""-import hashlib-import hmac-import time-from logging import getLogger-from pathlib import Path-from typing import Optional--from django.conf import settings--from mwmbl.tinysearchengine.indexer import Document, DocumentState, TinyIndex-from mwmbl.tokenizer import tokenize--logger = getLogger(__name__)---# Terms under this prefix are cache entries rather than words anybody searched for.-WIKI_CACHE_TERM_PREFIX = "#wiki-"--# A query Wikipedia returned nothing for is stored as a single document under this URL, so-# that "we asked and there was nothing" stays distinguishable from "we never asked".-# Without it those queries are re-fetched forever. It is filtered out on read and never-# leaves this module.-WIKI_CACHE_EMPTY_URL = "mwmbl:wiki-cache-empty"---def wiki_cache_path() -> Path:- return Path(settings.DATA_PATH) / settings.WIKI_CACHE_INDEX_NAME---_cache_index: Optional[TinyIndex] = None-_cache_index_path: Optional[Path] = None---def get_cache_index() -> TinyIndex:- """The read handle for this process, opened once and left open.-- Same lifetime as the main index in search_setup - mmap'd read-only for the life of the- worker. Writers open their own short-lived 'w' handle instead, as every other writer in- the codebase does; see store_wiki_results.-- Keyed on the path so that override_settings in the tests gets a fresh handle rather- than whichever index the first test to run happened to open.- """- global _cache_index, _cache_index_path- path = wiki_cache_path()- if _cache_index is None or _cache_index_path != path:- if _cache_index is not None:- _cache_index.__exit__(None, None, None)- _cache_index = TinyIndex(item_factory=Document, index_path=path)- _cache_index.__enter__()- _cache_index_path = path- return _cache_index---def wiki_cache_term(query: str) -> str:- """The term a query's Wikipedia results are cached under.-- Keyed on SECRET_KEY rather than a bare digest: a plain hash of a one- or two-word query- is trivially brute-forced from a wordlist, so a dump of the file would hand over the- queries. Keyed, the mapping is not reproducible without the key. Rotating SECRET_KEY- invalidates the cache, which is harmless - it is a cache.-- The query is normalised through tokenize() first, so "Python" and " python " share an- entry. The disk cache keyed on the request URL built from the raw string, so they did- not.- """- normalised = " ".join(tokenize(query))- digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8"), hashlib.sha256)- return WIKI_CACHE_TERM_PREFIX + digest.hexdigest()[:16]---def is_fresh(document: Document, now: int) -> bool:- """Whether a cached document is still within its TTL. Untimestamped means stale."""- ttl = (settings.WIKI_CACHE_NEGATIVE_TTL_SECONDS if document.url == WIKI_CACHE_EMPTY_URL- else settings.WIKI_CACHE_TTL_SECONDS)- return document.last_crawled is not None and now - document.last_crawled < ttl---def get_cached_wiki_results(query: str, now: Optional[int] = None) -> Optional[list[Document]]:- """A query's cached Wikipedia results.-- None means we have nothing for this query and should ask Wikipedia. A list - possibly- empty - means we already asked; an empty one is a query Wikipedia had nothing for.- """- if not settings.WIKI_CACHE_ENABLED:- return None-- now = int(time.time()) if now is None else now- term = wiki_cache_term(query)- try:- stored = get_cache_index().retrieve(term)- except FileNotFoundError:- logger.warning("No Wikipedia cache index at %s", wiki_cache_path())- return None- except Exception:- # A cache we cannot read is a cache miss, never a failed search. retrieve() already- # absorbs an unreadable page; this covers the file itself going away underneath us.- logger.exception("Could not read the Wikipedia cache index")- return None-- # retrieve() hands back the whole page, which is shared with every other query that- # hashes to it, and it also lets through documents with no term at all.- entries = [document for document in stored if document.term == term and is_fresh(document, now)]- if not entries:- return None-- # Handed back with term set to the query, exactly as a live fetch builds them. The- # hashed term is how the entry is stored, not something the ranker should ever see.- results = [Document(title=document.title, url=document.url, extract=document.extract,- score=document.score, term=query, state=document.state)- for document in entries if document.url != WIKI_CACHE_EMPTY_URL]- return sorted(results, key=lambda document: -(document.score or 0.0))---def store_wiki_results(query: str, documents: list[Document], now: Optional[int] = None) -> None:- """Store a query's Wikipedia results, replacing any existing entry for it.-- Never raises. A cache write that fails costs a re-fetch later; it must not cost a- search its results.- """- if not settings.WIKI_CACHE_ENABLED:- return-- now = int(time.time()) if now is None else now- term = wiki_cache_term(query)- entries = _entries_to_store(term, documents, now)-- try:- with TinyIndex(item_factory=Document, index_path=wiki_cache_path(), mode='w') as index:- page_index = index.get_key_page_index(term)- with index.page(page_index) as page:- # Newest first. store() drops the tail that does not fit, so ordering by- # age is what makes eviction fall on the oldest entries on the page.- # Emphatically not the ranker's ordering, which would evict by relevance.- # Dropping stale entries here is also the whole of expiry: a page under- # write pressure cleans itself, and a page nobody rewrites is not under- # pressure, so there is no sweep to schedule.- kept = sorted(- (document for document in page.documents- if document.term != term and is_fresh(document, now)),- key=lambda document: -(document.last_crawled or 0))- stored = page.store(entries + kept)- if stored < len(entries):- logger.warning(- "Wiki cache page %d held only %d of %d entries for one query",- page_index, stored, len(entries))- except Exception:- logger.exception("Could not store Wikipedia results in the cache index")---def _entries_to_store(term: str, documents: list[Document], now: int) -> list[Document]:- """The documents to write for this query, or the empty-result sentinel if there are none."""- usable = [document for document in documents if document.url and document.title]- if not usable:- return [Document(title="", url=WIKI_CACHE_EMPTY_URL, extract="", score=0.0, term=term,- state=DocumentState.FROM_WIKI, last_crawled=now)]-- return [Document(title=document.title, url=document.url, extract=document.extract,- score=document.score, term=term, state=DocumentState.FROM_WIKI,- last_crawled=now)- for document in usable]diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex afc5e15..41fa02c 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -337,12 +337,17 @@ BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index pe# than one per approval - see mwmbl.signals.BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600-# Wikipedia results are cached in a TinyIndex of their own (see mwmbl.indexer.wiki_cache).-# A dedicated file rather than the search index because a cache entry competing with real-# documents for a shared page evicts them - #359 measured that as a net NDCG loss - and-# because a separate file never enters the candidate pool for anyone else's query.-WIKI_CACHE_ENABLED = os.environ.get("WIKI_CACHE_ENABLED", "true").lower() != "false"-WIKI_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks-# A query Wikipedia has nothing for is worth remembering too, or it is re-fetched forever.-# Shorter, because an article appearing is a likelier change than an existing one moving.-WIKI_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60+# Results from external search providers - Wikipedia today, others later - are cached in a+# TinyIndex of their own (see mwmbl.indexer.external_cache). Entries are namespaced by+# source, so providers share the file without sharing entries. A dedicated file rather than+# the search index because a cache entry competing with real documents for a shared page+# evicts them - #359 measured that as a net NDCG loss - and because a separate file never+# enters the candidate pool for anyone else's query.+EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower() != "false"+# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six+# months; a provider whose results turn over faster wants its own, and the place to add that+# is here, keyed by source.+EXTERNAL_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks+# A query a provider has nothing for is worth remembering too, or it is re-fetched forever.+# Shorter, because a result appearing is a likelier change than an existing one moving.+EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex 9b44028..a598abd 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -22,8 +22,8 @@ INDEX_NAME = 'index-v2.tinysearch'NUM_PAGES = 2560-WIKI_CACHE_INDEX_NAME = 'wiki-cache.tinysearch'-WIKI_CACHE_NUM_PAGES = 2560+EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'+EXTERNAL_CACHE_NUM_PAGES = 2560URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex caf1261..38d2e04 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -31,10 +31,30 @@ INDEX_NAME = 'index-v2-400G.tinysearch'# 400GB indexNUM_PAGES = 102400000-WIKI_CACHE_INDEX_NAME = 'wiki-cache.tinysearch'-# 4GB. ~12 entries fit a 4KB page, but pages are hash-assigned, so keep the load factor-# well under that: this holds a few million queries before collisions start evicting early.-WIKI_CACHE_NUM_PAGES = 1_000_000+EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'+# ~15 GB. Sized by page occupancy rather than by naive capacity, because pages are assigned+# by hash: the queries per page are Poisson-distributed, not evenly spread, so the pages+# that run over start truncating while empty ones sit unused.+#+# A 4 KB page holds ~13 cached queries - 37 documents at 3 results a query - measured by+# packing the 11,693 real Wikipedia responses left in the old devdata/request_cache until+# the page overflowed. That is in line with the live index, which averages 35.4 documents a+# page, and it is worth stating how it was measured because a first pass using synthetic+# random-vocabulary text got 8, 60% low: zstd exploits the redundancy in real prose and+# there is a lot of it. Real extracts are ~154 characters.+#+# At an average load of L queries per page the fraction still resident is E[min(K, 13)] / L+# for K ~ Poisson(L), which gives, for one provider:+#+# L = 8.2 -> 99% retained -> 30.9M distinct queries+# L = 10.9 -> 95% retained -> 40.8M+# L = 12.7 -> 90% retained -> 47.6M+# L = 13 -> 89% retained -> 48.8M (the naive number)+#+# A query cached from several providers holds one entry per provider, so divide by the+# number of providers: with two, ~15M queries at 99% and ~24M at 90%. Longer extracts fit+# fewer per page, so a provider more verbose than Wikipedia costs more than its share.+EXTERNAL_CACHE_NUM_PAGES = 3_750_000URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.pyindex 9716918..8cd5738 100644--- a/mwmbl/settings_test.py+++ b/mwmbl/settings_test.py@@ -11,8 +11,8 @@ HAS_DATABASE = FalseDATA_PATH = "/tmp/test_mwmbl_data"INDEX_NAME = "index-v2.tinysearch"NUM_PAGES = 10-WIKI_CACHE_INDEX_NAME = "wiki-cache.tinysearch"-WIKI_CACHE_NUM_PAGES = 10+EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"+EXTERNAL_CACHE_NUM_PAGES = 10# Use fakeredis for cache so tests don't need a real Redis serverCACHES = {diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.pyindex 1605885..cdebeee 100644--- a/mwmbl/tinysearchengine/indexer.py+++ b/mwmbl/tinysearchengine/indexer.py@@ -74,6 +74,22 @@ class DocumentState(IntEnum):CURATED_STATES = {state for state in DocumentState if state.value >= 7}+class DocumentSource(IntEnum):+ """Which external provider a document came from. None means our own crawl.++ Only the external results cache sets this (see mwmbl.indexer.external_cache); documents+ in the search index leave it None, and as_tuple truncates trailing Nones, so they cost+ nothing for it. The values are written to disk, so they are append-only: never reuse a+ number for a different provider, or a stale cache entry becomes a lie about where its+ results came from.+ """+ WIKIPEDIA = 1+ # Reserved, not yet wired up to a provider. The number is here rather than added later+ # because it is the numbering registry: with only one member the source field cannot be+ # told apart from "unset" and the cache's per-source behaviour cannot be tested at all.+ STAAN = 2++@dataclassclass Document:title: str@@ -84,6 +100,7 @@ class Document:state: Optional[int] = Noneuser_ids: Optional[List[int]] = Nonelast_crawled: Optional[int] = None+ source: Optional[int] = Nonedef __init__(self,@@ -95,6 +112,7 @@ class Document:state: Optional[int | DocumentState] = None,user_ids: Optional[List[int]] = None,last_crawled: Optional[int] = None,+ source: Optional[int | DocumentSource] = None,):# Sometimes the title or extract may be None, probably because of user generated content# It's not allowed to be None though, or things will break@@ -106,6 +124,7 @@ class Document:self.state = None if state is None else DocumentState(state)self.user_ids = user_idsself.last_crawled = last_crawled+ self.source = None if source is None else DocumentSource(source)def as_tuple(self):"""@@ -120,6 +139,7 @@ class Document:None if self.state is None else self.state.value,self.user_ids,self.last_crawled,+ None if self.source is None else self.source.value,]while values[-1] is None:values = values[:-1]diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 09ec4b4..d36955b 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -22,9 +22,9 @@ from mwmbl.format import get_query_regexfrom mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklistfrom mwmbl.indexer.purge_queue import enqueue_for_purge-from mwmbl.indexer.wiki_cache import get_cached_wiki_results, store_wiki_results+from mwmbl.indexer.external_cache import get_cached_external_results, store_external_resultsfrom mwmbl.tinysearchengine.completer import Completer-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentSource, DocumentStatefrom mwmbl.tokenizer import tokenize, get_bigramsfrom mwmbl.utils import get_domain@@ -433,7 +433,7 @@ WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"# calls — e.g. an evaluation run that queries it once per gold query. Retry with# backoff, honouring any Retry-After, so a transient 429 doesn't silently drop a# query's wiki results. Live search never bursts, so this adds no latency in-# normal operation. Successful responses are cached for months (see mwmbl.indexer.wiki_cache).+# normal operation. Successful responses are cached for months (see mwmbl.indexer.external_cache).WIKI_RETRY = Retry(total=4, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504),allowed_methods=frozenset({"GET"}), respect_retry_after_header=True)@@ -488,7 +488,7 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:# Ahead of the circuit breaker deliberately: a cache hit costs Wikipedia nothing, so it# should still be served while we are backing off from a rate limit.- cached = get_cached_wiki_results(query)+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, query)if cached is not None:return cached[:max_wiki_results]@@ -527,13 +527,14 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:return []wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- max_wiki_results + 1 - i, query, state=DocumentState.FROM_WIKI)+ max_wiki_results + 1 - i, query, state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA)for i, result in enumerate(wiki_response['query']['search'][:max_wiki_results])]# Only a well-formed answer is stored, including one with no results in it. Every path# above returns [] without storing, so a transient failure is never remembered as# "Wikipedia has nothing for this".- store_wiki_results(query, wiki_results)+ store_external_results(DocumentSource.WIKIPEDIA, query, wiki_results)return wiki_resultsdiff --git a/test/test_wiki_cache.py b/test/test_external_cache.pysimilarity index 52%rename from test/test_wiki_cache.pyrename to test/test_external_cache.pyindex 988cb24..4bb68dd 100644--- a/test/test_wiki_cache.py+++ b/test/test_external_cache.py@@ -1,4 +1,4 @@-"""Tests for the dedicated Wikipedia cache index (mwmbl.indexer.wiki_cache).+"""Tests for the dedicated external results cache index (mwmbl.indexer.external_cache).Where these overlap with the tests on the #357 branch they are deliberately the sameassertions - the term derivation and the freshness rule are carried over unchanged. What is@@ -6,6 +6,7 @@ gone is everything that existed only because cache entries shared a file with recontent: the anonymisation gate, the Redis queue, the general-indexing path, and the guardsstopping a cache term reaching curation or /raw."""+import jsonimport randomimport stringimport time@@ -15,35 +16,40 @@ import pytestfrom django.test import override_settingsfrom mwmbl.apps import create_index-from mwmbl.indexer.wiki_cache import (- WIKI_CACHE_EMPTY_URL,- WIKI_CACHE_TERM_PREFIX,- get_cached_wiki_results,+from mwmbl.indexer.external_cache import (+ EXTERNAL_CACHE_EMPTY_URL,+ EXTERNAL_CACHE_TERM_PREFIX,+ external_cache_path,+ external_cache_term,+ get_cached_external_results,is_fresh,- store_wiki_results,- wiki_cache_path,- wiki_cache_term,+ store_external_results,)from mwmbl.tinysearchengine import rank-from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentState, TinyIndex+from mwmbl.tinysearchengine.indexer import (PAGE_SIZE, Document, DocumentSource, DocumentState,+ TinyIndex)from mwmbl.tinysearchengine.rank import Ranker, get_wiki_resultsNUM_PAGES = 8+# state and source are set because get_wiki_results sets them, and the cache preserves what+# the provider produced rather than stamping its own.PYTHON = Document("Python (programming language)", "https://en.wikipedia.org/wiki/Python",- "A high-level programming language.", 3.0)+ "A high-level programming language.", 3.0, state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA)MONTY = Document("Monty Python", "https://en.wikipedia.org/wiki/Monty_Python",- "A British comedy troupe.", 2.0)+ "A British comedy troupe.", 2.0, state=DocumentState.FROM_WIKI,+ source=DocumentSource.WIKIPEDIA)@pytest.fixturedef cache_index(tmp_path):- """A real, empty wiki cache index that the module-level helpers will pick up."""- with override_settings(DATA_PATH=str(tmp_path), WIKI_CACHE_INDEX_NAME="wiki-cache.tinysearch",- WIKI_CACHE_NUM_PAGES=NUM_PAGES, WIKI_CACHE_ENABLED=True):- TinyIndex.create(item_factory=Document, index_path=str(wiki_cache_path()),+ """A real, empty external results cache index that the module-level helpers pick up."""+ with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",+ EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES, EXTERNAL_CACHE_ENABLED=True):+ TinyIndex.create(item_factory=Document, index_path=str(external_cache_path()),num_pages=NUM_PAGES, page_size=PAGE_SIZE)- yield str(wiki_cache_path())+ yield str(external_cache_path())def _wiki_api_response(*titles):@@ -71,14 +77,14 @@ def _patched_wikipedia(response):def test_cache_term_is_stable_and_normalised():"""Whitespace and case must not split one query across several entries. The disk cachekeyed on the raw request URL, so " python " and "Python" missed each other."""- assert wiki_cache_term("python") == wiki_cache_term(" PYTHON ")- assert wiki_cache_term("monty python") == wiki_cache_term("Monty Python")- assert wiki_cache_term("python") != wiki_cache_term("monty python")+ assert external_cache_term("python") == external_cache_term(" PYTHON ")+ assert external_cache_term("monty python") == external_cache_term("Monty Python")+ assert external_cache_term("python") != external_cache_term("monty python")def test_cache_term_does_not_contain_the_query():- term = wiki_cache_term("something private")- assert term.startswith(WIKI_CACHE_TERM_PREFIX)+ term = external_cache_term("something private")+ assert term.startswith(EXTERNAL_CACHE_TERM_PREFIX)assert "something" not in termassert "private" not in term@@ -87,9 +93,9 @@ def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():"""A bare digest of a one-word query is brute-forceable from a wordlist; keyed onSECRET_KEY the term->query mapping is not reproducible without the key."""with override_settings(SECRET_KEY="one"):- first = wiki_cache_term("bananas")+ first = external_cache_term("bananas")with override_settings(SECRET_KEY="two"):- second = wiki_cache_term("bananas")+ second = external_cache_term("bananas")assert first != second@@ -99,9 +105,9 @@ def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():# ---------------------------------------------------------------------------def test_results_come_back_from_the_index(cache_index):- store_wiki_results("python", [PYTHON, MONTY])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])- cached = get_cached_wiki_results("python")+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")assert [document.url for document in cached] == [PYTHON.url, MONTY.url] # descending scoreassert [document.score for document in cached] == [3.0, 2.0]@@ -111,32 +117,123 @@ def test_results_come_back_from_the_index(cache_index):def test_a_hit_is_indistinguishable_from_a_live_fetch(cache_index):"""The hashed term is a storage detail. Handing it to the ranker would make a cachedresult differ from the identical one fetched live."""- store_wiki_results("python", [PYTHON])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])- cached = get_cached_wiki_results("python")+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")assert [document.term for document in cached] == ["python"]def test_a_normalised_variant_of_the_query_hits_the_same_entry(cache_index):- store_wiki_results("Monty Python", [MONTY])+ store_external_results(DocumentSource.WIKIPEDIA, "Monty Python", [MONTY])- assert get_cached_wiki_results(" monty python ") is not None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, " monty python ") is not Nonedef test_a_query_we_have_never_asked_about_is_a_miss(cache_index):- store_wiki_results("python", [PYTHON])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])- assert get_cached_wiki_results("something else entirely") is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "something else entirely") is None+++def _all_stored_documents(index_path):+ """Every document in the file, decompressed. Grepping the raw bytes proves little: the+ pages are zstd-compressed, so plaintext would not show up there either way."""+ with TinyIndex(Document, index_path, 'r') as index:+ return [document for page in range(index.num_pages) for document in index.get_page(page)]def test_the_file_holds_no_query_text(cache_index):- store_wiki_results("something private", [PYTHON])+ store_external_results(DocumentSource.WIKIPEDIA, "something private", [PYTHON])- with open(cache_index, "rb") as index_file:- raw = index_file.read()+ stored = _all_stored_documents(cache_index)- assert b"something private" not in raw+ assert stored, "nothing was written, so this would pass for the wrong reason"+ written = json.dumps([document.as_tuple() for document in stored])+ assert "something private" not in written+ assert "private" not in written+++# ---------------------------------------------------------------------------+# Several providers sharing the file+# ---------------------------------------------------------------------------++# The second provider is the whole reason the cache is keyed by source rather than by query+# alone, so the tests below exercise it even though nothing fetches from it yet.+OTHER_SOURCE = DocumentSource.STAAN++STAAN_RESULT = Document("Python tutorial", "https://example.com/python",+ "Somewhere that is not Wikipedia.", 9.0, source=DocumentSource.STAAN)+++def test_two_providers_do_not_share_an_entry_for_the_same_query(cache_index):+ """Providers share the term, and therefore the page, but not the entry: one provider+ answering a query must not stop the other being asked, or answer on its behalf."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])++ assert get_cached_external_results(OTHER_SOURCE, "python") is None++ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])++ assert [d.url for d in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [PYTHON.url]+ assert [d.url for d in get_cached_external_results(OTHER_SOURCE, "python")] == [STAAN_RESULT.url]+++def test_an_entry_says_which_provider_it_came_from_but_not_which_query(cache_index):+ """An entry has to be attributable to its provider - to count them, to drop one+ provider's entries, or to give one its own TTL - which the query must never be. The+ source is on the document; the query is not anywhere."""+ store_external_results(DocumentSource.WIKIPEDIA, "something private", [PYTHON])++ stored = _all_stored_documents(cache_index)++ assert [document for document in stored if document.source == DocumentSource.WIKIPEDIA]+ assert not any("private" in document.term for document in stored)+++def test_storing_one_provider_does_not_evict_another_for_the_same_query(cache_index):+ """The trap in sharing a term. The write path replaces "this query's entries", and if+ that means the term alone it wipes every other provider's results for the query - with+ no error, just a silent extra fetch every time."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])+ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [MONTY])++ assert [d.url for d in get_cached_external_results(OTHER_SOURCE, "python")] == [STAAN_RESULT.url]+ assert [d.url for d in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [MONTY.url]+++def test_both_providers_results_for_a_query_live_on_one_page(cache_index):+ """The reason the term does not name the source: one page read serves every provider,+ so asking the second costs no extra page fault."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])+ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])++ with TinyIndex(Document, cache_index, 'r') as index:+ page = index.get_page(index.get_key_page_index(external_cache_term("python")))++ assert {document.source for document in page} == {DocumentSource.WIKIPEDIA, DocumentSource.STAAN}+++def test_the_negative_sentinel_carries_its_source_too(cache_index):+ """is_fresh reads document.source off a page's other entries, where the term tells it+ nothing, so an entry with no results still has to say where it came from."""+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])++ stored = _all_stored_documents(cache_index)++ assert [document.source for document in stored] == [DocumentSource.WIKIPEDIA]+++def test_a_providers_state_and_source_survive_the_round_trip(cache_index):+ """The cache preserves what the provider produced rather than stamping its own, so a+ cached result stays attributable once there is a second provider."""+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])++ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")++ assert [document.state for document in cached] == [DocumentState.FROM_WIKI]+ assert [document.source for document in cached] == [DocumentSource.WIKIPEDIA]# ---------------------------------------------------------------------------@@ -145,10 +242,10 @@ def test_the_file_holds_no_query_text(cache_index):def test_a_stale_entry_is_not_a_cache_hit(cache_index):now = int(time.time())- store_wiki_results("python", [PYTHON], now=now)+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)- fresh = get_cached_wiki_results("python", now=now + 60)- stale = get_cached_wiki_results("python", now=now + 27 * 7 * 24 * 60 * 60)+ fresh = get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=now + 60)+ stale = get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=now + 27 * 7 * 24 * 60 * 60)assert fresh is not Noneassert stale is None@@ -165,13 +262,13 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):sweep to schedule and nothing accumulates on a page nobody writes to."""now = int(time.time())old_query, new_query = _colliding_queries(cache_index)- store_wiki_results(old_query, [PYTHON], now=now)+ store_external_results(DocumentSource.WIKIPEDIA, old_query, [PYTHON], now=now)much_later = now + 27 * 7 * 24 * 60 * 60- store_wiki_results(new_query, [MONTY], now=much_later)+ store_external_results(DocumentSource.WIKIPEDIA, new_query, [MONTY], now=much_later)with TinyIndex(Document, cache_index, 'r') as index:- page = index.get_page(index.get_key_page_index(wiki_cache_term(new_query)))+ page = index.get_page(index.get_key_page_index(external_cache_term(new_query)))assert [document.url for document in page] == [MONTY.url]@@ -180,39 +277,48 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):# Negative caching# ----------------------------------------------------------------------------def test_a_query_wikipedia_has_nothing_for_is_remembered(cache_index):+def test_a_query_a_provider_has_nothing_for_is_remembered(cache_index):"""Without this the query is re-fetched forever - #357 listed it as unfixable, becausea sentinel in the search index would surface in /raw and in every candidate list."""- store_wiki_results("nonsense query", [])+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])- assert get_cached_wiki_results("nonsense query") == []+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query") == []def test_an_empty_entry_is_not_returned_as_a_result(cache_index):- store_wiki_results("nonsense query", [])+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])- cached = get_cached_wiki_results("nonsense query")+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query")assert [document.url for document in cached] == []- assert WIKI_CACHE_EMPTY_URL not in [document.url for document in cached]+ assert EXTERNAL_CACHE_EMPTY_URL not in [document.url for document in cached]+++def test_one_providers_empty_result_is_not_anothers(cache_index):+ """Sharing a page makes this worth pinning down: "Wikipedia has nothing" must not read+ as "Staan has nothing", or the second provider is never asked."""+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])++ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query") == []+ assert get_cached_external_results(OTHER_SOURCE, "nonsense query") is Nonedef test_an_empty_entry_expires_sooner_than_a_real_one(cache_index):"""An article appearing is a likelier change than an existing one moving."""now = int(time.time())- store_wiki_results("nonsense query", [], now=now)- store_wiki_results("python", [PYTHON], now=now)+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [], now=now)+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)later = now + 8 * 24 * 60 * 60 # past the negative TTL, well inside the positive one- assert get_cached_wiki_results("nonsense query", now=later) is None- assert get_cached_wiki_results("python", now=later) is not None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query", now=later) is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=later) is not Nonedef test_documents_without_a_url_or_title_are_not_stored(cache_index):- store_wiki_results("python", [Document("", "", "", 1.0)])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [Document("", "", "", 1.0)])- assert get_cached_wiki_results("python") == []+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") == []# ---------------------------------------------------------------------------@@ -228,11 +334,11 @@ def test_a_cache_miss_calls_wikipedia_and_stores_the_result(cache_index):assert session.get.call_count == 1assert [document.url for document in results] == ["https://en.wikipedia.org/wiki/Python_(programming_language)"]- assert get_cached_wiki_results("python") is not None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is not Nonedef test_a_cache_hit_does_not_call_wikipedia(cache_index):- store_wiki_results("python", [PYTHON])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])mock, session = _patched_wikipedia(_wiki_api_response("Python"))try:@@ -247,7 +353,7 @@ def test_a_cache_hit_does_not_call_wikipedia(cache_index):def test_a_hit_is_served_while_the_circuit_breaker_is_open(cache_index):"""A cached answer costs Wikipedia nothing, so backing off from a rate limit is noreason to withhold it."""- store_wiki_results("python", [PYTHON])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])rank._trip_wiki_circuit()try:results = get_wiki_results("python", 3)@@ -266,7 +372,7 @@ def test_a_failed_fetch_is_not_remembered_as_an_empty_result(cache_index):finally:mock.stop()- assert get_cached_wiki_results("python") is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is Nonedef test_an_error_response_is_not_remembered_as_an_empty_result(cache_index):@@ -276,7 +382,7 @@ def test_an_error_response_is_not_remembered_as_an_empty_result(cache_index):finally:mock.stop()- assert get_cached_wiki_results("python") is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is Nonedef test_a_broken_cache_write_does_not_break_the_search(cache_index):@@ -284,7 +390,7 @@ def test_a_broken_cache_write_does_not_break_the_search(cache_index):results."""mock, _ = _patched_wikipedia(_wiki_api_response("Python"))try:- with patch("mwmbl.indexer.wiki_cache.TinyIndex", side_effect=OSError("disk gone")):+ with patch("mwmbl.indexer.external_cache.TinyIndex", side_effect=OSError("disk gone")):results = get_wiki_results("python", 3)finally:mock.stop()@@ -311,17 +417,17 @@ def test_the_query_is_truncated_before_it_reaches_wikipedia(cache_index):# ---------------------------------------------------------------------------def test_the_kill_switch_stops_reads_and_writes(cache_index):- with override_settings(WIKI_CACHE_ENABLED=False):- store_wiki_results("python", [PYTHON])- assert get_cached_wiki_results("python") is None+ with override_settings(EXTERNAL_CACHE_ENABLED=False):+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None- assert get_cached_wiki_results("python") is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is Nonedef test_a_missing_cache_file_is_a_miss_rather_than_an_error(tmp_path):- with override_settings(DATA_PATH=str(tmp_path), WIKI_CACHE_INDEX_NAME="absent.tinysearch",- WIKI_CACHE_ENABLED=True):- assert get_cached_wiki_results("python") is None+ with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="absent.tinysearch",+ EXTERNAL_CACHE_ENABLED=True):+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None# ---------------------------------------------------------------------------@@ -332,10 +438,10 @@ def _colliding_queries(index_path):"""Two queries whose cache terms land on the same page."""with TinyIndex(Document, index_path, 'r') as index:first = "collide-0"- first_page = index.get_key_page_index(wiki_cache_term(first))+ first_page = index.get_key_page_index(external_cache_term(first))for i in range(1, 10000):candidate = f"collide-{i}"- if index.get_key_page_index(wiki_cache_term(candidate)) == first_page:+ if index.get_key_page_index(external_cache_term(candidate)) == first_page:return first, candidateraise AssertionError("No colliding query found")@@ -354,20 +460,20 @@ def test_the_newest_entry_survives_a_full_page_and_the_oldest_is_evicted(cache_ibulky = [Document(f"Title {i}", f"https://en.wikipedia.org/wiki/{i}", _filler(2000), 3.0)for i in range(4)]- store_wiki_results(old_query, bulky, now=now)- assert get_cached_wiki_results(old_query, now=now) is not None+ store_external_results(DocumentSource.WIKIPEDIA, old_query, bulky, now=now)+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, old_query, now=now) is not None- store_wiki_results(new_query, bulky, now=now + 1)+ store_external_results(DocumentSource.WIKIPEDIA, new_query, bulky, now=now + 1)- assert get_cached_wiki_results(new_query, now=now + 1) is not None- assert get_cached_wiki_results(old_query, now=now + 1) is None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, new_query, now=now + 1) is not None+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, old_query, now=now + 1) is Nonedef test_an_entry_replaces_its_own_previous_results(cache_index):- store_wiki_results("python", [PYTHON, MONTY])- store_wiki_results("python", [MONTY])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])+ store_external_results(DocumentSource.WIKIPEDIA, "python", [MONTY])- assert [document.url for document in get_cached_wiki_results("python")] == [MONTY.url]+ assert [document.url for document in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [MONTY.url]# ---------------------------------------------------------------------------@@ -416,11 +522,11 @@ def test_search_calls_external_search_by_default(cache_index):def test_a_resized_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_path):"""Resizing a cache should be a config change, not a startup crash - the cost ofrebuilding is re-fetching."""- path = tmp_path / "wiki-cache.tinysearch"+ path = tmp_path / "external-cache.tinysearch"TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=4, page_size=PAGE_SIZE)with override_settings(DATA_PATH=str(tmp_path)):- create_index("wiki-cache.tinysearch", 16, rebuild_on_mismatch=True)+ create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)with TinyIndex(Document, str(path), 'r') as index:assert index.num_pages == 16diff --git a/test/test_rank.py b/test/test_rank.pyindex 05055e6..2e4b73c 100644--- a/test/test_rank.py+++ b/test/test_rank.py@@ -73,7 +73,7 @@ def _reset_wiki_circuit():def _get_wiki_results_with_session(session):# The cache is off in these tests: they are about what the fetch does when Wikipedia# misbehaves, and a hit would return before the fetch ever ran.- with override_settings(WIKI_CACHE_ENABLED=False), \+ with override_settings(EXTERNAL_CACHE_ENABLED=False), \patch.object(rank.requests, "Session") as mock_session:mock_session.return_value.__enter__.return_value = sessionreturn get_wiki_results("query", 5)@@ -113,7 +113,7 @@ def test_wiki_query_text_containing_429_is_not_mistaken_for_rate_limit():def test_open_wiki_circuit_short_circuits_without_calling_wikipedia():rank._trip_wiki_circuit()- with override_settings(WIKI_CACHE_ENABLED=False), \+ with override_settings(EXTERNAL_CACHE_ENABLED=False), \patch.object(rank.requests, "Session") as mock_session:results = get_wiki_results("query", 5)