Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Generalise the Wikipedia cache to any external provider, and size it for real pages

Daoud Clarke8850373d05eb
diff --git a/.gitignore b/.gitignore
index 4ea090d..d8fcb86 100644
--- a/.gitignore
+++ b/.gitignore
@@ -166,8 +166,8 @@ devdata/rankeval-2024-06/remote-datasets/
devdata/*.bloom
devdata/*.sqlite
devdata/request_cache/
-# Generated by apps.ready(); holds cached Wikipedia responses, not source.
-devdata/wiki-cache.tinysearch
+# Generated by apps.ready(); holds cached external provider responses, not source.
+devdata/external-cache.tinysearch
devdata/rankeval-*/
front-end/node_modules/
diff --git a/app.json b/app.json
index 5142598..9af476d 100644
--- a/app.json
+++ b/app.json
@@ -11,7 +11,7 @@
"path": "/api/v1/search/?s=rydnik",
"attempts": 1,
"interval": 600,
- "grace": 60,
+ "grace": 300,
"timeout": 10
}
]
diff --git a/mwmbl/apps.py b/mwmbl/apps.py
index 87c51f8..624a847 100644
--- a/mwmbl/apps.py
+++ b/mwmbl/apps.py
@@ -14,8 +14,9 @@ def create_index(index_name, num_pages, rebuild_on_mismatch=False):
rebuild_on_mismatch is for indexes whose contents are disposable. The search index is
not one of them - a size that disagrees with settings there means somebody changed
NUM_PAGES against a 400 GB file holding the only copy of the crawl, and refusing to
- start is the right answer. The Wikipedia cache is the opposite case: resizing it should
- be a config change, not a startup crash, and the cost of rebuilding is re-fetching.
+ start is the right answer. The external results cache is the opposite case: resizing it
+ should be a config change, not a startup crash, and the cost of rebuilding is
+ re-fetching.
"""
# Imports here to avoid AppRegistryNotReady exception
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, PAGE_SIZE
@@ -55,8 +56,10 @@ class MwmblConfig(AppConfig):
def ready(self):
create_index(settings.INDEX_NAME, settings.NUM_PAGES)
# Note this writes num_pages * 4 KB page by page rather than sparsely, so the first
- # boot after a size change pays for the whole file up front.
- create_index(settings.WIKI_CACHE_INDEX_NAME, settings.WIKI_CACHE_NUM_PAGES,
+ # boot after a size change pays for the whole file up front - ~17s for the 15 GB
+ # production cache at the ~0.9 GB/s measured locally, which is what the healthcheck
+ # grace in app.json has to cover.
+ create_index(settings.EXTERNAL_CACHE_INDEX_NAME, settings.EXTERNAL_CACHE_NUM_PAGES,
rebuild_on_mismatch=True)
if settings.HAS_DATABASE:
create_index_db()
diff --git a/mwmbl/indexer/blacklist_providers.py b/mwmbl/indexer/blacklist_providers.py
index 0ce2c49..f1ccdf1 100644
--- a/mwmbl/indexer/blacklist_providers.py
+++ b/mwmbl/indexer/blacklist_providers.py
@@ -152,7 +152,7 @@ class RemoteListBlacklistProvider(BlacklistProvider):
# *every* user of request_cache breaks - and it shares that volume with the index
# itself. (get_wiki_results() used to be the worst of those, retrying a live fetch
# 4 times per uncached query on the search path; it now has a cache of its own, see
- # mwmbl.indexer.wiki_cache.) A big periodic download has no business in a shared
+ # mwmbl.indexer.external_cache.) A big periodic download has no business in a shared
# small-response cache on the index volume.
response = requests.get(self.url, timeout=60)
response.raise_for_status()
diff --git a/mwmbl/indexer/external_cache.py b/mwmbl/indexer/external_cache.py
new file mode 100644
index 0000000..1e88086
--- /dev/null
+++ b/mwmbl/indexer/external_cache.py
@@ -0,0 +1,229 @@
+"""Results from external search providers, cached in a TinyIndex of their own.
+
+Wikipedia is the only provider using this today, but nothing here is Wikipedia-specific:
+every entry is namespaced by its source, so a second provider (Staan, say) caches alongside
+it in the same file without the two ever seeing each other's entries.
+
+get_wiki_results() used to cache through mwmbl.utils.request_cache, a requests-cache
+*filesystem* session rooted at REQUEST_CACHE_PATH - the same volume as the 400 GB index -
+with no LRU, no size cap and nothing ever calling delete(expired=True), so it grew without
+bound. Because the stored JSON holds the whole request, it also wrote the raw user query
+to disk, where it sat for the length of the expiry.
+
+This is a *separate* index file, not the search index. Caching into the search index was
+tried in #357 and measured in #359: a cache entry and a real document compete for the same
+4 KB page, and on a dense index storing one evicts about as much as it adds (-0.0054 NDCG
+overall, -0.1927 on the queries where it fired). A file of its own also means a cache entry
+can never surface in /raw or in the candidate pool for somebody else's query, which is what
+lets this module keep a negative-cache sentinel - see EXTERNAL_CACHE_EMPTY_URL.
+
+Entries are keyed by a keyed hash of the query alone and carry their source as an int in
+the document itself - a DocumentSource, exactly as the search index carries DocumentState.
+Every provider's results for one query therefore land on one page: the second provider's
+lookup reads a page the first one just warmed, rather than faulting in a second 4 KB page
+from a 15 GB file. So the file says which provider each entry came from, and nothing about
+which query. Be
+straight about what that does and does not do: it removes the query *text*, not the fact
+that somebody searched in that area, since an entry still holds the results that came back.
+That is strictly less than the disk cache gave away.
+"""
+import hashlib
+import hmac
+import time
+from logging import getLogger
+from pathlib import Path
+from typing import Optional
+
+from django.conf import settings
+
+from mwmbl.tinysearchengine.indexer import Document, DocumentSource, TinyIndex
+from mwmbl.tokenizer import tokenize
+
+logger = getLogger(__name__)
+
+
+# Terms under this prefix are cache entries rather than words anybody searched for.
+EXTERNAL_CACHE_TERM_PREFIX = "#ext-"
+
+# A query a provider returned nothing for is stored as a single document under this URL, so
+# that "we asked and there was nothing" stays distinguishable from "we never asked".
+# Without it those queries are re-fetched forever. It is filtered out on read and never
+# leaves this module.
+EXTERNAL_CACHE_EMPTY_URL = "mwmbl:external-cache-empty"
+
+
+def external_cache_path() -> Path:
+ return Path(settings.DATA_PATH) / settings.EXTERNAL_CACHE_INDEX_NAME
+
+
+_cache_index: Optional[TinyIndex] = None
+_cache_index_path: Optional[Path] = None
+
+
+def get_cache_index() -> TinyIndex:
+ """The read handle for this process, opened once and left open.
+
+ Same lifetime as the main index in search_setup - mmap'd read-only for the life of the
+ worker. Writers open their own short-lived 'w' handle instead, as every other writer in
+ the codebase does; see store_external_results.
+
+ Keyed on the path so that override_settings in the tests gets a fresh handle rather
+ than whichever index the first test to run happened to open.
+ """
+ global _cache_index, _cache_index_path
+ path = external_cache_path()
+ if _cache_index is None or _cache_index_path != path:
+ if _cache_index is not None:
+ _cache_index.__exit__(None, None, None)
+ _cache_index = TinyIndex(item_factory=Document, index_path=path)
+ _cache_index.__enter__()
+ _cache_index_path = path
+ return _cache_index
+
+
+def external_cache_term(query: str) -> str:
+ """The term a query's cached results are stored under, for every provider alike.
+
+ Deliberately not keyed on the source. All of a query's providers then share one page, so
+ asking a second provider costs no extra page fault - it reads the page the first lookup
+ just brought in. Which provider an entry belongs to is on the entry, as a DocumentSource
+ int, and that is what the readers filter on. It is also what is_fresh would key a
+ per-source TTL off, since a page's other entries tell it nothing through the term.
+
+ Keyed on SECRET_KEY rather than a bare digest: a plain hash of a one- or two-word query
+ is trivially brute-forced from a wordlist, so a dump of the file would hand over the
+ queries. Keyed, the mapping is not reproducible without the key. Rotating SECRET_KEY
+ invalidates the cache, which is harmless - it is a cache.
+
+ The query is normalised through tokenize() first, so "Python" and " python " share an
+ entry. The disk cache keyed on the request URL built from the raw string, so they did
+ not.
+
+ 64 bits of digest is enough. Two distinct queries colliding here would serve one
+ query's results for the other, so it is worth being explicit: at 30M live queries the
+ birthday probability of *any* such pair existing is 2e-5. Widening the term is not free
+ either - it is stored on every document, and 32 hex characters measured ~9% fewer
+ entries per page - so the short term is the better trade. The collisions that do matter
+ are page collisions, which are a different thing entirely; see EXTERNAL_CACHE_NUM_PAGES
+ in settings_prod.
+ """
+ normalised = " ".join(tokenize(query))
+ digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8"), hashlib.sha256)
+ return f"{EXTERNAL_CACHE_TERM_PREFIX}{digest.hexdigest()[:16]}"
+
+
+def is_fresh(document: Document, now: int) -> bool:
+ """Whether a cached document is still within its TTL. Untimestamped means stale.
+
+ This runs over a page's other entries too, which belong to other queries and other
+ providers, so it cannot recover the source from the term - document.source is the only
+ thing that identifies them, and it is where a per-source TTL would key off.
+ """
+ ttl = (settings.EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS if document.url == EXTERNAL_CACHE_EMPTY_URL
+ else settings.EXTERNAL_CACHE_TTL_SECONDS)
+ return document.last_crawled is not None and now - document.last_crawled < ttl
+
+
+def get_cached_external_results(source: DocumentSource, query: str,
+ now: Optional[int] = None) -> Optional[list[Document]]:
+ """A query's cached results from one provider.
+
+ None means we have nothing for this (source, query) and should ask the provider. A list
+ - possibly empty - means we already asked; an empty one is a query the provider had
+ nothing for.
+ """
+ if not settings.EXTERNAL_CACHE_ENABLED:
+ return None
+
+ now = int(time.time()) if now is None else now
+ term = external_cache_term(query)
+ try:
+ stored = get_cache_index().retrieve(term)
+ except FileNotFoundError:
+ logger.warning("No external results cache index at %s", external_cache_path())
+ return None
+ except Exception:
+ # A cache we cannot read is a cache miss, never a failed search. retrieve() already
+ # absorbs an unreadable page; this covers the file itself going away underneath us.
+ logger.exception("Could not read the external results cache index")
+ return None
+
+ # retrieve() hands back the whole page, which is shared with every other query that
+ # hashes to it, and it also lets through documents with no term at all. The term narrows
+ # that to this query - across all providers, since the term does not name one - and the
+ # source picks out the provider being asked about.
+ entries = [document for document in stored
+ if document.term == term and document.source == source and is_fresh(document, now)]
+ if not entries:
+ return None
+
+ # Handed back with term set to the query, and the provider's own state and source
+ # preserved, exactly as a live fetch builds them. The hashed term is how the entry is
+ # stored, not something the ranker should ever see.
+ results = [Document(title=document.title, url=document.url, extract=document.extract,
+ score=document.score, term=query, state=document.state,
+ source=document.source)
+ for document in entries if document.url != EXTERNAL_CACHE_EMPTY_URL]
+ return sorted(results, key=lambda document: -(document.score or 0.0))
+
+
+def store_external_results(source: DocumentSource, query: str, documents: list[Document],
+ now: Optional[int] = None) -> None:
+ """Store one provider's results for a query, replacing any existing entry for it.
+
+ Never raises. A cache write that fails costs a re-fetch later; it must not cost a
+ search its results.
+ """
+ if not settings.EXTERNAL_CACHE_ENABLED:
+ return
+
+ now = int(time.time()) if now is None else now
+ term = external_cache_term(query)
+ entries = _entries_to_store(source, term, documents, now)
+
+ try:
+ with TinyIndex(item_factory=Document, index_path=external_cache_path(), mode='w') as index:
+ page_index = index.get_key_page_index(term)
+ with index.page(page_index) as page:
+ # Newest first. store() drops the tail that does not fit, so ordering by
+ # age is what makes eviction fall on the oldest entries on the page.
+ # Emphatically not the ranker's ordering, which would evict by relevance.
+ # Dropping stale entries here is also the whole of expiry: a page under
+ # write pressure cleans itself, and a page nobody rewrites is not under
+ # pressure, so there is no sweep to schedule. Entries from other sources
+ # are ordinary neighbours here - they compete for the page like any other.
+ # Replaced entries are this query's *for this provider only*. Matching on
+ # the term alone would be wrong now that providers share it: storing one
+ # provider's results would silently drop every other provider's results for
+ # the same query.
+ kept = sorted(
+ (document for document in page.documents
+ if not (document.term == term and document.source == source)
+ and is_fresh(document, now)),
+ key=lambda document: -(document.last_crawled or 0))
+ stored = page.store(entries + kept)
+ if stored < len(entries):
+ logger.warning(
+ "External cache page %d held only %d of %d entries for one query",
+ page_index, stored, len(entries))
+ except Exception:
+ logger.exception("Could not store %s results in the external cache index", source.name)
+
+
+def _entries_to_store(source: DocumentSource, term: str, documents: list[Document],
+ now: int) -> list[Document]:
+ """The documents to write for this query, or the empty-result sentinel if there are none.
+
+ Every entry carries its source, the sentinel included - an entry has to say which
+ provider it came from without the key, and is_fresh has nothing else to go on. `state`
+ is carried through from whatever the provider produced rather than stamped here.
+ """
+ usable = [document for document in documents if document.url and document.title]
+ if not usable:
+ return [Document(title="", url=EXTERNAL_CACHE_EMPTY_URL, extract="", score=0.0, term=term,
+ last_crawled=now, source=source)]
+
+ return [Document(title=document.title, url=document.url, extract=document.extract,
+ score=document.score, term=term, state=document.state, last_crawled=now,
+ source=source)
+ for document in usable]
diff --git a/mwmbl/indexer/wiki_cache.py b/mwmbl/indexer/wiki_cache.py
deleted file mode 100644
index c61f856..0000000
--- a/mwmbl/indexer/wiki_cache.py
+++ /dev/null
@@ -1,181 +0,0 @@
-"""Wikipedia search results, cached in a TinyIndex of their own.
-
-get_wiki_results() used to cache through mwmbl.utils.request_cache, a requests-cache
-*filesystem* session rooted at REQUEST_CACHE_PATH - the same volume as the 400 GB index -
-with no LRU, no size cap and nothing ever calling delete(expired=True), so it grew without
-bound. Because the stored JSON holds the whole request, it also wrote the raw user query
-to disk, where it sat for the length of the expiry.
-
-This is a *separate* index file, not the search index. Caching into the search index was
-tried in #357 and measured in #359: a cache entry and a real document compete for the same
-4 KB page, and on a dense index storing one evicts about as much as it adds (-0.0054 NDCG
-overall, -0.1927 on the queries where it fired). A file of its own also means a cache entry
-can never surface in /raw or in the candidate pool for somebody else's query, which is what
-lets this module keep a negative-cache sentinel - see WIKI_CACHE_EMPTY_URL.
-
-Entries are keyed by a hash of the query, so the file holds no query text. Be straight
-about what that does and does not do: it removes the query *text*, not the fact that
-somebody searched in that area, since an entry still holds the articles that came back.
-That is strictly less than the disk cache gave away.
-"""
-import hashlib
-import hmac
-import time
-from logging import getLogger
-from pathlib import Path
-from typing import Optional
-
-from django.conf import settings
-
-from mwmbl.tinysearchengine.indexer import Document, DocumentState, TinyIndex
-from mwmbl.tokenizer import tokenize
-
-logger = getLogger(__name__)
-
-
-# Terms under this prefix are cache entries rather than words anybody searched for.
-WIKI_CACHE_TERM_PREFIX = "#wiki-"
-
-# A query Wikipedia returned nothing for is stored as a single document under this URL, so
-# that "we asked and there was nothing" stays distinguishable from "we never asked".
-# Without it those queries are re-fetched forever. It is filtered out on read and never
-# leaves this module.
-WIKI_CACHE_EMPTY_URL = "mwmbl:wiki-cache-empty"
-
-
-def wiki_cache_path() -> Path:
- return Path(settings.DATA_PATH) / settings.WIKI_CACHE_INDEX_NAME
-
-
-_cache_index: Optional[TinyIndex] = None
-_cache_index_path: Optional[Path] = None
-
-
-def get_cache_index() -> TinyIndex:
- """The read handle for this process, opened once and left open.
-
- Same lifetime as the main index in search_setup - mmap'd read-only for the life of the
- worker. Writers open their own short-lived 'w' handle instead, as every other writer in
- the codebase does; see store_wiki_results.
-
- Keyed on the path so that override_settings in the tests gets a fresh handle rather
- than whichever index the first test to run happened to open.
- """
- global _cache_index, _cache_index_path
- path = wiki_cache_path()
- if _cache_index is None or _cache_index_path != path:
- if _cache_index is not None:
- _cache_index.__exit__(None, None, None)
- _cache_index = TinyIndex(item_factory=Document, index_path=path)
- _cache_index.__enter__()
- _cache_index_path = path
- return _cache_index
-
-
-def wiki_cache_term(query: str) -> str:
- """The term a query's Wikipedia results are cached under.
-
- Keyed on SECRET_KEY rather than a bare digest: a plain hash of a one- or two-word query
- is trivially brute-forced from a wordlist, so a dump of the file would hand over the
- queries. Keyed, the mapping is not reproducible without the key. Rotating SECRET_KEY
- invalidates the cache, which is harmless - it is a cache.
-
- The query is normalised through tokenize() first, so "Python" and " python " share an
- entry. The disk cache keyed on the request URL built from the raw string, so they did
- not.
- """
- normalised = " ".join(tokenize(query))
- digest = hmac.new(settings.SECRET_KEY.encode("utf8"), normalised.encode("utf8"), hashlib.sha256)
- return WIKI_CACHE_TERM_PREFIX + digest.hexdigest()[:16]
-
-
-def is_fresh(document: Document, now: int) -> bool:
- """Whether a cached document is still within its TTL. Untimestamped means stale."""
- ttl = (settings.WIKI_CACHE_NEGATIVE_TTL_SECONDS if document.url == WIKI_CACHE_EMPTY_URL
- else settings.WIKI_CACHE_TTL_SECONDS)
- return document.last_crawled is not None and now - document.last_crawled < ttl
-
-
-def get_cached_wiki_results(query: str, now: Optional[int] = None) -> Optional[list[Document]]:
- """A query's cached Wikipedia results.
-
- None means we have nothing for this query and should ask Wikipedia. A list - possibly
- empty - means we already asked; an empty one is a query Wikipedia had nothing for.
- """
- if not settings.WIKI_CACHE_ENABLED:
- return None
-
- now = int(time.time()) if now is None else now
- term = wiki_cache_term(query)
- try:
- stored = get_cache_index().retrieve(term)
- except FileNotFoundError:
- logger.warning("No Wikipedia cache index at %s", wiki_cache_path())
- return None
- except Exception:
- # A cache we cannot read is a cache miss, never a failed search. retrieve() already
- # absorbs an unreadable page; this covers the file itself going away underneath us.
- logger.exception("Could not read the Wikipedia cache index")
- return None
-
- # retrieve() hands back the whole page, which is shared with every other query that
- # hashes to it, and it also lets through documents with no term at all.
- entries = [document for document in stored if document.term == term and is_fresh(document, now)]
- if not entries:
- return None
-
- # Handed back with term set to the query, exactly as a live fetch builds them. The
- # hashed term is how the entry is stored, not something the ranker should ever see.
- results = [Document(title=document.title, url=document.url, extract=document.extract,
- score=document.score, term=query, state=document.state)
- for document in entries if document.url != WIKI_CACHE_EMPTY_URL]
- return sorted(results, key=lambda document: -(document.score or 0.0))
-
-
-def store_wiki_results(query: str, documents: list[Document], now: Optional[int] = None) -> None:
- """Store a query's Wikipedia results, replacing any existing entry for it.
-
- Never raises. A cache write that fails costs a re-fetch later; it must not cost a
- search its results.
- """
- if not settings.WIKI_CACHE_ENABLED:
- return
-
- now = int(time.time()) if now is None else now
- term = wiki_cache_term(query)
- entries = _entries_to_store(term, documents, now)
-
- try:
- with TinyIndex(item_factory=Document, index_path=wiki_cache_path(), mode='w') as index:
- page_index = index.get_key_page_index(term)
- with index.page(page_index) as page:
- # Newest first. store() drops the tail that does not fit, so ordering by
- # age is what makes eviction fall on the oldest entries on the page.
- # Emphatically not the ranker's ordering, which would evict by relevance.
- # Dropping stale entries here is also the whole of expiry: a page under
- # write pressure cleans itself, and a page nobody rewrites is not under
- # pressure, so there is no sweep to schedule.
- kept = sorted(
- (document for document in page.documents
- if document.term != term and is_fresh(document, now)),
- key=lambda document: -(document.last_crawled or 0))
- stored = page.store(entries + kept)
- if stored < len(entries):
- logger.warning(
- "Wiki cache page %d held only %d of %d entries for one query",
- page_index, stored, len(entries))
- except Exception:
- logger.exception("Could not store Wikipedia results in the cache index")
-
-
-def _entries_to_store(term: str, documents: list[Document], now: int) -> list[Document]:
- """The documents to write for this query, or the empty-result sentinel if there are none."""
- usable = [document for document in documents if document.url and document.title]
- if not usable:
- return [Document(title="", url=WIKI_CACHE_EMPTY_URL, extract="", score=0.0, term=term,
- state=DocumentState.FROM_WIKI, last_crawled=now)]
-
- return [Document(title=document.title, url=document.url, extract=document.extract,
- score=document.score, term=term, state=DocumentState.FROM_WIKI,
- last_crawled=now)
- for document in usable]
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index afc5e15..41fa02c 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -337,12 +337,17 @@ BLACKLIST_PURGE_BATCH_SIZE = 1000 # documents removed from the index pe
# than one per approval - see mwmbl.signals.
BLACKLIST_SNAPSHOT_APPROVAL_DELAY_SECONDS = 600
-# Wikipedia results are cached in a TinyIndex of their own (see mwmbl.indexer.wiki_cache).
-# A dedicated file rather than the search index because a cache entry competing with real
-# documents for a shared page evicts them - #359 measured that as a net NDCG loss - and
-# because a separate file never enters the candidate pool for anyone else's query.
-WIKI_CACHE_ENABLED = os.environ.get("WIKI_CACHE_ENABLED", "true").lower() != "false"
-WIKI_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks
-# A query Wikipedia has nothing for is worth remembering too, or it is re-fetched forever.
-# Shorter, because an article appearing is a likelier change than an existing one moving.
-WIKI_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60
+# Results from external search providers - Wikipedia today, others later - are cached in a
+# TinyIndex of their own (see mwmbl.indexer.external_cache). Entries are namespaced by
+# source, so providers share the file without sharing entries. A dedicated file rather than
+# the search index because a cache entry competing with real documents for a shared page
+# evicts them - #359 measured that as a net NDCG loss - and because a separate file never
+# enters the candidate pool for anyone else's query.
+EXTERNAL_CACHE_ENABLED = os.environ.get("EXTERNAL_CACHE_ENABLED", "true").lower() != "false"
+# One TTL for every provider. Wikipedia articles move rarely, which is what justifies six
+# months; a provider whose results turn over faster wants its own, and the place to add that
+# is here, keyed by source.
+EXTERNAL_CACHE_TTL_SECONDS = 26 * 7 * 24 * 60 * 60 # 6 months; the disk cache kept 10 weeks
+# A query a provider has nothing for is worth remembering too, or it is re-fetched forever.
+# Shorter, because a result appearing is a likelier change than an existing one moving.
+EXTERNAL_CACHE_NEGATIVE_TTL_SECONDS = 7 * 24 * 60 * 60
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index 9b44028..a598abd 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -22,8 +22,8 @@ INDEX_NAME = 'index-v2.tinysearch'
NUM_PAGES = 2560
-WIKI_CACHE_INDEX_NAME = 'wiki-cache.tinysearch'
-WIKI_CACHE_NUM_PAGES = 2560
+EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'
+EXTERNAL_CACHE_NUM_PAGES = 2560
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")
URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")
diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.py
index caf1261..38d2e04 100644
--- a/mwmbl/settings_prod.py
+++ b/mwmbl/settings_prod.py
@@ -31,10 +31,30 @@ INDEX_NAME = 'index-v2-400G.tinysearch'
# 400GB index
NUM_PAGES = 102400000
-WIKI_CACHE_INDEX_NAME = 'wiki-cache.tinysearch'
-# 4GB. ~12 entries fit a 4KB page, but pages are hash-assigned, so keep the load factor
-# well under that: this holds a few million queries before collisions start evicting early.
-WIKI_CACHE_NUM_PAGES = 1_000_000
+EXTERNAL_CACHE_INDEX_NAME = 'external-cache.tinysearch'
+# ~15 GB. Sized by page occupancy rather than by naive capacity, because pages are assigned
+# by hash: the queries per page are Poisson-distributed, not evenly spread, so the pages
+# that run over start truncating while empty ones sit unused.
+#
+# A 4 KB page holds ~13 cached queries - 37 documents at 3 results a query - measured by
+# packing the 11,693 real Wikipedia responses left in the old devdata/request_cache until
+# the page overflowed. That is in line with the live index, which averages 35.4 documents a
+# page, and it is worth stating how it was measured because a first pass using synthetic
+# random-vocabulary text got 8, 60% low: zstd exploits the redundancy in real prose and
+# there is a lot of it. Real extracts are ~154 characters.
+#
+# At an average load of L queries per page the fraction still resident is E[min(K, 13)] / L
+# for K ~ Poisson(L), which gives, for one provider:
+#
+# L = 8.2 -> 99% retained -> 30.9M distinct queries
+# L = 10.9 -> 95% retained -> 40.8M
+# L = 12.7 -> 90% retained -> 47.6M
+# L = 13 -> 89% retained -> 48.8M (the naive number)
+#
+# A query cached from several providers holds one entry per provider, so divide by the
+# number of providers: with two, ~15M queries at 99% and ~24M at 90%. Longer extracts fit
+# fewer per page, so a provider more verbose than Wikipedia costs more than its share.
+EXTERNAL_CACHE_NUM_PAGES = 3_750_000
URLS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "urls-{year}-{month}.bloom")
URLS_BLOOM_FILTER_FALLBACK_PATH = str(Path(DATA_PATH) / "urls.bloom")
diff --git a/mwmbl/settings_test.py b/mwmbl/settings_test.py
index 9716918..8cd5738 100644
--- a/mwmbl/settings_test.py
+++ b/mwmbl/settings_test.py
@@ -11,8 +11,8 @@ HAS_DATABASE = False
DATA_PATH = "/tmp/test_mwmbl_data"
INDEX_NAME = "index-v2.tinysearch"
NUM_PAGES = 10
-WIKI_CACHE_INDEX_NAME = "wiki-cache.tinysearch"
-WIKI_CACHE_NUM_PAGES = 10
+EXTERNAL_CACHE_INDEX_NAME = "external-cache.tinysearch"
+EXTERNAL_CACHE_NUM_PAGES = 10
# Use fakeredis for cache so tests don't need a real Redis server
CACHES = {
diff --git a/mwmbl/tinysearchengine/indexer.py b/mwmbl/tinysearchengine/indexer.py
index 1605885..cdebeee 100644
--- a/mwmbl/tinysearchengine/indexer.py
+++ b/mwmbl/tinysearchengine/indexer.py
@@ -74,6 +74,22 @@ class DocumentState(IntEnum):
CURATED_STATES = {state for state in DocumentState if state.value >= 7}
+class DocumentSource(IntEnum):
+ """Which external provider a document came from. None means our own crawl.
+
+ Only the external results cache sets this (see mwmbl.indexer.external_cache); documents
+ in the search index leave it None, and as_tuple truncates trailing Nones, so they cost
+ nothing for it. The values are written to disk, so they are append-only: never reuse a
+ number for a different provider, or a stale cache entry becomes a lie about where its
+ results came from.
+ """
+ WIKIPEDIA = 1
+ # Reserved, not yet wired up to a provider. The number is here rather than added later
+ # because it is the numbering registry: with only one member the source field cannot be
+ # told apart from "unset" and the cache's per-source behaviour cannot be tested at all.
+ STAAN = 2
+
+
@dataclass
class Document:
title: str
@@ -84,6 +100,7 @@ class Document:
state: Optional[int] = None
user_ids: Optional[List[int]] = None
last_crawled: Optional[int] = None
+ source: Optional[int] = None
def __init__(
self,
@@ -95,6 +112,7 @@ class Document:
state: Optional[int | DocumentState] = None,
user_ids: Optional[List[int]] = None,
last_crawled: Optional[int] = None,
+ source: Optional[int | DocumentSource] = None,
):
# Sometimes the title or extract may be None, probably because of user generated content
# It's not allowed to be None though, or things will break
@@ -106,6 +124,7 @@ class Document:
self.state = None if state is None else DocumentState(state)
self.user_ids = user_ids
self.last_crawled = last_crawled
+ self.source = None if source is None else DocumentSource(source)
def as_tuple(self):
"""
@@ -120,6 +139,7 @@ class Document:
None if self.state is None else self.state.value,
self.user_ids,
self.last_crawled,
+ None if self.source is None else self.source.value,
]
while values[-1] is None:
values = values[:-1]
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 09ec4b4..d36955b 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -22,9 +22,9 @@ from mwmbl.format import get_query_regex
from mwmbl.hn_top_domains_filtered import DOMAINS
from mwmbl.indexer.blacklist_snapshot import get_snapshot_blacklist
from mwmbl.indexer.purge_queue import enqueue_for_purge
-from mwmbl.indexer.wiki_cache import get_cached_wiki_results, store_wiki_results
+from mwmbl.indexer.external_cache import get_cached_external_results, store_external_results
from mwmbl.tinysearchengine.completer import Completer
-from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
+from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentSource, DocumentState
from mwmbl.tokenizer import tokenize, get_bigrams
from mwmbl.utils import get_domain
@@ -433,7 +433,7 @@ WIKI_URL_FORMAT = "https://en.wikipedia.org/wiki/{title}"
# calls — e.g. an evaluation run that queries it once per gold query. Retry with
# backoff, honouring any Retry-After, so a transient 429 doesn't silently drop a
# query's wiki results. Live search never bursts, so this adds no latency in
-# normal operation. Successful responses are cached for months (see mwmbl.indexer.wiki_cache).
+# normal operation. Successful responses are cached for months (see mwmbl.indexer.external_cache).
WIKI_RETRY = Retry(total=4, backoff_factor=0.5, status_forcelist=(429, 502, 503, 504),
allowed_methods=frozenset({"GET"}), respect_retry_after_header=True)
@@ -488,7 +488,7 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:
# Ahead of the circuit breaker deliberately: a cache hit costs Wikipedia nothing, so it
# should still be served while we are backing off from a rate limit.
- cached = get_cached_wiki_results(query)
+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, query)
if cached is not None:
return cached[:max_wiki_results]
@@ -527,13 +527,14 @@ def get_wiki_results(s: str, max_wiki_results: int) -> list[Document]:
return []
wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
- max_wiki_results + 1 - i, query, state=DocumentState.FROM_WIKI)
+ max_wiki_results + 1 - i, query, state=DocumentState.FROM_WIKI,
+ source=DocumentSource.WIKIPEDIA)
for i, result in enumerate(wiki_response['query']['search'][:max_wiki_results])]
# Only a well-formed answer is stored, including one with no results in it. Every path
# above returns [] without storing, so a transient failure is never remembered as
# "Wikipedia has nothing for this".
- store_wiki_results(query, wiki_results)
+ store_external_results(DocumentSource.WIKIPEDIA, query, wiki_results)
return wiki_results
diff --git a/test/test_wiki_cache.py b/test/test_external_cache.py
similarity index 52%
rename from test/test_wiki_cache.py
rename to test/test_external_cache.py
index 988cb24..4bb68dd 100644
--- a/test/test_wiki_cache.py
+++ b/test/test_external_cache.py
@@ -1,4 +1,4 @@
-"""Tests for the dedicated Wikipedia cache index (mwmbl.indexer.wiki_cache).
+"""Tests for the dedicated external results cache index (mwmbl.indexer.external_cache).
Where these overlap with the tests on the #357 branch they are deliberately the same
assertions - the term derivation and the freshness rule are carried over unchanged. What is
@@ -6,6 +6,7 @@ gone is everything that existed only because cache entries shared a file with re
content: the anonymisation gate, the Redis queue, the general-indexing path, and the guards
stopping a cache term reaching curation or /raw.
"""
+import json
import random
import string
import time
@@ -15,35 +16,40 @@ import pytest
from django.test import override_settings
from mwmbl.apps import create_index
-from mwmbl.indexer.wiki_cache import (
- WIKI_CACHE_EMPTY_URL,
- WIKI_CACHE_TERM_PREFIX,
- get_cached_wiki_results,
+from mwmbl.indexer.external_cache import (
+ EXTERNAL_CACHE_EMPTY_URL,
+ EXTERNAL_CACHE_TERM_PREFIX,
+ external_cache_path,
+ external_cache_term,
+ get_cached_external_results,
is_fresh,
- store_wiki_results,
- wiki_cache_path,
- wiki_cache_term,
+ store_external_results,
)
from mwmbl.tinysearchengine import rank
-from mwmbl.tinysearchengine.indexer import PAGE_SIZE, Document, DocumentState, TinyIndex
+from mwmbl.tinysearchengine.indexer import (PAGE_SIZE, Document, DocumentSource, DocumentState,
+ TinyIndex)
from mwmbl.tinysearchengine.rank import Ranker, get_wiki_results
NUM_PAGES = 8
+# state and source are set because get_wiki_results sets them, and the cache preserves what
+# the provider produced rather than stamping its own.
PYTHON = Document("Python (programming language)", "https://en.wikipedia.org/wiki/Python",
- "A high-level programming language.", 3.0)
+ "A high-level programming language.", 3.0, state=DocumentState.FROM_WIKI,
+ source=DocumentSource.WIKIPEDIA)
MONTY = Document("Monty Python", "https://en.wikipedia.org/wiki/Monty_Python",
- "A British comedy troupe.", 2.0)
+ "A British comedy troupe.", 2.0, state=DocumentState.FROM_WIKI,
+ source=DocumentSource.WIKIPEDIA)
@pytest.fixture
def cache_index(tmp_path):
- """A real, empty wiki cache index that the module-level helpers will pick up."""
- with override_settings(DATA_PATH=str(tmp_path), WIKI_CACHE_INDEX_NAME="wiki-cache.tinysearch",
- WIKI_CACHE_NUM_PAGES=NUM_PAGES, WIKI_CACHE_ENABLED=True):
- TinyIndex.create(item_factory=Document, index_path=str(wiki_cache_path()),
+ """A real, empty external results cache index that the module-level helpers pick up."""
+ with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="external-cache.tinysearch",
+ EXTERNAL_CACHE_NUM_PAGES=NUM_PAGES, EXTERNAL_CACHE_ENABLED=True):
+ TinyIndex.create(item_factory=Document, index_path=str(external_cache_path()),
num_pages=NUM_PAGES, page_size=PAGE_SIZE)
- yield str(wiki_cache_path())
+ yield str(external_cache_path())
def _wiki_api_response(*titles):
@@ -71,14 +77,14 @@ def _patched_wikipedia(response):
def test_cache_term_is_stable_and_normalised():
"""Whitespace and case must not split one query across several entries. The disk cache
keyed on the raw request URL, so " python " and "Python" missed each other."""
- assert wiki_cache_term("python") == wiki_cache_term(" PYTHON ")
- assert wiki_cache_term("monty python") == wiki_cache_term("Monty Python")
- assert wiki_cache_term("python") != wiki_cache_term("monty python")
+ assert external_cache_term("python") == external_cache_term(" PYTHON ")
+ assert external_cache_term("monty python") == external_cache_term("Monty Python")
+ assert external_cache_term("python") != external_cache_term("monty python")
def test_cache_term_does_not_contain_the_query():
- term = wiki_cache_term("something private")
- assert term.startswith(WIKI_CACHE_TERM_PREFIX)
+ term = external_cache_term("something private")
+ assert term.startswith(EXTERNAL_CACHE_TERM_PREFIX)
assert "something" not in term
assert "private" not in term
@@ -87,9 +93,9 @@ def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():
"""A bare digest of a one-word query is brute-forceable from a wordlist; keyed on
SECRET_KEY the term->query mapping is not reproducible without the key."""
with override_settings(SECRET_KEY="one"):
- first = wiki_cache_term("bananas")
+ first = external_cache_term("bananas")
with override_settings(SECRET_KEY="two"):
- second = wiki_cache_term("bananas")
+ second = external_cache_term("bananas")
assert first != second
@@ -99,9 +105,9 @@ def test_cache_term_is_keyed_so_it_cannot_be_recomputed_without_the_secret():
# ---------------------------------------------------------------------------
def test_results_come_back_from_the_index(cache_index):
- store_wiki_results("python", [PYTHON, MONTY])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])
- cached = get_cached_wiki_results("python")
+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")
assert [document.url for document in cached] == [PYTHON.url, MONTY.url] # descending score
assert [document.score for document in cached] == [3.0, 2.0]
@@ -111,32 +117,123 @@ def test_results_come_back_from_the_index(cache_index):
def test_a_hit_is_indistinguishable_from_a_live_fetch(cache_index):
"""The hashed term is a storage detail. Handing it to the ranker would make a cached
result differ from the identical one fetched live."""
- store_wiki_results("python", [PYTHON])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
- cached = get_cached_wiki_results("python")
+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")
assert [document.term for document in cached] == ["python"]
def test_a_normalised_variant_of_the_query_hits_the_same_entry(cache_index):
- store_wiki_results("Monty Python", [MONTY])
+ store_external_results(DocumentSource.WIKIPEDIA, "Monty Python", [MONTY])
- assert get_cached_wiki_results(" monty python ") is not None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, " monty python ") is not None
def test_a_query_we_have_never_asked_about_is_a_miss(cache_index):
- store_wiki_results("python", [PYTHON])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
- assert get_cached_wiki_results("something else entirely") is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "something else entirely") is None
+
+
+def _all_stored_documents(index_path):
+ """Every document in the file, decompressed. Grepping the raw bytes proves little: the
+ pages are zstd-compressed, so plaintext would not show up there either way."""
+ with TinyIndex(Document, index_path, 'r') as index:
+ return [document for page in range(index.num_pages) for document in index.get_page(page)]
def test_the_file_holds_no_query_text(cache_index):
- store_wiki_results("something private", [PYTHON])
+ store_external_results(DocumentSource.WIKIPEDIA, "something private", [PYTHON])
- with open(cache_index, "rb") as index_file:
- raw = index_file.read()
+ stored = _all_stored_documents(cache_index)
- assert b"something private" not in raw
+ assert stored, "nothing was written, so this would pass for the wrong reason"
+ written = json.dumps([document.as_tuple() for document in stored])
+ assert "something private" not in written
+ assert "private" not in written
+
+
+# ---------------------------------------------------------------------------
+# Several providers sharing the file
+# ---------------------------------------------------------------------------
+
+# The second provider is the whole reason the cache is keyed by source rather than by query
+# alone, so the tests below exercise it even though nothing fetches from it yet.
+OTHER_SOURCE = DocumentSource.STAAN
+
+STAAN_RESULT = Document("Python tutorial", "https://example.com/python",
+ "Somewhere that is not Wikipedia.", 9.0, source=DocumentSource.STAAN)
+
+
+def test_two_providers_do_not_share_an_entry_for_the_same_query(cache_index):
+ """Providers share the term, and therefore the page, but not the entry: one provider
+ answering a query must not stop the other being asked, or answer on its behalf."""
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
+
+ assert get_cached_external_results(OTHER_SOURCE, "python") is None
+
+ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])
+
+ assert [d.url for d in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [PYTHON.url]
+ assert [d.url for d in get_cached_external_results(OTHER_SOURCE, "python")] == [STAAN_RESULT.url]
+
+
+def test_an_entry_says_which_provider_it_came_from_but_not_which_query(cache_index):
+ """An entry has to be attributable to its provider - to count them, to drop one
+ provider's entries, or to give one its own TTL - which the query must never be. The
+ source is on the document; the query is not anywhere."""
+ store_external_results(DocumentSource.WIKIPEDIA, "something private", [PYTHON])
+
+ stored = _all_stored_documents(cache_index)
+
+ assert [document for document in stored if document.source == DocumentSource.WIKIPEDIA]
+ assert not any("private" in document.term for document in stored)
+
+
+def test_storing_one_provider_does_not_evict_another_for_the_same_query(cache_index):
+ """The trap in sharing a term. The write path replaces "this query's entries", and if
+ that means the term alone it wipes every other provider's results for the query - with
+ no error, just a silent extra fetch every time."""
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
+ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [MONTY])
+
+ assert [d.url for d in get_cached_external_results(OTHER_SOURCE, "python")] == [STAAN_RESULT.url]
+ assert [d.url for d in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [MONTY.url]
+
+
+def test_both_providers_results_for_a_query_live_on_one_page(cache_index):
+ """The reason the term does not name the source: one page read serves every provider,
+ so asking the second costs no extra page fault."""
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
+ store_external_results(OTHER_SOURCE, "python", [STAAN_RESULT])
+
+ with TinyIndex(Document, cache_index, 'r') as index:
+ page = index.get_page(index.get_key_page_index(external_cache_term("python")))
+
+ assert {document.source for document in page} == {DocumentSource.WIKIPEDIA, DocumentSource.STAAN}
+
+
+def test_the_negative_sentinel_carries_its_source_too(cache_index):
+ """is_fresh reads document.source off a page's other entries, where the term tells it
+ nothing, so an entry with no results still has to say where it came from."""
+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])
+
+ stored = _all_stored_documents(cache_index)
+
+ assert [document.source for document in stored] == [DocumentSource.WIKIPEDIA]
+
+
+def test_a_providers_state_and_source_survive_the_round_trip(cache_index):
+ """The cache preserves what the provider produced rather than stamping its own, so a
+ cached result stays attributable once there is a second provider."""
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
+
+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "python")
+
+ assert [document.state for document in cached] == [DocumentState.FROM_WIKI]
+ assert [document.source for document in cached] == [DocumentSource.WIKIPEDIA]
# ---------------------------------------------------------------------------
@@ -145,10 +242,10 @@ def test_the_file_holds_no_query_text(cache_index):
def test_a_stale_entry_is_not_a_cache_hit(cache_index):
now = int(time.time())
- store_wiki_results("python", [PYTHON], now=now)
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)
- fresh = get_cached_wiki_results("python", now=now + 60)
- stale = get_cached_wiki_results("python", now=now + 27 * 7 * 24 * 60 * 60)
+ fresh = get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=now + 60)
+ stale = get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=now + 27 * 7 * 24 * 60 * 60)
assert fresh is not None
assert stale is None
@@ -165,13 +262,13 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):
sweep to schedule and nothing accumulates on a page nobody writes to."""
now = int(time.time())
old_query, new_query = _colliding_queries(cache_index)
- store_wiki_results(old_query, [PYTHON], now=now)
+ store_external_results(DocumentSource.WIKIPEDIA, old_query, [PYTHON], now=now)
much_later = now + 27 * 7 * 24 * 60 * 60
- store_wiki_results(new_query, [MONTY], now=much_later)
+ store_external_results(DocumentSource.WIKIPEDIA, new_query, [MONTY], now=much_later)
with TinyIndex(Document, cache_index, 'r') as index:
- page = index.get_page(index.get_key_page_index(wiki_cache_term(new_query)))
+ page = index.get_page(index.get_key_page_index(external_cache_term(new_query)))
assert [document.url for document in page] == [MONTY.url]
@@ -180,39 +277,48 @@ def test_stale_entries_are_dropped_when_the_page_is_rewritten(cache_index):
# Negative caching
# ---------------------------------------------------------------------------
-def test_a_query_wikipedia_has_nothing_for_is_remembered(cache_index):
+def test_a_query_a_provider_has_nothing_for_is_remembered(cache_index):
"""Without this the query is re-fetched forever - #357 listed it as unfixable, because
a sentinel in the search index would surface in /raw and in every candidate list."""
- store_wiki_results("nonsense query", [])
+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])
- assert get_cached_wiki_results("nonsense query") == []
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query") == []
def test_an_empty_entry_is_not_returned_as_a_result(cache_index):
- store_wiki_results("nonsense query", [])
+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])
- cached = get_cached_wiki_results("nonsense query")
+ cached = get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query")
assert [document.url for document in cached] == []
- assert WIKI_CACHE_EMPTY_URL not in [document.url for document in cached]
+ assert EXTERNAL_CACHE_EMPTY_URL not in [document.url for document in cached]
+
+
+def test_one_providers_empty_result_is_not_anothers(cache_index):
+ """Sharing a page makes this worth pinning down: "Wikipedia has nothing" must not read
+ as "Staan has nothing", or the second provider is never asked."""
+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [])
+
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query") == []
+ assert get_cached_external_results(OTHER_SOURCE, "nonsense query") is None
def test_an_empty_entry_expires_sooner_than_a_real_one(cache_index):
"""An article appearing is a likelier change than an existing one moving."""
now = int(time.time())
- store_wiki_results("nonsense query", [], now=now)
- store_wiki_results("python", [PYTHON], now=now)
+ store_external_results(DocumentSource.WIKIPEDIA, "nonsense query", [], now=now)
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON], now=now)
later = now + 8 * 24 * 60 * 60 # past the negative TTL, well inside the positive one
- assert get_cached_wiki_results("nonsense query", now=later) is None
- assert get_cached_wiki_results("python", now=later) is not None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "nonsense query", now=later) is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python", now=later) is not None
def test_documents_without_a_url_or_title_are_not_stored(cache_index):
- store_wiki_results("python", [Document("", "", "", 1.0)])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [Document("", "", "", 1.0)])
- assert get_cached_wiki_results("python") == []
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") == []
# ---------------------------------------------------------------------------
@@ -228,11 +334,11 @@ def test_a_cache_miss_calls_wikipedia_and_stores_the_result(cache_index):
assert session.get.call_count == 1
assert [document.url for document in results] == ["https://en.wikipedia.org/wiki/Python_(programming_language)"]
- assert get_cached_wiki_results("python") is not None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is not None
def test_a_cache_hit_does_not_call_wikipedia(cache_index):
- store_wiki_results("python", [PYTHON])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
mock, session = _patched_wikipedia(_wiki_api_response("Python"))
try:
@@ -247,7 +353,7 @@ def test_a_cache_hit_does_not_call_wikipedia(cache_index):
def test_a_hit_is_served_while_the_circuit_breaker_is_open(cache_index):
"""A cached answer costs Wikipedia nothing, so backing off from a rate limit is no
reason to withhold it."""
- store_wiki_results("python", [PYTHON])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
rank._trip_wiki_circuit()
try:
results = get_wiki_results("python", 3)
@@ -266,7 +372,7 @@ def test_a_failed_fetch_is_not_remembered_as_an_empty_result(cache_index):
finally:
mock.stop()
- assert get_cached_wiki_results("python") is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None
def test_an_error_response_is_not_remembered_as_an_empty_result(cache_index):
@@ -276,7 +382,7 @@ def test_an_error_response_is_not_remembered_as_an_empty_result(cache_index):
finally:
mock.stop()
- assert get_cached_wiki_results("python") is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None
def test_a_broken_cache_write_does_not_break_the_search(cache_index):
@@ -284,7 +390,7 @@ def test_a_broken_cache_write_does_not_break_the_search(cache_index):
results."""
mock, _ = _patched_wikipedia(_wiki_api_response("Python"))
try:
- with patch("mwmbl.indexer.wiki_cache.TinyIndex", side_effect=OSError("disk gone")):
+ with patch("mwmbl.indexer.external_cache.TinyIndex", side_effect=OSError("disk gone")):
results = get_wiki_results("python", 3)
finally:
mock.stop()
@@ -311,17 +417,17 @@ def test_the_query_is_truncated_before_it_reaches_wikipedia(cache_index):
# ---------------------------------------------------------------------------
def test_the_kill_switch_stops_reads_and_writes(cache_index):
- with override_settings(WIKI_CACHE_ENABLED=False):
- store_wiki_results("python", [PYTHON])
- assert get_cached_wiki_results("python") is None
+ with override_settings(EXTERNAL_CACHE_ENABLED=False):
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON])
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None
- assert get_cached_wiki_results("python") is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None
def test_a_missing_cache_file_is_a_miss_rather_than_an_error(tmp_path):
- with override_settings(DATA_PATH=str(tmp_path), WIKI_CACHE_INDEX_NAME="absent.tinysearch",
- WIKI_CACHE_ENABLED=True):
- assert get_cached_wiki_results("python") is None
+ with override_settings(DATA_PATH=str(tmp_path), EXTERNAL_CACHE_INDEX_NAME="absent.tinysearch",
+ EXTERNAL_CACHE_ENABLED=True):
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, "python") is None
# ---------------------------------------------------------------------------
@@ -332,10 +438,10 @@ def _colliding_queries(index_path):
"""Two queries whose cache terms land on the same page."""
with TinyIndex(Document, index_path, 'r') as index:
first = "collide-0"
- first_page = index.get_key_page_index(wiki_cache_term(first))
+ first_page = index.get_key_page_index(external_cache_term(first))
for i in range(1, 10000):
candidate = f"collide-{i}"
- if index.get_key_page_index(wiki_cache_term(candidate)) == first_page:
+ if index.get_key_page_index(external_cache_term(candidate)) == first_page:
return first, candidate
raise AssertionError("No colliding query found")
@@ -354,20 +460,20 @@ def test_the_newest_entry_survives_a_full_page_and_the_oldest_is_evicted(cache_i
bulky = [Document(f"Title {i}", f"https://en.wikipedia.org/wiki/{i}", _filler(2000), 3.0)
for i in range(4)]
- store_wiki_results(old_query, bulky, now=now)
- assert get_cached_wiki_results(old_query, now=now) is not None
+ store_external_results(DocumentSource.WIKIPEDIA, old_query, bulky, now=now)
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, old_query, now=now) is not None
- store_wiki_results(new_query, bulky, now=now + 1)
+ store_external_results(DocumentSource.WIKIPEDIA, new_query, bulky, now=now + 1)
- assert get_cached_wiki_results(new_query, now=now + 1) is not None
- assert get_cached_wiki_results(old_query, now=now + 1) is None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, new_query, now=now + 1) is not None
+ assert get_cached_external_results(DocumentSource.WIKIPEDIA, old_query, now=now + 1) is None
def test_an_entry_replaces_its_own_previous_results(cache_index):
- store_wiki_results("python", [PYTHON, MONTY])
- store_wiki_results("python", [MONTY])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [PYTHON, MONTY])
+ store_external_results(DocumentSource.WIKIPEDIA, "python", [MONTY])
- assert [document.url for document in get_cached_wiki_results("python")] == [MONTY.url]
+ assert [document.url for document in get_cached_external_results(DocumentSource.WIKIPEDIA, "python")] == [MONTY.url]
# ---------------------------------------------------------------------------
@@ -416,11 +522,11 @@ def test_search_calls_external_search_by_default(cache_index):
def test_a_resized_cache_index_is_rebuilt_rather_than_crashing_startup(tmp_path):
"""Resizing a cache should be a config change, not a startup crash - the cost of
rebuilding is re-fetching."""
- path = tmp_path / "wiki-cache.tinysearch"
+ path = tmp_path / "external-cache.tinysearch"
TinyIndex.create(item_factory=Document, index_path=str(path), num_pages=4, page_size=PAGE_SIZE)
with override_settings(DATA_PATH=str(tmp_path)):
- create_index("wiki-cache.tinysearch", 16, rebuild_on_mismatch=True)
+ create_index("external-cache.tinysearch", 16, rebuild_on_mismatch=True)
with TinyIndex(Document, str(path), 'r') as index:
assert index.num_pages == 16
diff --git a/test/test_rank.py b/test/test_rank.py
index 05055e6..2e4b73c 100644
--- a/test/test_rank.py
+++ b/test/test_rank.py
@@ -73,7 +73,7 @@ def _reset_wiki_circuit():
def _get_wiki_results_with_session(session):
# The cache is off in these tests: they are about what the fetch does when Wikipedia
# misbehaves, and a hit would return before the fetch ever ran.
- with override_settings(WIKI_CACHE_ENABLED=False), \
+ with override_settings(EXTERNAL_CACHE_ENABLED=False), \
patch.object(rank.requests, "Session") as mock_session:
mock_session.return_value.__enter__.return_value = session
return get_wiki_results("query", 5)
@@ -113,7 +113,7 @@ def test_wiki_query_text_containing_429_is_not_mistaken_for_rate_limit():
def test_open_wiki_circuit_short_circuits_without_calling_wikipedia():
rank._trip_wiki_circuit()
- with override_settings(WIKI_CACHE_ENABLED=False), \
+ with override_settings(EXTERNAL_CACHE_ENABLED=False), \
patch.object(rank.requests, "Session") as mock_session:
results = get_wiki_results("query", 5)