An open source, non-profit web search engine
Cache request responses in a common place
Daoud Clarked8cd13a95051
diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.pyindex e4f17f4..eb7f1eb 100644--- a/mwmbl/indexer/blacklist.py+++ b/mwmbl/indexer/blacklist.py@@ -1,13 +1,12 @@from datetime import timedelta-from requests_cache import CachedSession-from mwmbl.hn_top_domains_filtered import DOMAINSfrom mwmbl.settings import BLACKLIST_DOMAINS_URL, EXCLUDED_DOMAINS, DOMAIN_BLACKLIST_REGEX+from mwmbl.utils import request_cachedef get_blacklist_domains() -> set[str]:- with CachedSession(expire_after=timedelta(days=1)) as session:+ with request_cache(expire_after=timedelta(days=1)) as session:response = session.get(BLACKLIST_DOMAINS_URL)return set(response.text.split())diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.pyindex 1301e18..bc67f0a 100644--- a/mwmbl/settings_dev.py+++ b/mwmbl/settings_dev.py@@ -36,3 +36,5 @@ NUM_URLS_IN_BLOOM_FILTER = 100_000DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")NUM_DOMAINS_IN_BLOOM_FILTER = 100_000++REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache.sqlite"diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.pyindex 445e43b..9fad002 100644--- a/mwmbl/settings_prod.py+++ b/mwmbl/settings_prod.py@@ -41,3 +41,5 @@ NUM_URLS_IN_BLOOM_FILTER = 200_000_000DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")NUM_DOMAINS_IN_BLOOM_FILTER = 100_000_000++REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache.sqlite"diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex e9a1deb..c35baef 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -6,17 +6,14 @@ from abc import abstractmethodfrom datetime import timedeltafrom logging import getLoggerfrom operator import itemgetter-from typing import Optionalfrom urllib.parse import urlparse-import requests-from requests_cache import CachedSession--from mwmbl.format import format_result_with_pattern, get_query_regex-from mwmbl.tokenizer import tokenize, get_bigrams-from mwmbl.tinysearchengine.completer import Completer+from mwmbl.format import get_query_regexfrom mwmbl.hn_top_domains_filtered import DOMAINS+from mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState+from mwmbl.tokenizer import tokenize, get_bigrams+from mwmbl.utils import request_cachelogger = getLogger(__name__)@@ -271,12 +268,13 @@ class HeuristicAndWikiRanker(HeuristicRanker):max_wiki_results = 5def search(self, s: str, additional_results: list[Document]) -> list[Document]:- with CachedSession(expire_after=timedelta(weeks=12)) as session:- escaped_query = urllib.parse.quote(s, safe='')+ escaped_query = urllib.parse.quote(s, safe='')+ with request_cache(timedelta(weeks=10)) as session:wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()- wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),- 0.0, s, state=DocumentState.FROM_WIKI)- for result in wiki_response['query']['search'][:self.max_wiki_results]]++ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),+ 0.0, s, state=DocumentState.FROM_WIKI)+ for result in wiki_response['query']['search'][:self.max_wiki_results]]return super().search(s, wiki_results + additional_results)diff --git a/mwmbl/utils.py b/mwmbl/utils.pyindex c797553..4554ed5 100644--- a/mwmbl/utils.py+++ b/mwmbl/utils.py@@ -1,8 +1,11 @@import refrom dataclasses import dataclass+from datetime import timedeltafrom typing import Sequence+from django.conf import settingsfrom django.core.exceptions import ValidationError+from requests_cache import CachedSessionfrom mwmbl.indexer.index import tokenize_documentfrom mwmbl.tinysearchengine.indexer import Document, TinyIndex@@ -81,3 +84,6 @@ def validate_domain(domain_or_url: str):if domain is None or VALID_DOMAIN_REGEX.fullmatch(domain) is None:raise ValidationError("Invalid domain: %(domain)s", params={"domain": domain_or_url})++def request_cache(expire_after: timedelta) -> CachedSession:+ return CachedSession(expire_after=expire_after, backend="sqlite", cache_name=settings.REQUEST_CACHE_PATH)