Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Cache request responses in a common place

Daoud Clarked8cd13a95051
diff --git a/mwmbl/indexer/blacklist.py b/mwmbl/indexer/blacklist.py
index e4f17f4..eb7f1eb 100644
--- a/mwmbl/indexer/blacklist.py
+++ b/mwmbl/indexer/blacklist.py
@@ -1,13 +1,12 @@
from datetime import timedelta
-from requests_cache import CachedSession
-
from mwmbl.hn_top_domains_filtered import DOMAINS
from mwmbl.settings import BLACKLIST_DOMAINS_URL, EXCLUDED_DOMAINS, DOMAIN_BLACKLIST_REGEX
+from mwmbl.utils import request_cache
def get_blacklist_domains() -> set[str]:
- with CachedSession(expire_after=timedelta(days=1)) as session:
+ with request_cache(expire_after=timedelta(days=1)) as session:
response = session.get(BLACKLIST_DOMAINS_URL)
return set(response.text.split())
diff --git a/mwmbl/settings_dev.py b/mwmbl/settings_dev.py
index 1301e18..bc67f0a 100644
--- a/mwmbl/settings_dev.py
+++ b/mwmbl/settings_dev.py
@@ -36,3 +36,5 @@ NUM_URLS_IN_BLOOM_FILTER = 100_000
DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")
NUM_DOMAINS_IN_BLOOM_FILTER = 100_000
+
+REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache.sqlite"
diff --git a/mwmbl/settings_prod.py b/mwmbl/settings_prod.py
index 445e43b..9fad002 100644
--- a/mwmbl/settings_prod.py
+++ b/mwmbl/settings_prod.py
@@ -41,3 +41,5 @@ NUM_URLS_IN_BLOOM_FILTER = 200_000_000
DOMAIN_LINKS_BLOOM_FILTER_PATH = str(Path(DATA_PATH) / "links_{domain_group}.bloom")
NUM_DOMAINS_IN_BLOOM_FILTER = 100_000_000
+
+REQUEST_CACHE_PATH = f"{DATA_PATH}/request_cache.sqlite"
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index e9a1deb..c35baef 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -6,17 +6,14 @@ from abc import abstractmethod
from datetime import timedelta
from logging import getLogger
from operator import itemgetter
-from typing import Optional
from urllib.parse import urlparse
-import requests
-from requests_cache import CachedSession
-
-from mwmbl.format import format_result_with_pattern, get_query_regex
-from mwmbl.tokenizer import tokenize, get_bigrams
-from mwmbl.tinysearchengine.completer import Completer
+from mwmbl.format import get_query_regex
from mwmbl.hn_top_domains_filtered import DOMAINS
+from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document, DocumentState
+from mwmbl.tokenizer import tokenize, get_bigrams
+from mwmbl.utils import request_cache
logger = getLogger(__name__)
@@ -271,12 +268,13 @@ class HeuristicAndWikiRanker(HeuristicRanker):
max_wiki_results = 5
def search(self, s: str, additional_results: list[Document]) -> list[Document]:
- with CachedSession(expire_after=timedelta(weeks=12)) as session:
- escaped_query = urllib.parse.quote(s, safe='')
+ escaped_query = urllib.parse.quote(s, safe='')
+ with request_cache(timedelta(weeks=10)) as session:
wiki_response = session.get(WIKI_SEARCH_API_URL.format(query=escaped_query)).json()
- wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
- 0.0, s, state=DocumentState.FROM_WIKI)
- for result in wiki_response['query']['search'][:self.max_wiki_results]]
+
+ wiki_results = [Document(result['title'], get_wiki_url(result['title']), clean_html(result['snippet']),
+ 0.0, s, state=DocumentState.FROM_WIKI)
+ for result in wiki_response['query']['search'][:self.max_wiki_results]]
return super().search(s, wiki_results + additional_results)
diff --git a/mwmbl/utils.py b/mwmbl/utils.py
index c797553..4554ed5 100644
--- a/mwmbl/utils.py
+++ b/mwmbl/utils.py
@@ -1,8 +1,11 @@
import re
from dataclasses import dataclass
+from datetime import timedelta
from typing import Sequence
+from django.conf import settings
from django.core.exceptions import ValidationError
+from requests_cache import CachedSession
from mwmbl.indexer.index import tokenize_document
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
@@ -81,3 +84,6 @@ def validate_domain(domain_or_url: str):
if domain is None or VALID_DOMAIN_REGEX.fullmatch(domain) is None:
raise ValidationError("Invalid domain: %(domain)s", params={"domain": domain_or_url})
+
+def request_cache(expire_after: timedelta) -> CachedSession:
+ return CachedSession(expire_after=expire_after, backend="sqlite", cache_name=settings.REQUEST_CACHE_PATH)