Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Cache curated domains

Daoud Clarkeaf21f07bebb0
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 700ffc3..9105886 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -3,6 +3,7 @@ import pickle
from pathlib import Path
from django.conf import settings
+from django_cache_decorator import django_cache_decorator
from redis import Redis
from mwmbl.indexer.batch_cache import BatchCache
@@ -13,6 +14,7 @@ from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.ltr_rank import LTRRanker
+@django_cache_decorator(time=300)
def get_curated_domains() -> set[str]:
curated_domains = set(DomainSubmission.objects.filter(status="APPROVED").values_list('name', flat=True))
return curated_domains
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index c12584b..547ec7b 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -9,6 +9,7 @@ https://docs.djangoproject.com/en/4.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.2/ref/settings/
"""
+import os
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -177,3 +178,12 @@ FOOTER_LINKS = [
BATCH_DIR_NAME = 'batches'
DATA_UPLOAD_MAX_NUMBER_FIELDS = None
+
+
+CACHES = {
+ "default": {
+ "BACKEND": "django.core.cache.backends.redis.RedisCache",
+ "LOCATION": os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"),
+ "KEY_PREFIX": "c_",
+ }
+}
diff --git a/poetry.lock b/poetry.lock
index d3c51b9..0f4697a 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -563,6 +563,16 @@ requests-oauthlib = ">=0.3.0"
mfa = ["qrcode (>=7.0.0)"]
saml = ["python3-saml (>=1.15.0,<2.0.0)"]
+[[package]]
+name = "django-cache-decorator"
+version = "0.4.1"
+description = "Easily add caching to functions within a django project."
+optional = false
+python-versions = "*"
+files = [
+ {file = "django-cache-decorator-0.4.1.tar.gz", hash = "sha256:cbaf919b57b0095f4a06a1b356a339cc8c158eae65c6229a1255787c1510b3cc"},
+]
+
[[package]]
name = "django-htmx"
version = "1.18.0"
@@ -3010,4 +3020,4 @@ indexer = ["Levenshtein", "beautifulsoup4", "idna", "langdetect", "pyarrow", "py
[metadata]
lock-version = "2.0"
python-versions = ">=3.10,<3.11"
-content-hash = "d44a4f54a5a5ee4fd70ce03c11bd761dc2e8af94de206fd0423dc1fb128f0783"
+content-hash = "e97ead7ee7a04babb13f57ac86702f7c6f20bb51ae2b79604f0f06df3916182e"
diff --git a/pyproject.toml b/pyproject.toml
index 0e0c107..144ea99 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -48,6 +48,7 @@ objgraph = "^3.6.1"
psutil = "^6.0.0"
xgboost = "^2.1.0"
pydistinct = "^0.6.4"
+django-cache-decorator = "^0.4.1"
[tool.poetry.extras]
indexer = [