An open source, non-profit web search engine
Use gunicorn for parallelism
Daoud Clarke55ce3b37cfb7
diff --git a/mwmbl/main.py b/mwmbl/main.pyindex 8b086aa..df3d1d9 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -1,8 +1,9 @@+import multiprocessingimport osimport django-import uvicornfrom django.core.management import call_command+from gunicorn.app.base import BaseApplicationfrom redis import Redis@@ -33,7 +34,21 @@ def run():elif mwmbl_app == "count_urls":count_urls_continuously()elif mwmbl_app == "server":- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, lifespan="off", log_level="warning")+ workers = multiprocessing.cpu_count() * 2 + 1++ class GunicornApp(BaseApplication):+ def load_config(self):+ self.cfg.set("bind", "0.0.0.0:5000")+ self.cfg.set("workers", workers)+ self.cfg.set("worker_class", "uvicorn.workers.UvicornWorker")+ self.cfg.set("loglevel", "warning")+ self.cfg.set("timeout", 120)++ def load(self):+ from mwmbl.asgi import application+ return application++ GunicornApp().run()else:raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")diff --git a/pyproject.toml b/pyproject.tomlindex fbe9ca8..b1ae307 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -15,6 +15,7 @@ dependencies = ["zstandard>=0.16.0","mmh3>=3.0.0","uvicorn>=0.16.0",+ "gunicorn>=21.0.0","pyyaml==6.0","boto3==1.35.99","requests>=2.27.1",diff --git a/uv.lock b/uv.lockindex 0dea46b..f09dada 100644--- a/uv.lock+++ b/uv.lock@@ -437,6 +437,18 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/6f/27/b8b057a23f7777177e92d3a602fd866751b6b45014964548997e92e048fd/fakeredis-2.35.1-py3-none-any.whl", hash = "sha256:67d97e11f562b7870e11e5c30cf182270bfb2dd37f6707dba47cc6d91628d1b9", size = 129678, upload-time = "2026-04-12T17:05:56.86Z" },]+[[package]]+name = "gunicorn"+version = "25.3.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "packaging" },+]+sdist = { url = "https://files.pythonhosted.org/packages/c4/f4/e78fa054248fab913e2eab0332c6c2cb07421fca1ce56d8fe43b6aef57a4/gunicorn-25.3.0.tar.gz", hash = "sha256:f74e1b2f9f76f6cd1ca01198968bd2dd65830edc24b6e8e4d78de8320e2fe889", size = 634883, upload-time = "2026-03-27T00:00:26.092Z" }+wheels = [+ { url = "https://files.pythonhosted.org/packages/43/c8/8aaf447698c4d59aa853fd318eed300b5c9e44459f242ab8ead6c9c09792/gunicorn-25.3.0-py3-none-any.whl", hash = "sha256:cacea387dab08cd6776501621c295a904fe8e3b7aae9a1a3cbb26f4e7ed54660", size = 208403, upload-time = "2026-03-27T00:00:27.386Z" },+]+[[package]]name = "h11"version = "0.16.0"@@ -690,6 +702,7 @@ dependencies = [{ name = "django-redis" },{ name = "django-vite" },{ name = "dotenv" },+ { name = "gunicorn" },{ name = "lxml" },{ name = "lxml-html-clean" },{ name = "mmh3" },@@ -753,6 +766,7 @@ requires-dist = [{ name = "django-redis", specifier = ">=5.4.0" },{ name = "django-vite", specifier = ">=2.1.3" },{ name = "dotenv", specifier = ">=0.9.9" },+ { name = "gunicorn", specifier = ">=21.0.0" },{ name = "idna", marker = "extra == 'indexer'", specifier = "==3.3" },{ name = "langdetect", marker = "extra == 'indexer'", specifier = "==1.0.9" },{ name = "levenshtein", marker = "extra == 'indexer'", specifier = "==0.16.0" },