Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Use gunicorn for parallelism

Daoud Clarke55ce3b37cfb7
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 8b086aa..df3d1d9 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -1,8 +1,9 @@
+import multiprocessing
import os
import django
-import uvicorn
from django.core.management import call_command
+from gunicorn.app.base import BaseApplication
from redis import Redis
@@ -33,7 +34,21 @@ def run():
elif mwmbl_app == "count_urls":
count_urls_continuously()
elif mwmbl_app == "server":
- uvicorn.run("mwmbl.asgi:application", host="0.0.0.0", port=5000, lifespan="off", log_level="warning")
+ workers = multiprocessing.cpu_count() * 2 + 1
+
+ class GunicornApp(BaseApplication):
+ def load_config(self):
+ self.cfg.set("bind", "0.0.0.0:5000")
+ self.cfg.set("workers", workers)
+ self.cfg.set("worker_class", "uvicorn.workers.UvicornWorker")
+ self.cfg.set("loglevel", "warning")
+ self.cfg.set("timeout", 120)
+
+ def load(self):
+ from mwmbl.asgi import application
+ return application
+
+ GunicornApp().run()
else:
raise ValueError(f"Unknown MWMBL_APP: {mwmbl_app}")
diff --git a/pyproject.toml b/pyproject.toml
index fbe9ca8..b1ae307 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,6 +15,7 @@ dependencies = [
"zstandard>=0.16.0",
"mmh3>=3.0.0",
"uvicorn>=0.16.0",
+ "gunicorn>=21.0.0",
"pyyaml==6.0",
"boto3==1.35.99",
"requests>=2.27.1",
diff --git a/uv.lock b/uv.lock
index 0dea46b..f09dada 100644
--- a/uv.lock
+++ b/uv.lock
@@ -437,6 +437,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6f/27/b8b057a23f7777177e92d3a602fd866751b6b45014964548997e92e048fd/fakeredis-2.35.1-py3-none-any.whl", hash = "sha256:67d97e11f562b7870e11e5c30cf182270bfb2dd37f6707dba47cc6d91628d1b9", size = 129678, upload-time = "2026-04-12T17:05:56.86Z" },
]
+[[package]]
+name = "gunicorn"
+version = "25.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "packaging" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c4/f4/e78fa054248fab913e2eab0332c6c2cb07421fca1ce56d8fe43b6aef57a4/gunicorn-25.3.0.tar.gz", hash = "sha256:f74e1b2f9f76f6cd1ca01198968bd2dd65830edc24b6e8e4d78de8320e2fe889", size = 634883, upload-time = "2026-03-27T00:00:26.092Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/43/c8/8aaf447698c4d59aa853fd318eed300b5c9e44459f242ab8ead6c9c09792/gunicorn-25.3.0-py3-none-any.whl", hash = "sha256:cacea387dab08cd6776501621c295a904fe8e3b7aae9a1a3cbb26f4e7ed54660", size = 208403, upload-time = "2026-03-27T00:00:27.386Z" },
+]
+
[[package]]
name = "h11"
version = "0.16.0"
@@ -690,6 +702,7 @@ dependencies = [
{ name = "django-redis" },
{ name = "django-vite" },
{ name = "dotenv" },
+ { name = "gunicorn" },
{ name = "lxml" },
{ name = "lxml-html-clean" },
{ name = "mmh3" },
@@ -753,6 +766,7 @@ requires-dist = [
{ name = "django-redis", specifier = ">=5.4.0" },
{ name = "django-vite", specifier = ">=2.1.3" },
{ name = "dotenv", specifier = ">=0.9.9" },
+ { name = "gunicorn", specifier = ">=21.0.0" },
{ name = "idna", marker = "extra == 'indexer'", specifier = "==3.3" },
{ name = "langdetect", marker = "extra == 'indexer'", specifier = "==1.0.9" },
{ name = "levenshtein", marker = "extra == 'indexer'", specifier = "==0.16.0" },