Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Run the background task queue alongside the server

Daoud Clarke2ce514e05e14
diff --git a/mwmbl/main.py b/mwmbl/main.py
index 2144ecc..6329f98 100644
--- a/mwmbl/main.py
+++ b/mwmbl/main.py
@@ -1,11 +1,44 @@
+import logging
import multiprocessing
import os
+from time import sleep
import django
from django.core.management import call_command
from gunicorn.app.base import BaseApplication
from redis import Redis
+logger = logging.getLogger(__name__)
+
+# How long to wait before restarting the task queue if it ever exits. With --duration 0
+# process_tasks loops forever and, on Linux, does not even handle SIGTERM (its
+# SignalManager only binds that on Windows), so returning at all means something went
+# wrong - a database outage propagating out of run_next_task, say. Restarting matters
+# because the alternative is the queue silently stopping, which is how every periodic
+# task came to be months out of date in the first place.
+TASK_QUEUE_RESTART_SECONDS = 10
+
+
+def run_background_tasks():
+ """Run the django-background-tasks queue. Entry point for the child process.
+
+ Started with the 'spawn' method, so this gets a fresh interpreter and has to set
+ Django up itself. That is the point: a forked child would inherit the parent's
+ Postgres and Redis sockets, and two processes reading from one connection corrupts
+ both. Spawning costs a few seconds of startup, once per container.
+ """
+ django.setup()
+
+ # Importing the module is what registers the @background functions - the decorator
+ # runs at import time, and process_tasks' own autodiscover() only looks for
+ # <app>/tasks.py, which mwmbl does not have. test_process_tasks_worker.py guards this.
+ from mwmbl import background # noqa: F401
+
+ while True:
+ call_command("process_tasks")
+ logger.error("Background task queue exited; restarting in %ds", TASK_QUEUE_RESTART_SECONDS)
+ sleep(TASK_QUEUE_RESTART_SECONDS)
+
def run():
django.setup()
@@ -22,6 +55,11 @@ def run():
call_command("migrate")
+ # DEPRECATED: update_urls, update_batches, copy_indexes and count_urls are no longer
+ # deployed. "server" is the only app that runs, and it now also runs the background
+ # task queue (see below). They are kept here rather than deleted because the code
+ # they call is still reachable from the management commands and the standalone
+ # crawler; treat them as unmaintained.
mwmbl_app = os.environ["MWMBL_APP"]
if mwmbl_app == "update_urls":
redis: Redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)
@@ -34,17 +72,25 @@ def run():
elif mwmbl_app == "count_urls":
count_urls_continuously()
elif mwmbl_app == "process_tasks":
- # Runs the django-background-tasks queue. Without a process running this, the
- # @background functions in mwmbl.background only ever get *scheduled* - apps.ready()
- # writes a Task row and nothing executes it - so they sit pending indefinitely,
- # which is exactly what had happened to every periodic task in production.
- #
- # The tasks are registered by the @background decorator at import time, so this
- # relies on `from mwmbl import background` above having run; process_tasks' own
- # autodiscover() only looks for <app>/tasks.py, which mwmbl does not have.
- # test_process_tasks_worker.py guards that.
- call_command("process_tasks")
+ # The task queue on its own, for running it in a container of its own rather than
+ # alongside the server. Not currently deployed - RUN_BACKGROUND_TASKS on the
+ # server app is how it runs - but it is the way to isolate the queue if the tasks
+ # ever outgrow sharing a container with the web workers.
+ run_background_tasks()
elif mwmbl_app == "server":
+ if settings.RUN_BACKGROUND_TASKS:
+ # Before gunicorn forks, so there is one queue process per container rather
+ # than one per worker. daemon=True ties its lifetime to this process, so it
+ # goes away when the container stops.
+ #
+ # Without this, the @background functions in mwmbl.background only ever get
+ # *scheduled*: apps.ready() writes a Task row and nothing executes it. Every
+ # periodic task in production had been pending, unattempted, for months.
+ process = multiprocessing.get_context("spawn").Process(
+ target=run_background_tasks, name="background-tasks", daemon=True)
+ process.start()
+ logger.info("Started the background task queue (pid %d)", process.pid)
+
workers = multiprocessing.cpu_count() * 2 + 1
class GunicornApp(BaseApplication):
diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.py
index 23cea12..30d60ef 100644
--- a/mwmbl/settings_common.py
+++ b/mwmbl/settings_common.py
@@ -199,6 +199,17 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = None
# Gates database initialisation and background task scheduling; False for the crawler and tests.
HAS_DATABASE = True
+# Whether this container runs the django-background-tasks queue (see mwmbl.main). Note the
+# distinction from HAS_DATABASE above: that gates *scheduling* the tasks, this gates
+# *running* them.
+#
+# Opt-in rather than on by default because beta shares its database, index and Redis with
+# production. Exactly one deployment should run the queue, and it should be production:
+# the tasks report billable usage to Polar and write to the index, so a beta container
+# picking up the same rows would do both against production. Turn it on with
+# `dokku config:set <app> RUN_BACKGROUND_TASKS=true`.
+RUN_BACKGROUND_TASKS = os.environ.get("RUN_BACKGROUND_TASKS", "false").lower() == "true"
+
SENTRY_DSN = os.environ.get("SENTRY_DSN")
diff --git a/test/test_process_tasks_worker.py b/test/test_process_tasks_worker.py
index df74657..d16c2db 100644
--- a/test/test_process_tasks_worker.py
+++ b/test/test_process_tasks_worker.py
@@ -75,14 +75,105 @@ def recorded_commands(monkeypatch):
return calls
-def test_process_tasks_app_runs_the_queue(recorded_commands, monkeypatch):
- monkeypatch.setenv("MWMBL_APP", "process_tasks")
+class StopLoop(Exception):
+ """Breaks run_background_tasks' otherwise infinite restart loop."""
+
+
+def stop_after_two_runs(monkeypatch, recorded_commands):
+ def fake_sleep(seconds):
+ assert seconds == main.TASK_QUEUE_RESTART_SECONDS
+ if recorded_commands.count("process_tasks") >= 2:
+ raise StopLoop
+ monkeypatch.setattr(main, "sleep", fake_sleep)
+
+
+def test_the_queue_is_restarted_if_it_ever_exits(recorded_commands, stop_after_two_runs):
+ """process_tasks with --duration 0 loops forever and, on Linux, does not even handle
+ SIGTERM - its SignalManager only binds that on Windows. So returning means something
+ went wrong, and the queue has to come back: the failure being guarded against is the
+ tasks silently stopping, which is how they came to be months out of date."""
+ with pytest.raises(StopLoop):
+ main.run_background_tasks()
+
+ assert recorded_commands == ["process_tasks", "process_tasks"]
+
+
+class FakeGunicorn:
+ """Stands in for BaseApplication so the server branch does not start a real server."""
+ runs = 0
+
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def run(self):
+ FakeGunicorn.runs += 1
+
+
+def fake_server(monkeypatch):
+ FakeGunicorn.runs = 0
+ monkeypatch.setattr(main, "BaseApplication", FakeGunicorn)
+
+ spawned = []
+ monkeypatch.setattr(main.multiprocessing, "get_context",
+ lambda method: _FakeContext(method, spawned))
+ monkeypatch.setenv("MWMBL_APP", "server")
+ return spawned
+
+
+class _FakeContext:
+ def __init__(self, method, spawned):
+ self.method = method
+ self.spawned = spawned
+
+ def Process(self, target, name, daemon):
+ self.spawned.append({"method": self.method, "target": target, "name": name, "daemon": daemon})
+ return _FakeProcess()
+
+
+class _FakeProcess:
+ pid = 4242
+
+ def start(self):
+ pass
+
+
+def test_server_starts_the_queue_when_enabled(settings, fake_server, recorded_commands):
+ settings.RUN_BACKGROUND_TASKS = True
main.run()
- assert "process_tasks" in recorded_commands
+ assert len(fake_server) == 1
+ started = fake_server[0]
+ assert started["target"] is main.run_background_tasks
+ # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis
+ # sockets. Daemonic so it dies with the container rather than outliving it.
+ assert started["method"] == "spawn"
+ assert started["daemon"]
+ assert FakeGunicorn.runs == 1
+
+
+def test_server_does_not_start_the_queue_by_default(settings, fake_server, recorded_commands):
+ # Off unless opted in, because beta shares a database and index with production and
+ # only one deployment should be running the tasks.
+ settings.RUN_BACKGROUND_TASKS = False
+
+ main.run()
+
+ assert fake_server == []
+ assert FakeGunicorn.runs == 1
+
+
+def test_process_tasks_app_runs_the_queue(recorded_commands, stop_after_two_runs, monkeypatch):
+ monkeypatch.setenv("MWMBL_APP", "process_tasks")
+
+ with pytest.raises(StopLoop):
+ main.run()
+
# Migrations run first, as they do for every other MWMBL_APP.
- assert recorded_commands.index("migrate") < recorded_commands.index("process_tasks")
+ assert recorded_commands[0] == "migrate"
+ assert "process_tasks" in recorded_commands
def test_unknown_app_still_raises(recorded_commands, monkeypatch):