An open source, non-profit web search engine
Run the background task queue alongside the server
Daoud Clarke2ce514e05e14
diff --git a/mwmbl/main.py b/mwmbl/main.pyindex 2144ecc..6329f98 100644--- a/mwmbl/main.py+++ b/mwmbl/main.py@@ -1,11 +1,44 @@+import loggingimport multiprocessingimport os+from time import sleepimport djangofrom django.core.management import call_commandfrom gunicorn.app.base import BaseApplicationfrom redis import Redis+logger = logging.getLogger(__name__)++# How long to wait before restarting the task queue if it ever exits. With --duration 0+# process_tasks loops forever and, on Linux, does not even handle SIGTERM (its+# SignalManager only binds that on Windows), so returning at all means something went+# wrong - a database outage propagating out of run_next_task, say. Restarting matters+# because the alternative is the queue silently stopping, which is how every periodic+# task came to be months out of date in the first place.+TASK_QUEUE_RESTART_SECONDS = 10+++def run_background_tasks():+ """Run the django-background-tasks queue. Entry point for the child process.++ Started with the 'spawn' method, so this gets a fresh interpreter and has to set+ Django up itself. That is the point: a forked child would inherit the parent's+ Postgres and Redis sockets, and two processes reading from one connection corrupts+ both. Spawning costs a few seconds of startup, once per container.+ """+ django.setup()++ # Importing the module is what registers the @background functions - the decorator+ # runs at import time, and process_tasks' own autodiscover() only looks for+ # <app>/tasks.py, which mwmbl does not have. test_process_tasks_worker.py guards this.+ from mwmbl import background # noqa: F401++ while True:+ call_command("process_tasks")+ logger.error("Background task queue exited; restarting in %ds", TASK_QUEUE_RESTART_SECONDS)+ sleep(TASK_QUEUE_RESTART_SECONDS)+def run():django.setup()@@ -22,6 +55,11 @@ def run():call_command("migrate")+ # DEPRECATED: update_urls, update_batches, copy_indexes and count_urls are no longer+ # deployed. "server" is the only app that runs, and it now also runs the background+ # task queue (see below). They are kept here rather than deleted because the code+ # they call is still reachable from the management commands and the standalone+ # crawler; treat them as unmaintained.mwmbl_app = os.environ["MWMBL_APP"]if mwmbl_app == "update_urls":redis: Redis = Redis.from_url(os.environ.get("REDIS_URL", "redis://127.0.0.1:6379"), decode_responses=True)@@ -34,17 +72,25 @@ def run():elif mwmbl_app == "count_urls":count_urls_continuously()elif mwmbl_app == "process_tasks":- # Runs the django-background-tasks queue. Without a process running this, the- # @background functions in mwmbl.background only ever get *scheduled* - apps.ready()- # writes a Task row and nothing executes it - so they sit pending indefinitely,- # which is exactly what had happened to every periodic task in production.- #- # The tasks are registered by the @background decorator at import time, so this- # relies on `from mwmbl import background` above having run; process_tasks' own- # autodiscover() only looks for <app>/tasks.py, which mwmbl does not have.- # test_process_tasks_worker.py guards that.- call_command("process_tasks")+ # The task queue on its own, for running it in a container of its own rather than+ # alongside the server. Not currently deployed - RUN_BACKGROUND_TASKS on the+ # server app is how it runs - but it is the way to isolate the queue if the tasks+ # ever outgrow sharing a container with the web workers.+ run_background_tasks()elif mwmbl_app == "server":+ if settings.RUN_BACKGROUND_TASKS:+ # Before gunicorn forks, so there is one queue process per container rather+ # than one per worker. daemon=True ties its lifetime to this process, so it+ # goes away when the container stops.+ #+ # Without this, the @background functions in mwmbl.background only ever get+ # *scheduled*: apps.ready() writes a Task row and nothing executes it. Every+ # periodic task in production had been pending, unattempted, for months.+ process = multiprocessing.get_context("spawn").Process(+ target=run_background_tasks, name="background-tasks", daemon=True)+ process.start()+ logger.info("Started the background task queue (pid %d)", process.pid)+workers = multiprocessing.cpu_count() * 2 + 1class GunicornApp(BaseApplication):diff --git a/mwmbl/settings_common.py b/mwmbl/settings_common.pyindex 23cea12..30d60ef 100644--- a/mwmbl/settings_common.py+++ b/mwmbl/settings_common.py@@ -199,6 +199,17 @@ DATA_UPLOAD_MAX_NUMBER_FIELDS = None# Gates database initialisation and background task scheduling; False for the crawler and tests.HAS_DATABASE = True+# Whether this container runs the django-background-tasks queue (see mwmbl.main). Note the+# distinction from HAS_DATABASE above: that gates *scheduling* the tasks, this gates+# *running* them.+#+# Opt-in rather than on by default because beta shares its database, index and Redis with+# production. Exactly one deployment should run the queue, and it should be production:+# the tasks report billable usage to Polar and write to the index, so a beta container+# picking up the same rows would do both against production. Turn it on with+# `dokku config:set <app> RUN_BACKGROUND_TASKS=true`.+RUN_BACKGROUND_TASKS = os.environ.get("RUN_BACKGROUND_TASKS", "false").lower() == "true"+SENTRY_DSN = os.environ.get("SENTRY_DSN")diff --git a/test/test_process_tasks_worker.py b/test/test_process_tasks_worker.pyindex df74657..d16c2db 100644--- a/test/test_process_tasks_worker.py+++ b/test/test_process_tasks_worker.py@@ -75,14 +75,105 @@ def recorded_commands(monkeypatch):return calls-def test_process_tasks_app_runs_the_queue(recorded_commands, monkeypatch):- monkeypatch.setenv("MWMBL_APP", "process_tasks")+class StopLoop(Exception):+ """Breaks run_background_tasks' otherwise infinite restart loop."""+++def stop_after_two_runs(monkeypatch, recorded_commands):+ def fake_sleep(seconds):+ assert seconds == main.TASK_QUEUE_RESTART_SECONDS+ if recorded_commands.count("process_tasks") >= 2:+ raise StopLoop+ monkeypatch.setattr(main, "sleep", fake_sleep)+++def test_the_queue_is_restarted_if_it_ever_exits(recorded_commands, stop_after_two_runs):+ """process_tasks with --duration 0 loops forever and, on Linux, does not even handle+ SIGTERM - its SignalManager only binds that on Windows. So returning means something+ went wrong, and the queue has to come back: the failure being guarded against is the+ tasks silently stopping, which is how they came to be months out of date."""+ with pytest.raises(StopLoop):+ main.run_background_tasks()++ assert recorded_commands == ["process_tasks", "process_tasks"]+++class FakeGunicorn:+ """Stands in for BaseApplication so the server branch does not start a real server."""+ runs = 0++ def __init__(self, *args, **kwargs):+ pass++ def run(self):+ FakeGunicorn.runs += 1+++def fake_server(monkeypatch):+ FakeGunicorn.runs = 0+ monkeypatch.setattr(main, "BaseApplication", FakeGunicorn)++ spawned = []+ monkeypatch.setattr(main.multiprocessing, "get_context",+ lambda method: _FakeContext(method, spawned))+ monkeypatch.setenv("MWMBL_APP", "server")+ return spawned+++class _FakeContext:+ def __init__(self, method, spawned):+ self.method = method+ self.spawned = spawned++ def Process(self, target, name, daemon):+ self.spawned.append({"method": self.method, "target": target, "name": name, "daemon": daemon})+ return _FakeProcess()+++class _FakeProcess:+ pid = 4242++ def start(self):+ pass+++def test_server_starts_the_queue_when_enabled(settings, fake_server, recorded_commands):+ settings.RUN_BACKGROUND_TASKS = Truemain.run()- assert "process_tasks" in recorded_commands+ assert len(fake_server) == 1+ started = fake_server[0]+ assert started["target"] is main.run_background_tasks+ # Spawn, not fork: a forked child would inherit the parent's Postgres and Redis+ # sockets. Daemonic so it dies with the container rather than outliving it.+ assert started["method"] == "spawn"+ assert started["daemon"]+ assert FakeGunicorn.runs == 1+++def test_server_does_not_start_the_queue_by_default(settings, fake_server, recorded_commands):+ # Off unless opted in, because beta shares a database and index with production and+ # only one deployment should be running the tasks.+ settings.RUN_BACKGROUND_TASKS = False++ main.run()++ assert fake_server == []+ assert FakeGunicorn.runs == 1+++def test_process_tasks_app_runs_the_queue(recorded_commands, stop_after_two_runs, monkeypatch):+ monkeypatch.setenv("MWMBL_APP", "process_tasks")++ with pytest.raises(StopLoop):+ main.run()+# Migrations run first, as they do for every other MWMBL_APP.- assert recorded_commands.index("migrate") < recorded_commands.index("process_tasks")+ assert recorded_commands[0] == "migrate"+ assert "process_tasks" in recorded_commandsdef test_unknown_app_still_raises(recorded_commands, monkeypatch):