Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

Run the background job queue in exactly one process

jagd70095dc77bea356
diff --git a/src/config/startup.py b/src/config/startup.py
index 262cf1f6..2f9c6c9b 100644
--- a/src/config/startup.py
+++ b/src/config/startup.py
@@ -11,6 +11,9 @@ from flask import current_app
ENABLE_AUTO_DELETION = os.environ.get('ENABLE_AUTO_DELETION', 'false').lower() == 'true'
GLOBAL_RETENTION_DAYS = int(os.environ.get('GLOBAL_RETENTION_DAYS', '0'))
+# Cross-process election that decides which process runs the job queue
+JOB_QUEUE_OWNER_LOCK = 'speakr.job_queue_owner'
+
def initialize_file_monitor(app):
"""Initialize file monitor after app is fully loaded to avoid circular imports."""
@@ -109,13 +112,24 @@ def initialize_file_exporter(app):
def initialize_job_queue(app):
- """Initialize and start the background job queue with orphan recovery."""
+ """Initialize the background job queue; only the elected owner process recovers orphans and runs workers."""
try:
+ from src.database import db
from src.services.job_queue import job_queue
+ from src.utils.database import acquire_singleton_lock
# Initialize job queue with app context
job_queue.init_app(app)
+ # Only the elected owner recovers orphans and runs workers; the rest just enqueue
+ if not acquire_singleton_lock(db.engine, JOB_QUEUE_OWNER_LOCK, app.logger):
+ app.logger.info(
+ "Job queue owned by another process; this one will enqueue only"
+ )
+ return
+
+ job_queue.mark_as_owner()
+
# Recover any jobs that were processing when the app crashed
job_queue.recover_orphaned_jobs()
diff --git a/src/services/job_queue.py b/src/services/job_queue.py
index f18d9d51..5b618a80 100644
--- a/src/services/job_queue.py
+++ b/src/services/job_queue.py
@@ -73,6 +73,8 @@ class FairJobQueue:
self._last_user_id_summary = None
# Lock for claiming jobs (SQLite doesn't support row-level locking)
self._claim_lock = threading.Lock()
+ # Whether this process won the queue-owner election (see startup.initialize_job_queue)
+ self._is_owner = False
self._initialized = True
logger.info(f"FairJobQueue initialized: {TRANSCRIPTION_WORKERS} transcription workers, {SUMMARY_WORKERS} summary workers")
@@ -90,11 +92,22 @@ class FairJobQueue:
else:
yield
+ def mark_as_owner(self):
+ """Mark this process as the elected owner of the queue."""
+ self._is_owner = True
+
def start(self):
"""Start the worker threads for both queues."""
if self._running:
return
+ # Only the elected owner runs workers; this also covers the auto-start in enqueue()
+ if not self._is_owner:
+ logger.debug(
+ "Not the job queue owner; no worker threads started in this process"
+ )
+ return
+
self._running = True
# Start transcription workers
@@ -640,7 +653,7 @@ class FairJobQueue:
db.session.commit()
- # Auto-start workers if not running
+ # Auto-start workers if not running (no-op unless this process owns the queue)
if not self._running:
self.start()
diff --git a/src/utils/database.py b/src/utils/database.py
index 4fcf0aa2..490d38d0 100644
--- a/src/utils/database.py
+++ b/src/utils/database.py
@@ -107,6 +107,59 @@ def migration_lock(engine, logger=None, timeout=120):
pass
+# Singleton locks are held for the process lifetime; keeping the file handle
+# referenced here is what keeps them held.
+_held_singleton_locks = {}
+
+
+def acquire_singleton_lock(engine, name, logger=None):
+ """Elect exactly one process on this host to own a named responsibility, for its lifetime.
+
+ The counterpart of `migration_lock`: that serialises work every process must
+ do, this picks the one process that should do it at all, such as running the
+ job-queue workers. It does not wait (a loser returns False at once) and it
+ never releases: the holder keeps the lock until the process exits, so
+ leadership ends on process death, crash or SIGKILL included, and the
+ replacement process wins the next election.
+
+ A file lock is used for every backend: a session-level advisory lock lives
+ on a database connection nobody watches, and when that connection dies the
+ lock is gone while the owner still believes it holds it. Returns True when
+ this process holds `name`; asking again returns the original answer. If the
+ lock cannot be taken at all this process takes ownership anyway, failing
+ open like `migration_lock`, so the worst case is the pre-election behaviour
+ rather than an app that transcribes nothing.
+ """
+ if name in _held_singleton_locks:
+ return _held_singleton_locks[name][0]
+
+ # One lock file per (database, name) so unrelated databases elect independently
+ digest = hashlib.sha256(f'{engine.url}|{name}'.encode()).hexdigest()[:16]
+ lock_path = os.path.join(tempfile.gettempdir(), f'speakr_singleton_{digest}.lock')
+ handle = None
+ try:
+ handle = open(lock_path, 'w')
+ fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+ acquired = True
+ except BlockingIOError:
+ acquired = False
+ except Exception as e:
+ # Fail open: every process working (today's behaviour) beats none working
+ if logger:
+ logger.error("Could not run the '%s' election, taking ownership anyway: %s", name, e)
+ acquired = True
+ if not acquired and handle is not None:
+ handle.close()
+ handle = None
+
+ # Keep the file handle referenced so GC cannot release the lock
+ _held_singleton_locks[name] = (acquired, handle)
+
+ if logger:
+ logger.info("Singleton '%s': this process %s", name, "holds it" if acquired else "does not hold it")
+ return acquired
+
+
def ensure_migration_ledger(engine):
"""Create the table recording which one-shot migrations have already run."""
with engine.connect() as conn:
diff --git a/tests/test_job_queue_singleton.py b/tests/test_job_queue_singleton.py
new file mode 100644
index 00000000..92887543
--- /dev/null
+++ b/tests/test_job_queue_singleton.py
@@ -0,0 +1,223 @@
+"""The job queue must run in exactly one process.
+
+src/app.py executes run_startup_tasks() at module scope, so every process that
+imports it -- the entrypoint's schema check, the admin-user script, and each
+gunicorn worker -- used to start its own job-queue workers and run its own
+orphan recovery.
+
+recover_orphaned_jobs() resets every 'processing' job to 'queued' with no way
+to tell a crash-orphaned job from one a live sibling is transcribing right now.
+So a second process starting up un-claimed in-flight work, a worker re-claimed
+it, and the same audio went to the ASR service twice at once -- N times for N
+processes. That is why the deployment guidance was "never run more than one
+gunicorn worker".
+
+These tests pin the election that fixes it.
+"""
+
+import multiprocessing as mp
+import os
+
+import pytest
+from sqlalchemy import create_engine
+
+from src.app import app
+from src.database import db
+from src.models import ProcessingJob, Recording, User
+from src.services.job_queue import job_queue
+from src.utils.database import acquire_singleton_lock
+
+
+# Force 'fork' explicitly: the child must inherit the module state it is
+# testing, and the default start method varies by platform.
+try:
+ _MP = mp.get_context("fork")
+except ValueError: # pragma: no cover - non-POSIX
+ pytest.skip("requires fork", allow_module_level=True)
+
+
+def sqlite_url(tmp_path):
+ """A private database file, so these tests elect leaders in isolation."""
+ return f"sqlite:///{tmp_path / 'elect.db'}"
+
+
+def _contend(url, name, barrier, results):
+ """Child process: try to win the election, report whether it did."""
+ engine = create_engine(url)
+ if barrier is not None:
+ barrier.wait()
+ results.put(acquire_singleton_lock(engine, name))
+
+
+def _run_contenders(url, name, count):
+ barrier = _MP.Barrier(count)
+ results = _MP.Queue()
+ procs = [
+ _MP.Process(target=_contend, args=(url, name, barrier, results))
+ for _ in range(count)
+ ]
+ for p in procs:
+ p.start()
+ for p in procs:
+ p.join(timeout=30)
+ return [results.get() for _ in range(count)]
+
+
+# ---------------------------------------------------------------------------
+# the election itself
+# ---------------------------------------------------------------------------
+
+def test_exactly_one_process_wins_the_election(sqlite_url):
+ """The property the whole fix rests on."""
+ url = sqlite_url
+ won = _run_contenders(url, "speakr.test_owner", count=5)
+ assert sum(won) == 1, f"expected exactly one owner, got {sum(won)} of 5"
+
+
+def test_a_different_name_elects_a_separate_owner(sqlite_url):
+ """Responsibilities are independent; one owner must not block another."""
+ url = sqlite_url
+ assert sum(_run_contenders(url, "speakr.test_a", count=3)) == 1
+ assert sum(_run_contenders(url, "speakr.test_b", count=3)) == 1
+
+
+def test_leadership_is_released_when_the_holder_dies(sqlite_url):
+ """Leadership must survive a crash, not outlive it.
+
+ The lock is never explicitly released, so the only thing that frees it is
+ process death. If that did not work, one killed gunicorn worker would stop
+ all background processing until the container was restarted.
+ """
+ url = sqlite_url
+ name = "speakr.test_recovery"
+
+ # A child takes the lock and exits, holding it for its whole life.
+ assert _run_contenders(url, name, count=1) == [True]
+
+ # With the holder gone, the next process must be able to win.
+ assert _run_contenders(url, name, count=1) == [True]
+
+
+def test_asking_twice_in_one_process_is_stable(sqlite_url):
+ url = sqlite_url
+ engine = create_engine(url)
+ first = acquire_singleton_lock(engine, "speakr.test_idempotent")
+ second = acquire_singleton_lock(engine, "speakr.test_idempotent")
+ assert first is True
+ assert second is first
+
+
+# ---------------------------------------------------------------------------
+# what the election gates
+# ---------------------------------------------------------------------------
+
+def unowned_queue():
+ """The queue as a non-owner process sees it."""
+ previous = job_queue._is_owner
+ job_queue._is_owner = False
+ yield job_queue
+ job_queue._is_owner = previous
+ job_queue._running = False
+
+
+def test_a_non_owner_starts_no_workers(unowned_queue):
+ unowned_queue.start()
+ assert unowned_queue._running is False
+ assert unowned_queue._transcription_workers == []
+ assert unowned_queue._summary_workers == []
+
+
+def test_an_owner_starts_workers(unowned_queue):
+ unowned_queue.mark_as_owner()
+ unowned_queue.start()
+ assert unowned_queue._running is True
+ unowned_queue.stop()
+
+
+def test_enqueue_does_not_make_a_non_owner_start_working(unowned_queue):
+ """Close the back door in enqueue().
+
+ enqueue() auto-starts workers when the queue is idle. Without the guard in
+ start(), any web worker that accepted an upload would promote itself into a
+ transcription process -- reintroducing the duplicate submissions from a
+ different direction. The job must still be enqueued: a non-owner accepts
+ uploads, it just does not transcribe them.
+ """
+ unowned_queue.init_app(app)
+
+ with app.app_context():
+ user = User(username=f"sing_{os.getpid()}"[:20],
+ email=f"sing_{os.getpid()}@example.com")
+ if hasattr(user, "password"):
+ user.password = "unused"
+ db.session.add(user)
+ db.session.commit()
+
+ rec = Recording(user_id=user.id, title="singleton test",
+ audio_path="/tmp/singleton_test.mp3",
+ original_filename="singleton_test.mp3",
+ status="PENDING")
+ db.session.add(rec)
+ db.session.commit()
+
+ try:
+ job_id = unowned_queue.enqueue(user.id, rec.id, "transcribe")
+
+ # The work was accepted...
+ assert db.session.get(ProcessingJob, job_id) is not None
+ # ...but this process did not appoint itself to do it.
+ assert unowned_queue._running is False
+ assert unowned_queue._transcription_workers == []
+ finally:
+ for job in ProcessingJob.query.filter_by(recording_id=rec.id).all():
+ db.session.delete(job)
+ db.session.commit()
+ db.session.delete(rec)
+ db.session.delete(user)
+ db.session.commit()
+
+
+# ---------------------------------------------------------------------------
+# how the election fails
+# ---------------------------------------------------------------------------
+
+def test_an_election_error_leaves_this_process_in_charge(monkeypatch, sqlite_url):
+ """Fail open: the worst case must be today's behaviour, never a dead queue.
+
+ If the lock cannot be taken at all, refusing ownership would leave an app
+ that looks healthy, accepts uploads and never transcribes them. Every
+ process taking ownership instead degrades to the pre-election behaviour,
+ which processes work, at the cost of the duplicates this fix removes.
+ """
+ import src.utils.database as database
+
+ def broken_flock(*args, **kwargs):
+ raise OSError("simulated: lock directory not writable")
+
+ monkeypatch.setattr(database.fcntl, "flock", broken_flock)
+ engine = create_engine(sqlite_url)
+ assert acquire_singleton_lock(engine, "speakr.test_fail_open") is True
+
+
+def test_the_election_never_opens_a_database_connection():
+ """Leadership must not depend on a connection nobody watches.
+
+ A session-level advisory lock vanishes when its connection does -- server
+ restart, pooler, idle-timeout -- while the owner's local flag says it still
+ owns the queue. The next process then wins a second election and the
+ duplicate submissions are back. So the election must not use the database
+ at all, whichever backend it is.
+ """
+
+ class _EngineThatRefusesConnections:
+ name = "postgresql"
+ url = "postgresql://probe@nowhere/elect"
+
+ def connect(self):
+ raise AssertionError("the election opened a database connection")
+
+ assert acquire_singleton_lock(
+ _EngineThatRefusesConnections(), "speakr.test_no_connection"
+ ) is True