Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
10.4 KB
Raw Blame
"""The job queue must run in exactly one process.

src/app.py executes run_startup_tasks() at module scope, so every process that
imports it -- the entrypoint's schema check, the admin-user script, and each
gunicorn worker -- used to start its own job-queue workers and run its own
orphan recovery.

recover_orphaned_jobs() resets every 'processing' job to 'queued' with no way
to tell a crash-orphaned job from one a live sibling is transcribing right now.
So a second process starting up un-claimed in-flight work, a worker re-claimed
it, and the same audio went to the ASR service twice at once -- N times for N
processes. That is why the deployment guidance was "never run more than one
gunicorn worker".

These tests pin the election that fixes it.
"""

import multiprocessing as mp
import os

import pytest
from sqlalchemy import create_engine

from src.app import app
from src.database import db
from src.models import ProcessingJob, Recording, User
from src.services.job_queue import job_queue
from src.utils.database import acquire_singleton_lock


# Force 'fork' explicitly: the child must inherit the module state it is
# testing, and the default start method varies by platform.
try:
    _MP = mp.get_context("fork")
except ValueError:  # pragma: no cover - non-POSIX
    pytest.skip("requires fork", allow_module_level=True)


@pytest.fixture
def sqlite_url(tmp_path):
    """A private database file, so these tests elect leaders in isolation."""
    return f"sqlite:///{tmp_path / 'elect.db'}"


def _contend(url, name, barrier, results):
    """Child process: try to win the election, report whether it did."""
    engine = create_engine(url)
    if barrier is not None:
        barrier.wait(timeout=30)
    results.put(acquire_singleton_lock(engine, name))
    if barrier is not None:
        # Election is exclusive while the owner lives. Keep every contender
        # alive until all peers have attempted the lock; an early exit would
        # release it and legitimately let a later peer become the next owner.
        barrier.wait(timeout=30)


def _run_contenders(url, name, count):
    barrier = _MP.Barrier(count)
    results = _MP.Queue()
    procs = [
        _MP.Process(target=_contend, args=(url, name, barrier, results))
        for _ in range(count)
    ]
    try:
        for p in procs:
            p.start()
        for p in procs:
            p.join(timeout=30)
        assert all(p.exitcode==0 for p in procs), 'Election contender failed or did not finish'
        return [results.get(timeout=30) for _ in range(count)]
    finally:
        for p in procs:
            if p.is_alive():
                p.terminate()
                p.join(timeout=5)


# ---------------------------------------------------------------------------
# the election itself
# ---------------------------------------------------------------------------

def test_exactly_one_process_wins_the_election(sqlite_url):
    """The property the whole fix rests on."""
    url = sqlite_url
    won = _run_contenders(url, "speakr.test_owner", count=5)
    assert sum(won) == 1, f"expected exactly one owner, got {sum(won)} of 5"


def test_a_different_name_elects_a_separate_owner(sqlite_url):
    """Responsibilities are independent; one owner must not block another."""
    url = sqlite_url
    assert sum(_run_contenders(url, "speakr.test_a", count=3)) == 1
    assert sum(_run_contenders(url, "speakr.test_b", count=3)) == 1


def test_leadership_is_released_when_the_holder_dies(sqlite_url):
    """Leadership must survive a crash, not outlive it.

    The lock is never explicitly released, so the only thing that frees it is
    process death. If that did not work, one killed gunicorn worker would stop
    all background processing until the container was restarted.
    """
    url = sqlite_url
    name = "speakr.test_recovery"

    # A child takes the lock and exits, holding it for its whole life.
    assert _run_contenders(url, name, count=1) == [True]

    # With the holder gone, the next process must be able to win.
    assert _run_contenders(url, name, count=1) == [True]


def test_asking_twice_in_one_process_is_stable(sqlite_url):
    url = sqlite_url
    engine = create_engine(url)
    first = acquire_singleton_lock(engine, "speakr.test_idempotent")
    second = acquire_singleton_lock(engine, "speakr.test_idempotent")
    assert first is True
    assert second is first


# ---------------------------------------------------------------------------
# what the election gates
# ---------------------------------------------------------------------------

@pytest.fixture
def unowned_queue():
    """The queue as a non-owner process sees it.

    Importing src.app runs the election, and in a test process there is no
    competition, so the queue arrives here already owned and already running.
    Both flags have to be reset: start() returns early on `_running` before it
    ever consults ownership, so leaving it set would make these tests pass for
    the wrong reason.
    """
    previous_owner = job_queue._is_owner
    previous_running = job_queue._running
    job_queue._is_owner = False
    job_queue._running = False
    job_queue._transcription_workers = []
    job_queue._summary_workers = []
    yield job_queue
    job_queue._is_owner = previous_owner
    job_queue._running = previous_running


def test_a_non_owner_starts_no_workers(unowned_queue):
    unowned_queue.start()
    assert unowned_queue._running is False
    assert unowned_queue._transcription_workers == []
    assert unowned_queue._summary_workers == []


def test_an_owner_starts_workers(unowned_queue):
    unowned_queue.mark_as_owner()
    unowned_queue.start()
    assert unowned_queue._running is True
    unowned_queue.stop()


def test_enqueue_does_not_make_a_non_owner_start_working(unowned_queue):
    """Close the back door in enqueue().

    enqueue() auto-starts workers when the queue is idle. Without the guard in
    start(), any web worker that accepted an upload would promote itself into a
    transcription process -- reintroducing the duplicate submissions from a
    different direction. The job must still be enqueued: a non-owner accepts
    uploads, it just does not transcribe them.
    """
    unowned_queue.init_app(app)

    with app.app_context():
        user = User(username=f"sing_{os.getpid()}"[:20],
                    email=f"sing_{os.getpid()}@example.com")
        if hasattr(user, "password"):
            user.password = "unused"
        db.session.add(user)
        db.session.commit()

        rec = Recording(user_id=user.id, title="singleton test",
                        audio_path="/tmp/singleton_test.mp3",
                        original_filename="singleton_test.mp3",
                        status="PENDING")
        db.session.add(rec)
        db.session.commit()

        try:
            job_id = unowned_queue.enqueue(user.id, rec.id, "transcribe")

            # The work was accepted...
            assert db.session.get(ProcessingJob, job_id) is not None
            # ...but this process did not appoint itself to do it.
            assert unowned_queue._running is False
            assert unowned_queue._transcription_workers == []
        finally:
            for job in ProcessingJob.query.filter_by(recording_id=rec.id).all():
                db.session.delete(job)
            db.session.commit()
            db.session.delete(rec)
            db.session.delete(user)
            db.session.commit()


# ---------------------------------------------------------------------------
# how the election fails
# ---------------------------------------------------------------------------

def test_an_election_error_leaves_this_process_in_charge(monkeypatch, sqlite_url):
    """Fail open: the worst case must be today's behaviour, never a dead queue.

    If the lock cannot be taken at all, refusing ownership would leave an app
    that looks healthy, accepts uploads and never transcribes them. Every
    process taking ownership instead degrades to the pre-election behaviour,
    which processes work, at the cost of the duplicates this fix removes.
    """
    import src.utils.database as database

    def broken_flock(*args, **kwargs):
        raise OSError("simulated: lock directory not writable")

    monkeypatch.setattr(database.fcntl, "flock", broken_flock)
    engine = create_engine(sqlite_url)
    assert acquire_singleton_lock(engine, "speakr.test_fail_open") is True


def test_the_election_never_opens_a_database_connection(tmp_path):
    """Leadership must not depend on a connection nobody watches.

    A session-level advisory lock vanishes when its connection does -- server
    restart, pooler, idle-timeout -- while the owner's local flag says it still
    owns the queue. The next process then wins a second election and the
    duplicate submissions are back. So the election must not use the database
    at all, whichever backend it is.
    """

    class _EngineThatRefusesConnections:
        name = "postgresql"
        url = f"postgresql://probe@nowhere/{tmp_path.name}-{os.getpid()}"

        def connect(self):
            raise AssertionError("the election opened a database connection")

    assert acquire_singleton_lock(
        _EngineThatRefusesConnections(), "speakr.test_no_connection"
    ) is True


def test_the_lock_file_does_not_follow_a_planted_symlink(tmp_path):
    """The lock path is predictable and lives in the shared temp dir. On a
    bare-metal host another local user could plant a symlink there pointing
    at, say, the SQLite database; a plain open(..., 'w') would follow it and
    truncate the target on the next startup. O_NOFOLLOW refuses the symlink."""
    import hashlib, os, tempfile
    from unittest.mock import patch
    from src.utils import database as db_utils

    class _Engine:
        url = 'sqlite:///symlink-probe'
    name = 'speakr.test_symlink'
    digest = hashlib.sha256(f'{_Engine.url}|{name}'.encode()).hexdigest()[:16]

    victim = tmp_path / 'precious.db'
    victim.write_bytes(b'do not truncate me')
    lock_path = tmp_path / f'speakr_singleton_{digest}.lock'
    os.symlink(victim, lock_path)

    with patch.object(tempfile, 'gettempdir', return_value=str(tmp_path)):
        db_utils._held_singleton_locks.pop(name, None)
        # Failing open is the documented behaviour when the lock cannot be
        # taken; what matters here is that the victim file is untouched.
        acquire_singleton_lock(_Engine(), name)

    assert victim.read_bytes() == b'do not truncate me', 'the symlink was followed and the target truncated'