Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
15.8 KB
Raw Blame
"""
Application startup functions.
"""

import os
import time
import threading
from datetime import datetime, timedelta
from flask import current_app
from src.utils.localization import api_error

ENABLE_AUTO_DELETION = os.environ.get('ENABLE_AUTO_DELETION', 'false').lower() == 'true'
GLOBAL_RETENTION_DAYS = int(os.environ.get('GLOBAL_RETENTION_DAYS', '0'))

# Cross-process elections for work that must run in exactly one process.
#
# src/app.py calls run_startup_tasks() at module scope, so EVERY process that
# imports it runs this file: the entrypoint's schema check, the admin-user
# script, and each gunicorn worker (three by default). Anything here that
# starts a thread, watches a directory or sweeps a table therefore ran that
# many times over.
#
# That is not a hypothetical. It is #384: orphan recovery in every booting
# process un-claimed jobs a sibling was mid-way through transcribing, and the
# same audio went to the ASR service once per worker. The process-local
# `global _x_started` guards several of these used are worthless here, because
# each process has its own copy of the global.
#
# The fix is an election per responsibility. Route background work through
# start_single_instance() below rather than calling it directly, so that
# adding a new one forces the question "which process should run this?" to be
# answered rather than skipped.
JOB_QUEUE_OWNER_LOCK = 'speakr.job_queue_owner'
FILE_MONITOR_OWNER_LOCK = 'speakr.file_monitor_owner'
AUTO_DELETION_OWNER_LOCK = 'speakr.auto_deletion_owner'
SESSION_CLEANUP_OWNER_LOCK = 'speakr.session_cleanup_owner'
WEBHOOK_DISPATCHER_OWNER_LOCK = 'speakr.webhook_dispatcher_owner'
VOICE_EMBEDDING_CHECK_OWNER_LOCK = 'speakr.voice_embedding_check_owner'

# Every lock name this module elects on. The test suite asserts that each
# background starter in run_startup_tasks() goes through one of these.
SINGLE_INSTANCE_LOCKS = (
    JOB_QUEUE_OWNER_LOCK,
    FILE_MONITOR_OWNER_LOCK,
    AUTO_DELETION_OWNER_LOCK,
    SESSION_CLEANUP_OWNER_LOCK,
    WEBHOOK_DISPATCHER_OWNER_LOCK,
    VOICE_EMBEDDING_CHECK_OWNER_LOCK,
)


def start_single_instance(app, lock_name, start, description):
    """Run `start` only in the process that wins the election for `lock_name`.

    Losing is the normal outcome for most processes and is logged at debug;
    it is not a failure. The election fails open, so the worst case is the
    pre-election behaviour rather than work that never runs at all.
    """
    from src.database import db
    from src.utils.database import acquire_singleton_lock

    try:
        if not acquire_singleton_lock(db.engine, lock_name, app.logger):
            app.logger.debug("%s runs in another process; skipping here", description)
            return False
    except Exception as e:
        # Never let the election itself cost us the work it guards.
        app.logger.error(
            "Could not run the election for %s, starting it here anyway: %s", description, e)

    try:
        start()
        return True
    except Exception as e:
        app.logger.warning("%s failed to start: %s", description, e)
        return False


def initialize_file_monitor(app):
    """Initialize file monitor after app is fully loaded to avoid circular imports."""
    try:
        # Import here to avoid circular imports
        import src.file_monitor as file_monitor
        file_monitor.start_file_monitor()
        app.logger.info("File monitor initialization completed")
    except Exception as e:
        app.logger.warning(f"File monitor initialization failed: {e}")

def get_file_monitor_functions(app):
    """Get file monitor functions, handling import errors gracefully."""
    try:
        import src.file_monitor as file_monitor
        return file_monitor.start_file_monitor, file_monitor.stop_file_monitor, file_monitor.get_file_monitor_status
    except ImportError as e:
        app.logger.warning(f"File monitor not available: {e}")

        # Create stub functions if file_monitor is not available
        def start_file_monitor():
            pass
        def stop_file_monitor():
            pass
        def get_file_monitor_status():
            return {'running': False, **api_error('file_monitor_unavailable')}

        return start_file_monitor, stop_file_monitor, get_file_monitor_status

# --- Auto-Processing API Endpoints ---
def initialize_auto_deletion_scheduler(app):
    """Initialize the daily auto-deletion scheduler if enabled."""
    from src.services.retention import process_auto_deletion

    if not ENABLE_AUTO_DELETION:
        app.logger.info("Auto-deletion scheduler not started (ENABLE_AUTO_DELETION=false)")
        return

    if GLOBAL_RETENTION_DAYS <= 0:
        app.logger.info("Auto-deletion scheduler not started (GLOBAL_RETENTION_DAYS not set)")
        return

    def run_daily_deletion():
        """Background thread that runs auto-deletion daily at 2 AM."""
        import time
        from datetime import datetime, timedelta

        app.logger.info("Auto-deletion scheduler started - will run daily at 2:00 AM")

        while True:
            try:
                # Calculate time until next 2 AM
                now = datetime.now()
                next_run = now.replace(hour=2, minute=0, second=0, microsecond=0)

                # If it's past 2 AM today, schedule for tomorrow
                if now.hour >= 2:
                    next_run += timedelta(days=1)

                sleep_seconds = (next_run - now).total_seconds()

                app.logger.info(f"Next auto-deletion scheduled for: {next_run.strftime('%Y-%m-%d %H:%M:%S')} (in {sleep_seconds/3600:.1f} hours)")

                # Sleep until next run time
                time.sleep(sleep_seconds)

                # Run auto-deletion
                app.logger.info("Running scheduled auto-deletion...")
                with app.app_context():
                    stats = process_auto_deletion()
                    app.logger.info(f"Scheduled auto-deletion completed: {stats}")

            except Exception as e:
                app.logger.error(f"Error in auto-deletion scheduler: {e}", exc_info=True)
                # Sleep for 1 hour before retrying on error
                time.sleep(3600)

    # Start the scheduler thread
    import threading
    scheduler_thread = threading.Thread(target=run_daily_deletion, daemon=True, name="AutoDeletionScheduler")
    scheduler_thread.start()
    app.logger.info("✅ Auto-deletion scheduler initialized - running daily at 2:00 AM")


def initialize_file_exporter(app):
    """Initialize file exporter after app is fully loaded."""
    try:
        from src.file_exporter import initialize_export_directory, ENABLE_AUTO_EXPORT
        if ENABLE_AUTO_EXPORT:
            initialize_export_directory()
            app.logger.info("✅ Auto-export initialized")
        else:
            app.logger.info("ℹ️  Auto-export: Disabled (set ENABLE_AUTO_EXPORT=true to enable)")
    except Exception as e:
        app.logger.warning(f"File exporter initialization failed: {e}")


def bind_job_queue(app):
    """Make the queue usable in this process. Every process needs this.

    Binding is not the same as owning: a process that lost the election still
    accepts uploads and enqueues them, it just runs no workers.
    """
    from src.services.job_queue import job_queue
    job_queue.init_app(app)


def start_job_queue(app):
    """Recover orphaned jobs and run the workers. Owner process only.

    Recovery is the reason this must not run everywhere: it resets every row
    in `processing` back to `queued` with no way to tell a job abandoned by a
    crash from one a sibling process is transcribing right now (#384).
    """
    from src.services.job_queue import job_queue

    job_queue.mark_as_owner()
    job_queue.recover_orphaned_jobs()
    job_queue.start()

    status = job_queue.get_queue_status()
    t_queue = status['transcription_queue']
    s_queue = status['summary_queue']
    app.logger.info(
        f"Job queues started: "
        f"transcription ({t_queue['workers']} workers, {t_queue['queued']} queued), "
        f"summary ({s_queue['workers']} workers, {s_queue['queued']} queued)"
    )


def initialize_job_queue(app):
    """Backwards-compatible wrapper: bind, elect, then start if we won."""
    bind_job_queue(app)
    start_single_instance(app, JOB_QUEUE_OWNER_LOCK,
                          lambda: start_job_queue(app), 'job queue')


# Guards a SECOND START INSIDE ONE PROCESS (Flask --reload, gunicorn --reload,
# container restart-on-change). It cannot guard against other processes,
# because each has its own copy of it; start_single_instance() does that.
_cleanup_thread_started = False


def initialize_recording_session_cleanup(app):
    """Sweep expired recording sessions on a background thread.

    Issue #287 (c)(d): in-progress recording sessions accumulate chunk
    files in ``UPLOAD_FOLDER/_sessions/<id>/``. The session row's
    ``last_seen_at`` is updated on every chunk POST; sessions that go
    silent for longer than ``RECORDING_SESSION_TTL_HOURS`` (default 24)
    are reaped here and their on-disk dirs removed.

    Runs on the same pattern as the other schedulers in this module
    (daemon thread with a sleep loop), with a configurable interval via
    ``RECORDING_SESSION_CLEANUP_INTERVAL_SECONDS`` (default 3600 = hourly).
    """
    import os
    import threading
    import time

    global _cleanup_thread_started
    if _cleanup_thread_started:
        # Reload or duplicate init; the existing daemon is fine.
        app.logger.info("Recording-session cleanup thread already running; skipping duplicate init")
        return

    interval = int(os.environ.get('RECORDING_SESSION_CLEANUP_INTERVAL_SECONDS', '3600'))
    if interval <= 0:
        app.logger.info("Recording-session cleanup disabled (interval <= 0)")
        return

    def _loop():
        from src.api.recording_sessions import cleanup_expired_sessions
        app.logger.info(f"Recording-session cleanup scheduler started (interval={interval}s)")
        # First sweep happens after the interval, not at boot, to avoid
        # contention with other startup tasks.
        while True:
            try:
                time.sleep(interval)
                reaped = cleanup_expired_sessions(app=app)
                if reaped:
                    app.logger.info(f"Recording-session cleanup reaped {reaped} session(s)")
                # Parts of an upload join that never completed.
                with app.app_context():
                    from src.services.upload_join import cleanup_stale_parts
                    removed = cleanup_stale_parts()
                if removed:
                    app.logger.info(f"Upload-join cleanup removed {removed} stale part(s)")
                # Changes-feed tombstones past RECORDING_TOMBSTONE_DAYS.
                with app.app_context():
                    from src.services.recording_changes import prune_tombstones
                    pruned = prune_tombstones()
                if pruned:
                    app.logger.info(f"Pruned {pruned} recording tombstone(s)")
            except Exception as e:
                app.logger.error(f"Recording-session cleanup error: {e}", exc_info=True)
                # Don't tight-loop on failure
                time.sleep(60)

    thread = threading.Thread(target=_loop, daemon=True, name="RecordingSessionCleanup")
    thread.start()
    _cleanup_thread_started = True
    app.logger.info("✅ Recording-session cleanup scheduler initialized")


def initialize_voice_embedding_check(app):
    """Check the voice embedding backend on a background thread (#380).

    Off the boot path on purpose: it makes a real transcription request, and a
    slow or unreachable ASR service must not delay the app coming up. A
    failure to check is logged and leaves the previous verdict alone; a
    backend that is down is not a backend that changed.
    """
    import threading

    if os.environ.get('DISABLE_VOICE_EMBEDDING_CHECK', 'false').lower() == 'true':
        app.logger.info("Voice embedding check disabled (DISABLE_VOICE_EMBEDDING_CHECK=true)")
        return

    def _run():
        from src.services.voice_embedding_check import check_voice_embeddings
        try:
            with app.app_context():
                result = check_voice_embeddings(app)
                app.config['VOICE_EMBEDDING_STATUS'] = result
        except Exception as e:
            app.logger.warning(f"Voice embedding check failed to run: {e}")

    threading.Thread(target=_run, daemon=True, name="VoiceEmbeddingCheck").start()


def run_startup_tasks(app):
    """Run all startup tasks that need to happen after app creation."""
    from src.models import SystemSetting

    with app.app_context():
        # Set dynamic MAX_CONTENT_LENGTH from DB settings. The Werkzeug
        # ceiling is the higher of the two limits because audio-only video
        # uploads can exceed max_file_size_mb (only the extracted audio
        # has to fit that). The view-level handler re-checks each request
        # against the right effective limit.
        max_file_size_mb = int(SystemSetting.get_setting('max_file_size_mb', 250))
        max_audio_only_video_mb = int(
            SystemSetting.get_setting('max_audio_only_video_size_mb', max_file_size_mb * 4)
        )
        wsgi_ceiling_mb = max(max_file_size_mb, max_audio_only_video_mb)
        app.config['MAX_CONTENT_LENGTH'] = wsgi_ceiling_mb * 1024 * 1024
        app.logger.info(
            f"Set MAX_CONTENT_LENGTH to {wsgi_ceiling_mb}MB "
            f"(max_file_size_mb={max_file_size_mb}, "
            f"max_audio_only_video_size_mb={max_audio_only_video_mb})"
        )

        # Bind the queue in EVERY process, so any of them can accept an
        # upload and enqueue it. Only the owner elected below runs workers.
        bind_job_queue(app)

        # Everything below that runs in the background goes through
        # start_single_instance(). Adding one here without an election means
        # it runs once per gunicorn worker plus twice more from the
        # entrypoint scripts; see the comment on SINGLE_INSTANCE_LOCKS.
        start_single_instance(
            app, JOB_QUEUE_OWNER_LOCK,
            lambda: start_job_queue(app), 'job queue')

        # Watches one directory. Several watchers racing over the same
        # dropped file is how one upload becomes several recordings.
        start_single_instance(
            app, FILE_MONITOR_OWNER_LOCK,
            lambda: initialize_file_monitor(app), 'automated file processing')

        # Deletes recordings on a retention policy. Concurrent sweeps race
        # each other over the same rows and files.
        start_single_instance(
            app, AUTO_DELETION_OWNER_LOCK,
            lambda: initialize_auto_deletion_scheduler(app), 'auto-deletion scheduler')

        # Reaps abandoned recording sessions (#287 c/d).
        start_single_instance(
            app, SESSION_CLEANUP_OWNER_LOCK,
            lambda: initialize_recording_session_cleanup(app), 'recording-session cleanup')

        # Posts webhook deliveries (#275). Its own guard is a module global,
        # which is per-process, so without this every worker ran a dispatcher
        # and each POSTed the same due rows.
        from src.services.webhook_dispatch import start_dispatcher_thread
        start_single_instance(
            app, WEBHOOK_DISPATCHER_OWNER_LOCK,
            lambda: start_dispatcher_thread(app), 'webhook dispatcher')

        # Checks that the backend still returns voice embeddings compatible
        # with the stored profiles (#380). Runs on a thread because it makes a
        # real transcription request and must not hold up the boot.
        start_single_instance(
            app, VOICE_EMBEDDING_CHECK_OWNER_LOCK,
            lambda: initialize_voice_embedding_check(app), 'voice embedding check')

        # Not elected: this only creates a directory and logs. No thread, no
        # sweep, nothing to duplicate. If it ever grows a background task it
        # needs to move up into the block above.
        initialize_file_exporter(app)