Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
32.9 KB
Raw Blame
"""Webhook dispatch service (#275).

Two responsibilities:

1. ``emit_webhook_event(user_id, event_type, data, *, app=None)`` — the
   call-site API. Cheap and synchronous: looks up the user's subscribed
   webhooks, creates a ``WebhookDelivery`` row per match, and returns.
   Safe to call from request handlers and from the job_queue worker.
2. ``run_dispatcher_pass(app)`` — the background sweep. Picks up
   ``pending`` deliveries plus ``failed`` deliveries whose
   ``next_retry_at`` has elapsed, POSTs them with the HMAC signature,
   and updates the row. Auto-pauses the parent webhook after N
   consecutive failures.

The dispatcher is launched from ``src/config/startup.py`` as a daemon
thread, matching the pattern used by the recording-session cleanup
sweep.

Security:

- HMAC-SHA256 signature in ``Speakr-Signature`` (``sha256=<hex>``).
- ``Speakr-Delivery-Id`` and ``Speakr-Event`` headers for idempotency
  and cheap routing.
- ``Speakr-Timestamp`` so receivers can reject stale deliveries.
- SSRF guard: outbound URLs are resolved and validated against a
  private-network blocklist. ``allow_http`` only opens the http://
  scheme; private hosts still require the admin allowlist.
"""

import hashlib
import hmac
import ipaddress
import json
import logging
import os
import re
import socket
import threading
import time
import uuid
from datetime import datetime, timedelta
from typing import Iterable
from urllib.parse import urlparse

import requests
from requests.adapters import HTTPAdapter
from urllib3.poolmanager import PoolManager
from urllib3.connection import HTTPConnection, HTTPSConnection
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool

from src.database import db
from src.models import Webhook, WebhookDelivery, WEBHOOK_EVENT_TYPES


logger = logging.getLogger(__name__)


# ---- Config knobs (read fresh each call so tests can patch env) ------------

def _delivery_timeout() -> float:
    try:
        return float(os.environ.get('WEBHOOK_DELIVERY_TIMEOUT_SECONDS', '10'))
    except (TypeError, ValueError):
        return 10.0


def _max_attempts() -> int:
    try:
        return max(1, int(os.environ.get('WEBHOOK_MAX_ATTEMPTS', '5')))
    except (TypeError, ValueError):
        return 5


def _autopause_failures() -> int:
    try:
        return max(1, int(os.environ.get('WEBHOOK_AUTOPAUSE_FAILURES', '10')))
    except (TypeError, ValueError):
        return 10


def _trial_interval_seconds() -> int:
    """How often an auto-paused webhook gets a trial-fire delivery.

    Defaults to one hour: long enough that a flapping endpoint doesn't
    flood the receiver, short enough that a recovered endpoint resumes
    within a working day. Set 0 to disable trial-fires entirely (the
    webhook then stays paused until the user manually re-enables).
    """
    try:
        return max(0, int(os.environ.get('WEBHOOK_TRIAL_INTERVAL_SECONDS', '3600')))
    except (TypeError, ValueError):
        return 3600


def _dispatcher_interval() -> int:
    try:
        return max(1, int(os.environ.get('WEBHOOK_DISPATCHER_INTERVAL_SECONDS', '5')))
    except (TypeError, ValueError):
        return 5


def _max_per_user() -> int:
    try:
        return max(1, int(os.environ.get('WEBHOOK_MAX_PER_USER', '10')))
    except (TypeError, ValueError):
        return 10


def _global_enabled() -> bool:
    return os.environ.get('WEBHOOK_GLOBAL_ENABLED', 'true').lower() != 'false'


def _intranet_host_allowlist():
    """Return a compiled regex matching admin-allowlisted intranet hosts.

    Off by default: the env var is empty so the regex matches nothing,
    meaning private IPs are blocked. Operators who genuinely want to
    POST to internal services set ``WEBHOOK_INTRANET_HOST_ALLOWLIST`` to
    a regex like ``^(home\\.lan|192\\.168\\.1\\.\\d+)$``.
    """
    raw = os.environ.get('WEBHOOK_INTRANET_HOST_ALLOWLIST', '')
    if not raw:
        return None
    try:
        return re.compile(raw)
    except re.error as e:
        logger.warning(f"Invalid WEBHOOK_INTRANET_HOST_ALLOWLIST regex {raw!r}: {e}")
        return None


# Retry delays in seconds for attempts 1, 2, 3, 4, 5+.
_RETRY_DELAYS = [0, 30, 120, 600, 3600]


def _delay_for_attempt(attempt_count: int) -> int:
    """Return the delay (seconds) BEFORE this attempt. attempt_count is
    1-indexed: 1 = first attempt (immediate)."""
    if attempt_count <= 0:
        return 0
    if attempt_count - 1 < len(_RETRY_DELAYS):
        return _RETRY_DELAYS[attempt_count - 1]
    return _RETRY_DELAYS[-1]


# ---- URL safety / SSRF guard ----------------------------------------------

def _is_blocked_ip(ip_str) -> bool:
    """True if ``ip_str`` is a private/loopback/link-local/reserved/etc. address.

    Fails closed: an unparseable value is treated as blocked. Shared by the
    URL validation (below) and the connect-time peer check (SSRFGuardAdapter)
    so the two can never disagree.
    """
    try:
        ip = ipaddress.ip_address(ip_str)
    except ValueError:
        return True
    return (
        ip.is_loopback or ip.is_private or ip.is_link_local
        or ip.is_reserved or ip.is_multicast or ip.is_unspecified
        or (ip.version == 6 and ip.is_site_local)
    )


class _WebhookSSRFError(Exception):
    """Raised at connect time when a webhook's real peer IP is a blocked address."""


def _guard_peer(conn) -> None:
    """Abort the connection if its actual peer IP is blocked.

    Runs after ``connect()`` (TCP + TLS) but before any HTTP request bytes are
    sent, so a rebound host is dropped before it can receive the payload or leak
    a response. This closes the DNS-rebinding TOCTOU window that a separate
    validate-then-``requests.post`` (two independent DNS lookups) leaves open
    (GHSA-2m89-mxcv-gv93).
    """
    try:
        peer_ip = conn.sock.getpeername()[0]
    except Exception as e:  # pragma: no cover - defensive
        _quiet_close(conn)
        raise _WebhookSSRFError(f'could not determine peer address: {e}')
    if _is_blocked_ip(peer_ip):
        _quiet_close(conn)
        raise _WebhookSSRFError(f'connection resolved to a blocked address ({peer_ip})')


def _quiet_close(conn) -> None:
    try:
        conn.close()
    except Exception:  # pragma: no cover
        pass


class _GuardedHTTPConnection(HTTPConnection):
    def connect(self):
        super().connect()
        _guard_peer(self)


class _GuardedHTTPSConnection(HTTPSConnection):
    def connect(self):
        super().connect()
        _guard_peer(self)


class _GuardedHTTPConnectionPool(HTTPConnectionPool):
    ConnectionCls = _GuardedHTTPConnection


class _GuardedHTTPSConnectionPool(HTTPSConnectionPool):
    ConnectionCls = _GuardedHTTPSConnection


class _SSRFGuardPoolManager(PoolManager):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.pool_classes_by_scheme = {
            'http': _GuardedHTTPConnectionPool,
            'https': _GuardedHTTPSConnectionPool,
        }


class SSRFGuardAdapter(HTTPAdapter):
    """requests adapter that verifies the socket's real peer IP after connect
    and aborts on a private/loopback/etc. address (see _guard_peer)."""

    def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
        self.poolmanager = _SSRFGuardPoolManager(
            num_pools=connections, maxsize=maxsize, block=block, **pool_kwargs
        )


def ssrf_guarded_session() -> requests.Session:
    """A requests Session whose connections are peer-IP-verified. Retries are
    disabled so a blocked connection fails fast rather than re-resolving."""
    session = requests.Session()
    adapter = SSRFGuardAdapter(max_retries=0)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


def _host_is_allowlisted(url: str) -> bool:
    """True if the URL's host matches WEBHOOK_INTRANET_HOST_ALLOWLIST — an
    operator opt-in that intentionally permits an internal host, so the
    connect-time peer guard is relaxed for it (matching is_url_safe_for_webhook)."""
    try:
        host = (urlparse(url).hostname or '').lower()
    except Exception:
        return False
    allowlist = _intranet_host_allowlist()
    return bool(host and allowlist and allowlist.search(host))


def _http_post(url, **kwargs):
    """The single outbound-POST seam for webhook delivery.

    Public hosts go through the SSRF-guard session, which verifies the real
    connected peer IP and aborts on a private/loopback address (closing the
    DNS-rebinding TOCTOU window). Explicitly allowlisted intranet hosts use the
    plain client by operator choice. Tests patch this function.
    """
    if _host_is_allowlisted(url):
        return requests.post(url, **kwargs)
    return ssrf_guarded_session().post(url, **kwargs)


def is_url_safe_for_webhook(url: str, allow_http: bool = False, language=None) -> tuple:
    """Validate a webhook URL.

    Returns ``(ok: bool, reason: str)``. ``ok`` is False when the URL
    points at a private/loopback/link-local IP (unless the
    ``WEBHOOK_INTRANET_HOST_ALLOWLIST`` regex matches the hostname), or
    uses an unsupported scheme. ``reason`` is in ``language``, else the
    interface language of the current request (#477).
    """
    problem = webhook_url_problem(url, allow_http)
    if problem is None:
        return True, ''
    from src.utils.localization import translate
    code, params = problem
    return False, translate(f'server.apiError.{code}', language, **params)


def webhook_url_problem(url: str, allow_http: bool = False):
    """The check behind is_url_safe_for_webhook: None for a safe URL, else
    (server.apiError code, placeholder values)."""
    if not url:
        return 'url_required', {}
    try:
        parsed = urlparse(url)
    except Exception:
        return 'webhook_url_unparseable', {}

    scheme = (parsed.scheme or '').lower()
    if scheme not in ('http', 'https'):
        return 'webhook_url_bad_scheme', {}
    if scheme == 'http' and not allow_http:
        return 'webhook_url_http_not_allowed', {}

    host = parsed.hostname
    if not host:
        return 'webhook_url_missing_host', {}

    # Always resolve and inspect every address the hostname maps to,
    # THEN decide whether the allowlist permits a private/loopback IP.
    # Old behaviour returned early on an allowlist hostname match,
    # which (a) skipped DNS resolution entirely and (b) let a careless
    # regex like `^localhost` match `localhost.evil.com`. Now the
    # allowlist only relaxes the private-IP check; the resolution runs
    # unconditionally, and the allowlist is matched against the lower-
    # cased host so case quirks don't help an attacker.
    try:
        addrinfos = socket.getaddrinfo(host, None)
    except socket.gaierror:
        return 'webhook_dns_failed', {'host': host}

    allowlist = _intranet_host_allowlist()
    host_lower = host.lower()
    allowlist_matches = bool(allowlist and allowlist.search(host_lower))

    for _af, _stype, _proto, _name, sockaddr in addrinfos:
        ip_str = sockaddr[0]
        try:
            ipaddress.ip_address(ip_str)
        except ValueError:
            continue
        if _is_blocked_ip(ip_str) and not allowlist_matches:
            return 'webhook_url_private_address', {'ip': ip_str}

    return None


# ---- Signing & headers -----------------------------------------------------

def sign_payload(secret: str, body: bytes) -> str:
    """Compute the Speakr-Signature header value for ``body``."""
    mac = hmac.new(secret.encode('utf-8'), body, hashlib.sha256)
    return f'sha256={mac.hexdigest()}'


def sign_payload_v2(secret: str, t: int, body: bytes) -> str:
    """One v1= value of Speakr-Signature-V2: HMAC-SHA256 over "<t>." + body.

    The send time t is inside the signature, so a receiver can check
    freshness on every attempt while the body stays byte-identical.
    """
    mac = hmac.new(secret.encode('utf-8'), f'{t}.'.encode('utf-8') + body, hashlib.sha256)
    return mac.hexdigest()


def signature_v2_header(webhook, t: int, body: bytes) -> str:
    """t=<unix seconds>,v1=<new secret>[,v1=<previous secret during the rotation grace>]."""
    parts = [f't={t}', f'v1={sign_payload_v2(webhook.secret, t, body)}']
    previous = getattr(webhook, 'previous_secret', None)
    expires = getattr(webhook, 'previous_secret_expires_at', None)
    if isinstance(previous, str) and previous and isinstance(expires, datetime) and expires > datetime.utcnow():
        parts.append(f'v1={sign_payload_v2(previous, t, body)}')
    return ','.join(parts)


def _build_envelope(event_id: str, event_type: str, user_id: int, data: dict):
    now = datetime.utcnow().isoformat() + 'Z'
    return {
        'id': event_id,
        'type': event_type,
        'timestamp': now,
        'occurred_at': now,
        'user_id': user_id,
        'data': data or {},
    }


def serialize_envelope(envelope: dict) -> str:
    """Canonical JSON serialization for webhook payloads.

    Compact form (no whitespace after ``,`` or ``:``) so receivers
    debugging with tools like webhook.site can copy-paste the displayed
    body and compute a matching HMAC without canonicalisation pitfalls.
    The wire format is also what other webhook providers (Stripe,
    GitHub, Mux) use, which means common signature-verification
    libraries Just Work.

    All three webhook payload producers (emit_webhook_event,
    test_fire, replay_delivery) MUST go through this function so the
    bytes signed are always the bytes sent.
    """
    return json.dumps(envelope, separators=(',', ':'), ensure_ascii=False)


# ---- Public entry point: enqueue ------------------------------------------

def emit_webhook_event(user_id: int, event_type: str, data: dict, *, app=None) -> int:
    """Enqueue a webhook delivery for each subscribed endpoint.

    Returns the count of WebhookDelivery rows created (0 if no enabled
    webhooks are subscribed). Safe to call from request handlers; the
    actual HTTP POST happens off-request in the dispatcher loop.
    """
    if not _global_enabled():
        return 0
    if event_type not in WEBHOOK_EVENT_TYPES:
        logger.warning(f"emit_webhook_event called with unknown type {event_type!r}; dropping")
        return 0

    def _do_enqueue():
        return enqueue_event(db.session, user_id, event_type, data)

    if app is not None:
        with app.app_context():
            return _do_enqueue()
    return _do_enqueue()


def _with_updated_at(session, event_type, data, user_id=None):
    """data of a recording.* event carries the recording's updated_at (spec W4)."""
    data = dict(data or {})
    if event_type.startswith('recording.') and 'updated_at' not in data and data.get('recording_id'):
        from src.models import Recording
        rec = session.get(Recording, data['recording_id'])
        if rec is not None:
            data['updated_at'] = rec.updated_at_z()
    if event_type.startswith('recording.') and event_type != 'recording.deleted' and data.get('recording_id') \
            and 'external_refs' not in data:
        from src.models import RecordingExternalRef
        data['external_refs'] = [r.to_dict() for r in session.query(RecordingExternalRef).filter_by(
            recording_id=data['recording_id'], user_id=user_id).order_by(RecordingExternalRef.id)]
    return data


def _merge_pending_update(session, webhook_id, data):
    """Fold a recording.updated into a not-yet-attempted delivery for the same
    recording and fields (autosave bursts give one delivery). Returns True when
    merged."""
    candidates = (session.query(WebhookDelivery)
                  .filter_by(webhook_id=webhook_id, event_type='recording.updated',
                             status='pending', attempt_count=0)
                  .order_by(WebhookDelivery.id.desc()).limit(20).all())
    for delivery in candidates:
        try:
            envelope = json.loads(delivery.payload)
        except (TypeError, ValueError):
            continue
        old = envelope.get('data') or {}
        if (old.get('recording_id') == data.get('recording_id')
                and sorted(old.get('fields_changed') or []) == sorted(data.get('fields_changed') or [])):
            envelope['data'] = data
            delivery.payload = serialize_envelope(envelope)
            return True
    return False


def _shared_targets(session, event_type, data):
    """(webhook, recipient id) for recipients of a shared recording whose
    webhook asks for shared recordings (spec W6)."""
    rid = data.get('recording_id') if isinstance(data, dict) else None
    if not event_type.startswith('recording.') or not rid:
        return []
    from src import app as app_module
    if not getattr(app_module, 'ENABLE_INTERNAL_SHARING', False):
        return []
    from src.models import InternalShare, Recording
    recording = session.get(Recording, rid)
    if recording is None or recording.status != 'COMPLETED':
        return []
    recipients = [uid for (uid,) in session.query(InternalShare.shared_with_user_id)
                  .filter(InternalShare.recording_id == rid)]
    if not recipients:
        return []
    hooks = (session.query(Webhook)
             .filter(Webhook.user_id.in_(recipients), Webhook.enabled.is_(True), Webhook.include_shared.is_(True))
             .all())
    return [(wh, wh.user_id) for wh in hooks if event_type in wh.event_list]


def enqueue_event(session, user_id, event_type, data):
    """Write one pending delivery per subscribed webhook through ``session``
    and commit it: the owner's webhooks, and recipients' webhooks that ask for
    shared recordings. Returns the number of deliveries written or merged."""
    subscriptions = session.query(Webhook).filter_by(user_id=user_id, enabled=True).all()
    targets = [(w, user_id) for w in subscriptions if event_type in w.event_list]
    targets += _shared_targets(session, event_type, data)
    if not targets:
        return 0
    event_id = str(uuid.uuid4())
    first_attempt_at = datetime.utcnow()
    created = 0
    per_user = {}
    for wh, viewer in targets:
        if viewer not in per_user:
            viewer_data = _with_updated_at(session, event_type, data, viewer)
            if viewer != user_id:
                viewer_data = dict(viewer_data, owner_user_id=user_id)
            per_user[viewer] = viewer_data
        if _deliver_one(session, wh, viewer, event_type, per_user[viewer], event_id, first_attempt_at):
            created += 1
    if created:
        session.commit()
    return created


def _deliver_one(session, wh, viewer, event_type, data, event_id, first_attempt_at):
    """Queue (or merge) one delivery; True when a delivery was written or merged."""
    if event_type == 'recording.updated' and _merge_pending_update(session, wh.id, data):
        return True
    envelope = _build_envelope(event_id, event_type, viewer, data)
    session.add(WebhookDelivery(
        webhook_id=wh.id,
        event_id=event_id,
        event_type=event_type,
        payload=serialize_envelope(envelope),
        status='pending',
        next_retry_at=first_attempt_at,
    ))
    return True


# ---- Dispatcher: one pass over due deliveries -----------------------------

def _post_delivery(delivery: WebhookDelivery, webhook: Webhook) -> tuple:
    """Run one HTTP attempt. Returns (response_status, body_preview, error).

    On network/timeout error, response_status is None and error is set.
    """
    body_bytes = (delivery.payload or '').encode('utf-8')

    # Re-validate the URL at SEND time, not just at create/update. The
    # create-time check can be defeated by DNS rebinding: register a webhook on
    # a domain that resolves to a public IP when validated, then repoint it at
    # 127.0.0.1 / 169.254.169.254 / an internal host before delivery. Re-running
    # the resolve-and-block check immediately before the POST closes that window
    # (the stored response preview is readable back via the deliveries API, so
    # an internal target would otherwise be an exfiltration channel). A transient
    # DNS failure is NOT treated as a block — no resolution means no connection
    # and thus no SSRF; we let requests surface it as a normal network error.
    # English: the reason goes into the stored delivery log, and the check
    # below matches its text.
    safe, reason = is_url_safe_for_webhook(webhook.url, allow_http=bool(webhook.allow_http), language='en')
    if not safe and 'resolves to a private' in reason:
        return None, None, f'blocked at delivery: {reason}'[:500]

    signature = sign_payload(webhook.secret, body_bytes)
    import time as _time
    sent_at = int(_time.time())
    headers = {
        'Content-Type': 'application/json',
        'User-Agent': f'Speakr-Webhook/1.0',
        'Speakr-Signature': signature,
        'Speakr-Signature-V2': signature_v2_header(webhook, sent_at, body_bytes),
        'Speakr-Attempt': str((delivery.attempt_count or 0) + 1),
        'Speakr-Delivery-Id': delivery.event_id,
        'Speakr-Event': delivery.event_type,
        'Speakr-Timestamp': datetime.utcnow().isoformat() + 'Z',
    }
    # Delivery goes through _http_post, which verifies the ACTUAL connected peer
    # IP (not a fresh, separate DNS lookup) and aborts on a private/loopback
    # address — closing the rebinding window left by the resolve-and-block check
    # above. Allowlisted intranet hosts are exempt by operator choice.
    try:
        resp = _http_post(
            webhook.url,
            data=body_bytes,
            headers=headers,
            timeout=_delivery_timeout(),
            allow_redirects=False,
        )
        preview = (resp.text or '')[:2000]
        return resp.status_code, preview, None
    except _WebhookSSRFError as e:
        return None, None, f'blocked at delivery: {e}'[:500]
    except requests.RequestException as e:
        # urllib3 may surface the connect-time abort wrapped as a ConnectionError.
        msg = str(e)
        if 'blocked address' in msg or 'peer address' in msg:
            return None, None, f'blocked at delivery: {msg}'[:500]
        return None, None, msg[:500]


def _is_retryable_status(status_code) -> bool:
    if status_code is None:
        return True  # network error
    if 200 <= status_code < 300:
        return False
    if status_code in (408, 429):
        return True
    if 500 <= status_code < 600:
        return True
    # 3xx redirects (we disabled allow_redirects), 4xx others = permanent
    return False


def _apply_attempt_result(delivery: WebhookDelivery, status_code, body_preview, error):
    """Mutate the delivery row based on the HTTP outcome. Returns the
    new status string."""
    delivery.attempt_count = (delivery.attempt_count or 0) + 1
    delivery.response_status = status_code
    delivery.response_body_preview = body_preview
    delivery.error_message = error

    if status_code is not None and 200 <= status_code < 300:
        delivery.status = 'success'
        delivery.delivered_at = datetime.utcnow()
        delivery.next_retry_at = None
        return 'success'

    if not _is_retryable_status(status_code) or delivery.attempt_count >= _max_attempts():
        delivery.status = 'permanent_failure'
        delivery.next_retry_at = None
        return 'permanent_failure'

    delivery.status = 'failed'
    delay = _delay_for_attempt(delivery.attempt_count + 1)
    delivery.next_retry_at = datetime.utcnow() + timedelta(seconds=delay)
    return 'failed'


def _due_deliveries(now: datetime, limit: int = 100):
    """Find deliveries that should be attempted now: pending OR failed
    with elapsed next_retry_at."""
    return (
        WebhookDelivery.query
        .filter(
            WebhookDelivery.status.in_(('pending', 'failed')),
            (WebhookDelivery.next_retry_at == None) | (WebhookDelivery.next_retry_at <= now),  # noqa: E711
        )
        .order_by(WebhookDelivery.created_at.asc())
        .limit(limit)
        .all()
    )


def run_dispatcher_pass(app=None, limit: int = 100) -> dict:
    """Run one sweep of the dispatcher. Returns a counters dict.

    Safe to call from tests; for production use, the background thread
    started in startup.py invokes this on a fixed interval.
    """
    if not _global_enabled():
        return {'attempted': 0, 'success': 0, 'failed': 0, 'permanent_failure': 0}

    def _run_one():
        now = datetime.utcnow()
        due = _due_deliveries(now, limit=limit)
        attempted = 0
        outcomes = {'success': 0, 'failed': 0, 'permanent_failure': 0}
        autopause_thresh = _autopause_failures()
        trial_interval_s = _trial_interval_seconds()
        # Track which auto-paused webhooks have already been trialed
        # this pass so we attempt at most one delivery per webhook per
        # interval (the rest of their queue waits for the trial result).
        trialed_this_pass = set()
        for delivery in due:
            webhook = delivery.webhook
            if webhook is None:
                delivery.status = 'permanent_failure'
                delivery.next_retry_at = None
                continue
            if not webhook.enabled:
                if not webhook.auto_paused:
                    # User explicitly disabled; abandon queued rows.
                    delivery.status = 'permanent_failure'
                    delivery.next_retry_at = None
                    continue
                # Auto-paused webhook. Try at most ONE delivery per
                # pass as a trial. Subsequent queued deliveries for the
                # same webhook are pushed past the trial interval so
                # we recheck them after the next trial window.
                if webhook.id in trialed_this_pass:
                    delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)
                    delivery.status = 'failed'
                    continue
                trialed_this_pass.add(webhook.id)
            status_code, body_preview, error = _post_delivery(delivery, webhook)
            outcome = _apply_attempt_result(delivery, status_code, body_preview, error)
            attempted += 1
            outcomes[outcome] = outcomes.get(outcome, 0) + 1

            # Update the parent webhook's health counters.
            webhook.last_delivery_at = datetime.utcnow()
            if outcome == 'success':
                # Successful delivery resets failure counter AND
                # unpauses an auto-paused webhook so its remaining
                # queue can flush on subsequent passes.
                webhook.consecutive_failures = 0
                if webhook.auto_paused:
                    webhook.enabled = True
                    webhook.auto_paused = False
            else:
                if outcome == 'permanent_failure':
                    webhook.consecutive_failures = (webhook.consecutive_failures or 0) + 1
                    if webhook.consecutive_failures >= autopause_thresh and not webhook.auto_paused:
                        # First time crossing the threshold: pause and
                        # schedule the next trial 1h out (configurable
                        # via WEBHOOK_TRIAL_INTERVAL_SECONDS).
                        webhook.enabled = False
                        webhook.auto_paused = True
                        delivery.status = 'failed'
                        delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)
                elif webhook.auto_paused:
                    # Failed trial of an already-paused webhook: push
                    # next trial forward by the interval.
                    delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)

        if attempted or trialed_this_pass:
            db.session.commit()
        return {'attempted': attempted, **outcomes}

    if app is not None:
        with app.app_context():
            return _run_one()
    return _run_one()


# ---- Background dispatcher thread -----------------------------------------

_dispatcher_thread_started = False
_dispatcher_thread_lock = threading.Lock()


def _recover_stuck_pending_deliveries(app, stale_after_seconds: int = 300):
    """On dispatcher startup, reset deliveries that look orphaned.

    If the dispatcher died between fetching a delivery and committing
    its outcome, the row stays ``status='pending'`` with attempts > 0
    and ``next_retry_at`` either None or in the past. Without
    intervention the next dispatcher pass would already pick those up
    (status='pending' qualifies), but the ``created_at`` ordering means
    if the worker died on a specific row mid-update, the stale
    in-memory state can leave the row neither picked up nor cleaned.
    Scan once at startup and forcibly schedule them for immediate retry.
    """
    with app.app_context():
        try:
            cutoff = datetime.utcnow() - timedelta(seconds=stale_after_seconds)
            stuck = WebhookDelivery.query.filter(
                WebhookDelivery.status == 'pending',
                WebhookDelivery.attempt_count > 0,
                WebhookDelivery.created_at < cutoff,
            ).all()
            if not stuck:
                return 0
            now = datetime.utcnow()
            for d in stuck:
                d.next_retry_at = now
            db.session.commit()
            app.logger.info(
                f"Webhook dispatcher: reset {len(stuck)} stuck pending deliveries at startup"
            )
            return len(stuck)
        except Exception as e:
            app.logger.warning(f"Webhook stuck-delivery recovery failed: {e}", exc_info=True)
            return 0


def start_dispatcher_thread(app):
    """Spawn the daemon dispatcher thread (idempotent)."""
    global _dispatcher_thread_started
    with _dispatcher_thread_lock:
        if _dispatcher_thread_started:
            return

        # Recover any deliveries left in a half-processed pending state
        # by a previous dispatcher that died mid-pass.
        _recover_stuck_pending_deliveries(app)

        interval = _dispatcher_interval()

        def _loop():
            app.logger.info(f"Webhook dispatcher started (interval={interval}s)")
            while True:
                try:
                    time.sleep(interval)
                    if not _global_enabled():
                        continue
                    counters = run_dispatcher_pass(app=app)
                    if counters.get('attempted'):
                        app.logger.info(
                            f"Webhook dispatcher pass: attempted={counters['attempted']} "
                            f"success={counters.get('success', 0)} "
                            f"failed={counters.get('failed', 0)} "
                            f"permanent_failure={counters.get('permanent_failure', 0)}"
                        )
                except Exception as e:
                    app.logger.error(f"Webhook dispatcher error: {e}", exc_info=True)
                    time.sleep(min(60, interval * 2))

        t = threading.Thread(target=_loop, daemon=True, name="WebhookDispatcher")
        t.start()
        _dispatcher_thread_started = True
        app.logger.info("✅ Webhook dispatcher thread initialized")


def emit_recording_created(recording):
    """recording.created, from every path that creates a recording (#412 audit P11).

    Upload, upload join, API v1 and integration uploads, the share target, the
    watch folder, merges and recording-session finalize all call this, so a
    subscriber hears about every new recording. Best-effort: never raises.
    """
    try:
        emit_webhook_event(
            user_id=recording.user_id,
            event_type='recording.created',
            data={
                'recording_id': recording.id,
                'title': recording.title,
                'file_size': recording.file_size,
                'original_filename': recording.original_filename,
            },
        )
    except Exception as e:
        logger.warning(f"Webhook emit (recording.created) failed for {getattr(recording, 'id', None)}: {e}")


def emit_summary_result(recording, ok, error=None):
    """recording.summary.completed / .failed from the summary step itself (#412 audit P12).

    The job queue used to infer these from the job type: the first summary of
    an upload (run inside the transcription job) never fired, a summary that
    failed inside its job fired "completed", and "failed" never fired.
    Best-effort: never raises.
    """
    try:
        if ok:
            data = {
                'recording_id': recording.id,
                'title': recording.title,
                'language': recording.transcription_language,
                'audio_duration_seconds': recording.audio_duration_seconds,
                'transcription_duration_seconds': recording.transcription_duration_seconds,
                'summarization_duration_seconds': recording.summarization_duration_seconds,
            }
            emit_webhook_event(user_id=recording.user_id, event_type='recording.summary.completed',
                               data={k: v for k, v in data.items() if v is not None})
        else:
            emit_webhook_event(user_id=recording.user_id, event_type='recording.summary.failed',
                               data={'recording_id': recording.id, 'title': recording.title,
                                     'error': (error or '')[:500]})
    except Exception as e:
        logger.warning(f"Webhook emit (summary result) failed for {getattr(recording, 'id', None)}: {e}")