Speakr is a personal, self-hosted web application designed for transcribing audio recordings
Python
"""Webhook dispatch service (#275).
Two responsibilities:
1. ``emit_webhook_event(user_id, event_type, data, *, app=None)`` — the
call-site API. Cheap and synchronous: looks up the user's subscribed
webhooks, creates a ``WebhookDelivery`` row per match, and returns.
Safe to call from request handlers and from the job_queue worker.
2. ``run_dispatcher_pass(app)`` — the background sweep. Picks up
``pending`` deliveries plus ``failed`` deliveries whose
``next_retry_at`` has elapsed, POSTs them with the HMAC signature,
and updates the row. Auto-pauses the parent webhook after N
consecutive failures.
The dispatcher is launched from ``src/config/startup.py`` as a daemon
thread, matching the pattern used by the recording-session cleanup
sweep.
Security:
- HMAC-SHA256 signature in ``Speakr-Signature`` (``sha256=<hex>``).
- ``Speakr-Delivery-Id`` and ``Speakr-Event`` headers for idempotency
and cheap routing.
- ``Speakr-Timestamp`` so receivers can reject stale deliveries.
- SSRF guard: outbound URLs are resolved and validated against a
private-network blocklist. ``allow_http`` only opens the http://
scheme; private hosts still require the admin allowlist.
"""
import hashlib
import hmac
import ipaddress
import json
import logging
import os
import re
import socket
import threading
import time
import uuid
from datetime import datetime, timedelta
from typing import Iterable
from urllib.parse import urlparse
import requests
from requests.adapters import HTTPAdapter
from urllib3.poolmanager import PoolManager
from urllib3.connection import HTTPConnection, HTTPSConnection
from urllib3.connectionpool import HTTPConnectionPool, HTTPSConnectionPool
from src.database import db
from src.models import Webhook, WebhookDelivery, WEBHOOK_EVENT_TYPES
logger = logging.getLogger(__name__)
# ---- Config knobs (read fresh each call so tests can patch env) ------------
def _delivery_timeout() -> float:
try:
return float(os.environ.get('WEBHOOK_DELIVERY_TIMEOUT_SECONDS', '10'))
except (TypeError, ValueError):
return 10.0
def _max_attempts() -> int:
try:
return max(1, int(os.environ.get('WEBHOOK_MAX_ATTEMPTS', '5')))
except (TypeError, ValueError):
return 5
def _autopause_failures() -> int:
try:
return max(1, int(os.environ.get('WEBHOOK_AUTOPAUSE_FAILURES', '10')))
except (TypeError, ValueError):
return 10
def _trial_interval_seconds() -> int:
"""How often an auto-paused webhook gets a trial-fire delivery.
Defaults to one hour: long enough that a flapping endpoint doesn't
flood the receiver, short enough that a recovered endpoint resumes
within a working day. Set 0 to disable trial-fires entirely (the
webhook then stays paused until the user manually re-enables).
"""
try:
return max(0, int(os.environ.get('WEBHOOK_TRIAL_INTERVAL_SECONDS', '3600')))
except (TypeError, ValueError):
return 3600
def _dispatcher_interval() -> int:
try:
return max(1, int(os.environ.get('WEBHOOK_DISPATCHER_INTERVAL_SECONDS', '5')))
except (TypeError, ValueError):
return 5
def _max_per_user() -> int:
try:
return max(1, int(os.environ.get('WEBHOOK_MAX_PER_USER', '10')))
except (TypeError, ValueError):
return 10
def _global_enabled() -> bool:
return os.environ.get('WEBHOOK_GLOBAL_ENABLED', 'true').lower() != 'false'
def _intranet_host_allowlist():
"""Return a compiled regex matching admin-allowlisted intranet hosts.
Off by default: the env var is empty so the regex matches nothing,
meaning private IPs are blocked. Operators who genuinely want to
POST to internal services set ``WEBHOOK_INTRANET_HOST_ALLOWLIST`` to
a regex like ``^(home\\.lan|192\\.168\\.1\\.\\d+)$``.
"""
raw = os.environ.get('WEBHOOK_INTRANET_HOST_ALLOWLIST', '')
if not raw:
return None
try:
return re.compile(raw)
except re.error as e:
logger.warning(f"Invalid WEBHOOK_INTRANET_HOST_ALLOWLIST regex {raw!r}: {e}")
return None
# Retry delays in seconds for attempts 1, 2, 3, 4, 5+.
_RETRY_DELAYS = [0, 30, 120, 600, 3600]
def _delay_for_attempt(attempt_count: int) -> int:
"""Return the delay (seconds) BEFORE this attempt. attempt_count is
1-indexed: 1 = first attempt (immediate)."""
if attempt_count <= 0:
return 0
if attempt_count - 1 < len(_RETRY_DELAYS):
return _RETRY_DELAYS[attempt_count - 1]
return _RETRY_DELAYS[-1]
# ---- URL safety / SSRF guard ----------------------------------------------
def _is_blocked_ip(ip_str) -> bool:
"""True if ``ip_str`` is a private/loopback/link-local/reserved/etc. address.
Fails closed: an unparseable value is treated as blocked. Shared by the
URL validation (below) and the connect-time peer check (SSRFGuardAdapter)
so the two can never disagree.
"""
try:
ip = ipaddress.ip_address(ip_str)
except ValueError:
return True
return (
ip.is_loopback or ip.is_private or ip.is_link_local
or ip.is_reserved or ip.is_multicast or ip.is_unspecified
or (ip.version == 6 and ip.is_site_local)
)
class _WebhookSSRFError(Exception):
"""Raised at connect time when a webhook's real peer IP is a blocked address."""
def _guard_peer(conn) -> None:
"""Abort the connection if its actual peer IP is blocked.
Runs after ``connect()`` (TCP + TLS) but before any HTTP request bytes are
sent, so a rebound host is dropped before it can receive the payload or leak
a response. This closes the DNS-rebinding TOCTOU window that a separate
validate-then-``requests.post`` (two independent DNS lookups) leaves open
(GHSA-2m89-mxcv-gv93).
"""
try:
peer_ip = conn.sock.getpeername()[0]
except Exception as e: # pragma: no cover - defensive
_quiet_close(conn)
raise _WebhookSSRFError(f'could not determine peer address: {e}')
if _is_blocked_ip(peer_ip):
_quiet_close(conn)
raise _WebhookSSRFError(f'connection resolved to a blocked address ({peer_ip})')
def _quiet_close(conn) -> None:
try:
conn.close()
except Exception: # pragma: no cover
pass
class _GuardedHTTPConnection(HTTPConnection):
def connect(self):
super().connect()
_guard_peer(self)
class _GuardedHTTPSConnection(HTTPSConnection):
def connect(self):
super().connect()
_guard_peer(self)
class _GuardedHTTPConnectionPool(HTTPConnectionPool):
ConnectionCls = _GuardedHTTPConnection
class _GuardedHTTPSConnectionPool(HTTPSConnectionPool):
ConnectionCls = _GuardedHTTPSConnection
class _SSRFGuardPoolManager(PoolManager):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.pool_classes_by_scheme = {
'http': _GuardedHTTPConnectionPool,
'https': _GuardedHTTPSConnectionPool,
}
class SSRFGuardAdapter(HTTPAdapter):
"""requests adapter that verifies the socket's real peer IP after connect
and aborts on a private/loopback/etc. address (see _guard_peer)."""
def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
self.poolmanager = _SSRFGuardPoolManager(
num_pools=connections, maxsize=maxsize, block=block, **pool_kwargs
)
def ssrf_guarded_session() -> requests.Session:
"""A requests Session whose connections are peer-IP-verified. Retries are
disabled so a blocked connection fails fast rather than re-resolving."""
session = requests.Session()
adapter = SSRFGuardAdapter(max_retries=0)
session.mount('http://', adapter)
session.mount('https://', adapter)
return session
def _host_is_allowlisted(url: str) -> bool:
"""True if the URL's host matches WEBHOOK_INTRANET_HOST_ALLOWLIST — an
operator opt-in that intentionally permits an internal host, so the
connect-time peer guard is relaxed for it (matching is_url_safe_for_webhook)."""
try:
host = (urlparse(url).hostname or '').lower()
except Exception:
return False
allowlist = _intranet_host_allowlist()
return bool(host and allowlist and allowlist.search(host))
def _http_post(url, **kwargs):
"""The single outbound-POST seam for webhook delivery.
Public hosts go through the SSRF-guard session, which verifies the real
connected peer IP and aborts on a private/loopback address (closing the
DNS-rebinding TOCTOU window). Explicitly allowlisted intranet hosts use the
plain client by operator choice. Tests patch this function.
"""
if _host_is_allowlisted(url):
return requests.post(url, **kwargs)
return ssrf_guarded_session().post(url, **kwargs)
def is_url_safe_for_webhook(url: str, allow_http: bool = False, language=None) -> tuple:
"""Validate a webhook URL.
Returns ``(ok: bool, reason: str)``. ``ok`` is False when the URL
points at a private/loopback/link-local IP (unless the
``WEBHOOK_INTRANET_HOST_ALLOWLIST`` regex matches the hostname), or
uses an unsupported scheme. ``reason`` is in ``language``, else the
interface language of the current request (#477).
"""
problem = webhook_url_problem(url, allow_http)
if problem is None:
return True, ''
from src.utils.localization import translate
code, params = problem
return False, translate(f'server.apiError.{code}', language, **params)
def webhook_url_problem(url: str, allow_http: bool = False):
"""The check behind is_url_safe_for_webhook: None for a safe URL, else
(server.apiError code, placeholder values)."""
if not url:
return 'url_required', {}
try:
parsed = urlparse(url)
except Exception:
return 'webhook_url_unparseable', {}
scheme = (parsed.scheme or '').lower()
if scheme not in ('http', 'https'):
return 'webhook_url_bad_scheme', {}
if scheme == 'http' and not allow_http:
return 'webhook_url_http_not_allowed', {}
host = parsed.hostname
if not host:
return 'webhook_url_missing_host', {}
# Always resolve and inspect every address the hostname maps to,
# THEN decide whether the allowlist permits a private/loopback IP.
# Old behaviour returned early on an allowlist hostname match,
# which (a) skipped DNS resolution entirely and (b) let a careless
# regex like `^localhost` match `localhost.evil.com`. Now the
# allowlist only relaxes the private-IP check; the resolution runs
# unconditionally, and the allowlist is matched against the lower-
# cased host so case quirks don't help an attacker.
try:
addrinfos = socket.getaddrinfo(host, None)
except socket.gaierror:
return 'webhook_dns_failed', {'host': host}
allowlist = _intranet_host_allowlist()
host_lower = host.lower()
allowlist_matches = bool(allowlist and allowlist.search(host_lower))
for _af, _stype, _proto, _name, sockaddr in addrinfos:
ip_str = sockaddr[0]
try:
ipaddress.ip_address(ip_str)
except ValueError:
continue
if _is_blocked_ip(ip_str) and not allowlist_matches:
return 'webhook_url_private_address', {'ip': ip_str}
return None
# ---- Signing & headers -----------------------------------------------------
def sign_payload(secret: str, body: bytes) -> str:
"""Compute the Speakr-Signature header value for ``body``."""
mac = hmac.new(secret.encode('utf-8'), body, hashlib.sha256)
return f'sha256={mac.hexdigest()}'
def sign_payload_v2(secret: str, t: int, body: bytes) -> str:
"""One v1= value of Speakr-Signature-V2: HMAC-SHA256 over "<t>." + body.
The send time t is inside the signature, so a receiver can check
freshness on every attempt while the body stays byte-identical.
"""
mac = hmac.new(secret.encode('utf-8'), f'{t}.'.encode('utf-8') + body, hashlib.sha256)
return mac.hexdigest()
def signature_v2_header(webhook, t: int, body: bytes) -> str:
"""t=<unix seconds>,v1=<new secret>[,v1=<previous secret during the rotation grace>]."""
parts = [f't={t}', f'v1={sign_payload_v2(webhook.secret, t, body)}']
previous = getattr(webhook, 'previous_secret', None)
expires = getattr(webhook, 'previous_secret_expires_at', None)
if isinstance(previous, str) and previous and isinstance(expires, datetime) and expires > datetime.utcnow():
parts.append(f'v1={sign_payload_v2(previous, t, body)}')
return ','.join(parts)
def _build_envelope(event_id: str, event_type: str, user_id: int, data: dict):
now = datetime.utcnow().isoformat() + 'Z'
return {
'id': event_id,
'type': event_type,
'timestamp': now,
'occurred_at': now,
'user_id': user_id,
'data': data or {},
}
def serialize_envelope(envelope: dict) -> str:
"""Canonical JSON serialization for webhook payloads.
Compact form (no whitespace after ``,`` or ``:``) so receivers
debugging with tools like webhook.site can copy-paste the displayed
body and compute a matching HMAC without canonicalisation pitfalls.
The wire format is also what other webhook providers (Stripe,
GitHub, Mux) use, which means common signature-verification
libraries Just Work.
All three webhook payload producers (emit_webhook_event,
test_fire, replay_delivery) MUST go through this function so the
bytes signed are always the bytes sent.
"""
return json.dumps(envelope, separators=(',', ':'), ensure_ascii=False)
# ---- Public entry point: enqueue ------------------------------------------
def emit_webhook_event(user_id: int, event_type: str, data: dict, *, app=None) -> int:
"""Enqueue a webhook delivery for each subscribed endpoint.
Returns the count of WebhookDelivery rows created (0 if no enabled
webhooks are subscribed). Safe to call from request handlers; the
actual HTTP POST happens off-request in the dispatcher loop.
"""
if not _global_enabled():
return 0
if event_type not in WEBHOOK_EVENT_TYPES:
logger.warning(f"emit_webhook_event called with unknown type {event_type!r}; dropping")
return 0
def _do_enqueue():
return enqueue_event(db.session, user_id, event_type, data)
if app is not None:
with app.app_context():
return _do_enqueue()
return _do_enqueue()
def _with_updated_at(session, event_type, data, user_id=None):
"""data of a recording.* event carries the recording's updated_at (spec W4)."""
data = dict(data or {})
if event_type.startswith('recording.') and 'updated_at' not in data and data.get('recording_id'):
from src.models import Recording
rec = session.get(Recording, data['recording_id'])
if rec is not None:
data['updated_at'] = rec.updated_at_z()
if event_type.startswith('recording.') and event_type != 'recording.deleted' and data.get('recording_id') \
and 'external_refs' not in data:
from src.models import RecordingExternalRef
data['external_refs'] = [r.to_dict() for r in session.query(RecordingExternalRef).filter_by(
recording_id=data['recording_id'], user_id=user_id).order_by(RecordingExternalRef.id)]
return data
def _merge_pending_update(session, webhook_id, data):
"""Fold a recording.updated into a not-yet-attempted delivery for the same
recording and fields (autosave bursts give one delivery). Returns True when
merged."""
candidates = (session.query(WebhookDelivery)
.filter_by(webhook_id=webhook_id, event_type='recording.updated',
status='pending', attempt_count=0)
.order_by(WebhookDelivery.id.desc()).limit(20).all())
for delivery in candidates:
try:
envelope = json.loads(delivery.payload)
except (TypeError, ValueError):
continue
old = envelope.get('data') or {}
if (old.get('recording_id') == data.get('recording_id')
and sorted(old.get('fields_changed') or []) == sorted(data.get('fields_changed') or [])):
envelope['data'] = data
delivery.payload = serialize_envelope(envelope)
return True
return False
def _shared_targets(session, event_type, data):
"""(webhook, recipient id) for recipients of a shared recording whose
webhook asks for shared recordings (spec W6)."""
rid = data.get('recording_id') if isinstance(data, dict) else None
if not event_type.startswith('recording.') or not rid:
return []
from src import app as app_module
if not getattr(app_module, 'ENABLE_INTERNAL_SHARING', False):
return []
from src.models import InternalShare, Recording
recording = session.get(Recording, rid)
if recording is None or recording.status != 'COMPLETED':
return []
recipients = [uid for (uid,) in session.query(InternalShare.shared_with_user_id)
.filter(InternalShare.recording_id == rid)]
if not recipients:
return []
hooks = (session.query(Webhook)
.filter(Webhook.user_id.in_(recipients), Webhook.enabled.is_(True), Webhook.include_shared.is_(True))
.all())
return [(wh, wh.user_id) for wh in hooks if event_type in wh.event_list]
def enqueue_event(session, user_id, event_type, data):
"""Write one pending delivery per subscribed webhook through ``session``
and commit it: the owner's webhooks, and recipients' webhooks that ask for
shared recordings. Returns the number of deliveries written or merged."""
subscriptions = session.query(Webhook).filter_by(user_id=user_id, enabled=True).all()
targets = [(w, user_id) for w in subscriptions if event_type in w.event_list]
targets += _shared_targets(session, event_type, data)
if not targets:
return 0
event_id = str(uuid.uuid4())
first_attempt_at = datetime.utcnow()
created = 0
per_user = {}
for wh, viewer in targets:
if viewer not in per_user:
viewer_data = _with_updated_at(session, event_type, data, viewer)
if viewer != user_id:
viewer_data = dict(viewer_data, owner_user_id=user_id)
per_user[viewer] = viewer_data
if _deliver_one(session, wh, viewer, event_type, per_user[viewer], event_id, first_attempt_at):
created += 1
if created:
session.commit()
return created
def _deliver_one(session, wh, viewer, event_type, data, event_id, first_attempt_at):
"""Queue (or merge) one delivery; True when a delivery was written or merged."""
if event_type == 'recording.updated' and _merge_pending_update(session, wh.id, data):
return True
envelope = _build_envelope(event_id, event_type, viewer, data)
session.add(WebhookDelivery(
webhook_id=wh.id,
event_id=event_id,
event_type=event_type,
payload=serialize_envelope(envelope),
status='pending',
next_retry_at=first_attempt_at,
))
return True
# ---- Dispatcher: one pass over due deliveries -----------------------------
def _post_delivery(delivery: WebhookDelivery, webhook: Webhook) -> tuple:
"""Run one HTTP attempt. Returns (response_status, body_preview, error).
On network/timeout error, response_status is None and error is set.
"""
body_bytes = (delivery.payload or '').encode('utf-8')
# Re-validate the URL at SEND time, not just at create/update. The
# create-time check can be defeated by DNS rebinding: register a webhook on
# a domain that resolves to a public IP when validated, then repoint it at
# 127.0.0.1 / 169.254.169.254 / an internal host before delivery. Re-running
# the resolve-and-block check immediately before the POST closes that window
# (the stored response preview is readable back via the deliveries API, so
# an internal target would otherwise be an exfiltration channel). A transient
# DNS failure is NOT treated as a block — no resolution means no connection
# and thus no SSRF; we let requests surface it as a normal network error.
# English: the reason goes into the stored delivery log, and the check
# below matches its text.
safe, reason = is_url_safe_for_webhook(webhook.url, allow_http=bool(webhook.allow_http), language='en')
if not safe and 'resolves to a private' in reason:
return None, None, f'blocked at delivery: {reason}'[:500]
signature = sign_payload(webhook.secret, body_bytes)
import time as _time
sent_at = int(_time.time())
headers = {
'Content-Type': 'application/json',
'User-Agent': f'Speakr-Webhook/1.0',
'Speakr-Signature': signature,
'Speakr-Signature-V2': signature_v2_header(webhook, sent_at, body_bytes),
'Speakr-Attempt': str((delivery.attempt_count or 0) + 1),
'Speakr-Delivery-Id': delivery.event_id,
'Speakr-Event': delivery.event_type,
'Speakr-Timestamp': datetime.utcnow().isoformat() + 'Z',
}
# Delivery goes through _http_post, which verifies the ACTUAL connected peer
# IP (not a fresh, separate DNS lookup) and aborts on a private/loopback
# address — closing the rebinding window left by the resolve-and-block check
# above. Allowlisted intranet hosts are exempt by operator choice.
try:
resp = _http_post(
webhook.url,
data=body_bytes,
headers=headers,
timeout=_delivery_timeout(),
allow_redirects=False,
)
preview = (resp.text or '')[:2000]
return resp.status_code, preview, None
except _WebhookSSRFError as e:
return None, None, f'blocked at delivery: {e}'[:500]
except requests.RequestException as e:
# urllib3 may surface the connect-time abort wrapped as a ConnectionError.
msg = str(e)
if 'blocked address' in msg or 'peer address' in msg:
return None, None, f'blocked at delivery: {msg}'[:500]
return None, None, msg[:500]
def _is_retryable_status(status_code) -> bool:
if status_code is None:
return True # network error
if 200 <= status_code < 300:
return False
if status_code in (408, 429):
return True
if 500 <= status_code < 600:
return True
# 3xx redirects (we disabled allow_redirects), 4xx others = permanent
return False
def _apply_attempt_result(delivery: WebhookDelivery, status_code, body_preview, error):
"""Mutate the delivery row based on the HTTP outcome. Returns the
new status string."""
delivery.attempt_count = (delivery.attempt_count or 0) + 1
delivery.response_status = status_code
delivery.response_body_preview = body_preview
delivery.error_message = error
if status_code is not None and 200 <= status_code < 300:
delivery.status = 'success'
delivery.delivered_at = datetime.utcnow()
delivery.next_retry_at = None
return 'success'
if not _is_retryable_status(status_code) or delivery.attempt_count >= _max_attempts():
delivery.status = 'permanent_failure'
delivery.next_retry_at = None
return 'permanent_failure'
delivery.status = 'failed'
delay = _delay_for_attempt(delivery.attempt_count + 1)
delivery.next_retry_at = datetime.utcnow() + timedelta(seconds=delay)
return 'failed'
def _due_deliveries(now: datetime, limit: int = 100):
"""Find deliveries that should be attempted now: pending OR failed
with elapsed next_retry_at."""
return (
WebhookDelivery.query
.filter(
WebhookDelivery.status.in_(('pending', 'failed')),
(WebhookDelivery.next_retry_at == None) | (WebhookDelivery.next_retry_at <= now), # noqa: E711
)
.order_by(WebhookDelivery.created_at.asc())
.limit(limit)
.all()
)
def run_dispatcher_pass(app=None, limit: int = 100) -> dict:
"""Run one sweep of the dispatcher. Returns a counters dict.
Safe to call from tests; for production use, the background thread
started in startup.py invokes this on a fixed interval.
"""
if not _global_enabled():
return {'attempted': 0, 'success': 0, 'failed': 0, 'permanent_failure': 0}
def _run_one():
now = datetime.utcnow()
due = _due_deliveries(now, limit=limit)
attempted = 0
outcomes = {'success': 0, 'failed': 0, 'permanent_failure': 0}
autopause_thresh = _autopause_failures()
trial_interval_s = _trial_interval_seconds()
# Track which auto-paused webhooks have already been trialed
# this pass so we attempt at most one delivery per webhook per
# interval (the rest of their queue waits for the trial result).
trialed_this_pass = set()
for delivery in due:
webhook = delivery.webhook
if webhook is None:
delivery.status = 'permanent_failure'
delivery.next_retry_at = None
continue
if not webhook.enabled:
if not webhook.auto_paused:
# User explicitly disabled; abandon queued rows.
delivery.status = 'permanent_failure'
delivery.next_retry_at = None
continue
# Auto-paused webhook. Try at most ONE delivery per
# pass as a trial. Subsequent queued deliveries for the
# same webhook are pushed past the trial interval so
# we recheck them after the next trial window.
if webhook.id in trialed_this_pass:
delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)
delivery.status = 'failed'
continue
trialed_this_pass.add(webhook.id)
status_code, body_preview, error = _post_delivery(delivery, webhook)
outcome = _apply_attempt_result(delivery, status_code, body_preview, error)
attempted += 1
outcomes[outcome] = outcomes.get(outcome, 0) + 1
# Update the parent webhook's health counters.
webhook.last_delivery_at = datetime.utcnow()
if outcome == 'success':
# Successful delivery resets failure counter AND
# unpauses an auto-paused webhook so its remaining
# queue can flush on subsequent passes.
webhook.consecutive_failures = 0
if webhook.auto_paused:
webhook.enabled = True
webhook.auto_paused = False
else:
if outcome == 'permanent_failure':
webhook.consecutive_failures = (webhook.consecutive_failures or 0) + 1
if webhook.consecutive_failures >= autopause_thresh and not webhook.auto_paused:
# First time crossing the threshold: pause and
# schedule the next trial 1h out (configurable
# via WEBHOOK_TRIAL_INTERVAL_SECONDS).
webhook.enabled = False
webhook.auto_paused = True
delivery.status = 'failed'
delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)
elif webhook.auto_paused:
# Failed trial of an already-paused webhook: push
# next trial forward by the interval.
delivery.next_retry_at = now + timedelta(seconds=trial_interval_s)
if attempted or trialed_this_pass:
db.session.commit()
return {'attempted': attempted, **outcomes}
if app is not None:
with app.app_context():
return _run_one()
return _run_one()
# ---- Background dispatcher thread -----------------------------------------
_dispatcher_thread_started = False
_dispatcher_thread_lock = threading.Lock()
def _recover_stuck_pending_deliveries(app, stale_after_seconds: int = 300):
"""On dispatcher startup, reset deliveries that look orphaned.
If the dispatcher died between fetching a delivery and committing
its outcome, the row stays ``status='pending'`` with attempts > 0
and ``next_retry_at`` either None or in the past. Without
intervention the next dispatcher pass would already pick those up
(status='pending' qualifies), but the ``created_at`` ordering means
if the worker died on a specific row mid-update, the stale
in-memory state can leave the row neither picked up nor cleaned.
Scan once at startup and forcibly schedule them for immediate retry.
"""
with app.app_context():
try:
cutoff = datetime.utcnow() - timedelta(seconds=stale_after_seconds)
stuck = WebhookDelivery.query.filter(
WebhookDelivery.status == 'pending',
WebhookDelivery.attempt_count > 0,
WebhookDelivery.created_at < cutoff,
).all()
if not stuck:
return 0
now = datetime.utcnow()
for d in stuck:
d.next_retry_at = now
db.session.commit()
app.logger.info(
f"Webhook dispatcher: reset {len(stuck)} stuck pending deliveries at startup"
)
return len(stuck)
except Exception as e:
app.logger.warning(f"Webhook stuck-delivery recovery failed: {e}", exc_info=True)
return 0
def start_dispatcher_thread(app):
"""Spawn the daemon dispatcher thread (idempotent)."""
global _dispatcher_thread_started
with _dispatcher_thread_lock:
if _dispatcher_thread_started:
return
# Recover any deliveries left in a half-processed pending state
# by a previous dispatcher that died mid-pass.
_recover_stuck_pending_deliveries(app)
interval = _dispatcher_interval()
def _loop():
app.logger.info(f"Webhook dispatcher started (interval={interval}s)")
while True:
try:
time.sleep(interval)
if not _global_enabled():
continue
counters = run_dispatcher_pass(app=app)
if counters.get('attempted'):
app.logger.info(
f"Webhook dispatcher pass: attempted={counters['attempted']} "
f"success={counters.get('success', 0)} "
f"failed={counters.get('failed', 0)} "
f"permanent_failure={counters.get('permanent_failure', 0)}"
)
except Exception as e:
app.logger.error(f"Webhook dispatcher error: {e}", exc_info=True)
time.sleep(min(60, interval * 2))
t = threading.Thread(target=_loop, daemon=True, name="WebhookDispatcher")
t.start()
_dispatcher_thread_started = True
app.logger.info("✅ Webhook dispatcher thread initialized")
def emit_recording_created(recording):
"""recording.created, from every path that creates a recording (#412 audit P11).
Upload, upload join, API v1 and integration uploads, the share target, the
watch folder, merges and recording-session finalize all call this, so a
subscriber hears about every new recording. Best-effort: never raises.
"""
try:
emit_webhook_event(
user_id=recording.user_id,
event_type='recording.created',
data={
'recording_id': recording.id,
'title': recording.title,
'file_size': recording.file_size,
'original_filename': recording.original_filename,
},
)
except Exception as e:
logger.warning(f"Webhook emit (recording.created) failed for {getattr(recording, 'id', None)}: {e}")
def emit_summary_result(recording, ok, error=None):
"""recording.summary.completed / .failed from the summary step itself (#412 audit P12).
The job queue used to infer these from the job type: the first summary of
an upload (run inside the transcription job) never fired, a summary that
failed inside its job fired "completed", and "failed" never fired.
Best-effort: never raises.
"""
try:
if ok:
data = {
'recording_id': recording.id,
'title': recording.title,
'language': recording.transcription_language,
'audio_duration_seconds': recording.audio_duration_seconds,
'transcription_duration_seconds': recording.transcription_duration_seconds,
'summarization_duration_seconds': recording.summarization_duration_seconds,
}
emit_webhook_event(user_id=recording.user_id, event_type='recording.summary.completed',
data={k: v for k, v in data.items() if v is not None})
else:
emit_webhook_event(user_id=recording.user_id, event_type='recording.summary.failed',
data={'recording_id': recording.id, 'title': recording.title,
'error': (error or '')[:500]})
except Exception as e:
logger.warning(f"Webhook emit (summary result) failed for {getattr(recording, 'id', None)}: {e}")