Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python
19.0 KB
Raw Blame
"""Detect that the transcription backend's voice embedding model has changed.

Voice profiles are stored as raw vectors with no record of what produced them.
If the backend starts returning embeddings from a different model, those
vectors live in a different space and stop matching, and the failure is
completely silent: `find_matching_speakers()` compares each stored profile in
turn and a mismatched pair raises, which it treats as a corrupt profile and
skips. The user sees voice matching quietly stop working, with nothing in the
logs to explain it (#380).

Dimension alone does not detect this. Two different models can both emit 256
dimensions, and nothing in Speakr enforces a dimension anyway: serialize is
`np.array(...).tobytes()`, deserialize is `np.frombuffer(...)`, and similarity
reshapes to (1, -1). All three work at any length.

The backend cannot be asked either. An ASR response carries the embeddings and
nothing about the model that made them; the connector knows which
*transcription* model it requested and never learns which *embedding* model the
service chose.

So Speakr asks the only question that has a reliable answer: it sends a fixed
clip and compares the embedding that comes back with the one stored when the
reference was taken. Measured against a whisperx-asr-service backend:

    same clip, same model, repeated       cosine 1.000000 (bit-identical)
    different voice, same model           cosine 0.270
    same voice, same words, 5s not 12s    cosine 0.516

A different model puts the clip in an unrelated space, so the separation is
wide and the threshold below has a lot of room. The last row is the reason the
clip is a committed fixture rather than anything generated: this comparison is
sensitive to the exact audio, so the reference is only meaningful against
byte-identical input.
"""

import hashlib
import json
import logging
import math
import os
from datetime import datetime

logger = logging.getLogger(__name__)

# Bump this when the clip changes. A new version re-baselines every instance on
# upgrade rather than warning all of them, because a different clip cannot be
# compared with a reference taken from the old one.
CANARY_CLIP_VERSION = 'v1'
CANARY_FILENAME = f'voice_canary_{CANARY_CLIP_VERSION}.wav'
CANARY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
                           'assets', CANARY_FILENAME)

# Same input through the same model measured exactly 1.0 here, and the nearest
# wrong answer measured 0.52. 0.98 leaves room for a backend whose GPU kernels
# are not bit-deterministic (that kind of noise stays above 0.9999) while
# staying far above anything that is genuinely a different model.
SIMILARITY_THRESHOLD = 0.98

SETTING_KEY = 'voice_embedding_reference'

STATUS_OK = 'ok'
STATUS_CHANGED = 'changed'
STATUS_UNKNOWN = 'unknown'


class CanaryUnavailable(Exception):
    """The probe could not be run, which is not the same as a mismatch."""


def cosine_similarity(a, b):
    """Plain cosine, no numpy, so this works without the embeddings extra."""
    if not a or not b or len(a) != len(b):
        return None
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    if not na or not nb:
        return None
    return dot / (na * nb)


def default_transcription_model():
    """The model an upload gets with no tag, folder or per-upload choice.

    The probe sends the same one, so it tests the configuration recordings
    actually use and never depends on the service's own default (#409).
    """
    try:
        from src.services.transcription_defaults import resolve_transcription_model
        return resolve_transcription_model(None)
    except Exception as e:
        logger.debug(f"Could not resolve the default transcription model: {e}")
        return None


def backend_fingerprint():
    """Identify the configured backend well enough to notice reconfiguration.

    This is a TRIGGER, never the alarm. Renaming a host changes it without
    changing a single embedding, and a service that quietly upgrades its own
    model changes every embedding without touching it. Either way the probe
    decides; this only says when it is worth running one.
    """
    try:
        from src.services.transcription.registry import get_registry
        registry = get_registry()
        connector = registry.get_active_connector()
        parts = [
            registry.get_active_connector_name() or '',
            str(getattr(connector, 'base_url', '') or ''),
            # The model the probe sends. Without an admin default this is the
            # connector's own model, as before, so stored fingerprints stay
            # valid; choosing a default model triggers one fresh probe.
            str(default_transcription_model() or getattr(connector, 'model', '') or ''),
        ]
    except Exception as e:
        logger.debug(f"Could not fingerprint the transcription backend: {e}")
        parts = ['unknown']
    return hashlib.sha256('|'.join(parts).encode()).hexdigest()[:16]


def embeddings_supported():
    """Whether the active connector returns speaker embeddings at all."""
    try:
        from src.services.transcription.base import TranscriptionCapability
        from src.services.transcription.registry import get_registry
        connector = get_registry().get_active_connector()
        return bool(connector.supports(TranscriptionCapability.SPEAKER_EMBEDDINGS))
    except Exception as e:
        logger.debug(f"Could not determine speaker-embedding support: {e}")
        return False


def probe_backend():
    """Send the canary through the active connector and return its embedding.

    Raises CanaryUnavailable when the probe cannot be completed. That is
    deliberately distinct from a mismatch: a backend that is merely down must
    never be reported as a changed model.
    """
    from src.services.transcription.base import TranscriptionRequest
    from src.services.transcription.registry import get_registry

    if not os.path.exists(CANARY_PATH):
        raise CanaryUnavailable(f'canary clip missing at {CANARY_PATH}')

    try:
        connector = get_registry().get_active_connector()
    except Exception as e:
        raise CanaryUnavailable(f'no active transcription connector: {e}')

    try:
        with open(CANARY_PATH, 'rb') as f:
            response = connector.transcribe(TranscriptionRequest(
                audio_file=f,
                filename=CANARY_FILENAME,
                mime_type='audio/wav',
                language='en',
                diarize=True,
                min_speakers=1,
                max_speakers=1,
                model=default_transcription_model(),
            ))
    except Exception as e:
        raise CanaryUnavailable(f'transcription of the canary failed: {e}')

    embeddings = getattr(response, 'speaker_embeddings', None) or {}
    if len(embeddings) != 1:
        # More than one speaker means the backend's diarizer split a
        # single-voice clip, so the result is not comparable with a reference
        # taken from one segment.
        raise CanaryUnavailable(
            f'expected exactly one speaker in the canary, got {len(embeddings)}')

    vector = list(embeddings.values())[0]
    if not vector:
        raise CanaryUnavailable('backend returned an empty embedding')
    return [float(x) for x in vector]


def load_reference():
    from src.models import SystemSetting
    raw = SystemSetting.get_setting(SETTING_KEY, None)
    if not raw:
        return None
    try:
        return json.loads(raw)
    except (TypeError, ValueError):
        logger.warning('Stored voice embedding reference is not valid JSON; ignoring it')
        return None


def save_reference(data):
    from src.models import SystemSetting
    # set_setting commits for us.
    SystemSetting.set_setting(
        SETTING_KEY, json.dumps(data), setting_type='string',
        description='Reference embedding of the bundled canary clip, used to detect '
                    'that the transcription backend changed its voice embedding model')


def get_status(app=None):
    """What the admin UI shows. Never runs a probe."""
    reference = load_reference() or {}
    return {
        'status': reference.get('status', STATUS_UNKNOWN),
        'dimension': reference.get('dimension'),
        'clip_version': reference.get('clip_version'),
        'checked_at': reference.get('checked_at'),
        'similarity': reference.get('last_similarity'),
        'detail': reference.get('detail'),
        'supported': embeddings_supported(),
        'spaces': spaces_status(),
    }


def check_voice_embeddings(app, force=False):
    """Probe the backend and compare against the stored reference.

    Returns the same dict shape as get_status(). Never raises: a failure to
    check is reported, not thrown, because this runs on a startup thread.
    """
    if not embeddings_supported():
        logger.debug('Active connector returns no speaker embeddings; skipping the canary')
        # Whatever was wrong with the old backend's embeddings no longer
        # applies: this one has none. Leaving the notice up would tell the
        # admin to fix something that cannot be fixed from here.
        _clear_notification(app)
        _spaces(lambda vp: vp.clear_current_space())
        return {'status': STATUS_UNKNOWN, 'supported': False,
                'detail': 'the active connector does not return speaker embeddings'}

    reference = load_reference()
    fingerprint = backend_fingerprint()

    # A stored reference from a different clip cannot be compared with this
    # one, so treat it as no reference and re-baseline.
    if reference and reference.get('clip_version') != CANARY_CLIP_VERSION:
        logger.info('Canary clip changed since the reference was taken; re-baselining')
        reference = None

    # First start with embedding spaces: the model the existing profiles were
    # built with is the one the reference was taken from. It becomes the
    # legacy space, which every profile stored without a space belongs to.
    if reference and reference.get('embedding'):
        _spaces(lambda vp: _initialize_spaces(vp, reference))

    if reference and not force and reference.get('backend_fingerprint') == fingerprint:
        # Nothing about the configuration moved. The probe costs a real
        # transcription, so it is not run on every boot for no reason.
        logger.debug('Transcription backend unchanged since the last voice embedding check')
        return get_status(app)

    try:
        vector = probe_backend()
    except CanaryUnavailable as e:
        logger.warning(canary_failure_message(e))
        # Leave any existing reference and its status untouched. A backend
        # that is down is not a backend that changed.
        status = get_status(app)
        status['detail'] = f'check could not run: {e}'
        return status

    now = datetime.utcnow().isoformat()

    # Each model gets its own voice space; recognising one seen before brings
    # its profiles back, a new one starts empty.
    _spaces(lambda vp: vp.register_space(vector, fingerprint))

    if reference is None:
        save_reference({
            'clip_version': CANARY_CLIP_VERSION,
            'dimension': len(vector),
            'embedding': vector,
            'backend_fingerprint': fingerprint,
            'checked_at': now,
            'status': STATUS_OK,
            'last_similarity': 1.0,
            'detail': 'baseline recorded',
        })
        logger.info(f'Recorded a voice embedding baseline ({len(vector)} dimensions)')
        # A fresh baseline is by definition a good state. This is also the
        # path rebaseline() lands on, so without this the admin who did what
        # the banner told them to do would keep seeing the warning.
        _clear_notification(app)
        return get_status(app)

    stored = reference.get('embedding') or []
    if len(stored) != len(vector):
        detail = (f'the backend now returns {len(vector)}-dimensional embeddings, '
                  f'and stored voice profiles are {len(stored)}-dimensional')
        similarity = None
    else:
        similarity = cosine_similarity(stored, vector)
        detail = None

    changed = similarity is None or similarity < SIMILARITY_THRESHOLD
    if changed and detail is None:
        detail = (f'the same clip now embeds differently (similarity '
                  f'{similarity:.4f}, expected at least {SIMILARITY_THRESHOLD})')

    reference.update({
        'backend_fingerprint': fingerprint,
        'checked_at': now,
        'last_similarity': similarity,
        'status': STATUS_CHANGED if changed else STATUS_OK,
        'detail': detail if changed else None,
    })
    # The reference embedding itself is NOT replaced on a mismatch. It is what
    # the existing voice profiles were built against, so it stays until the
    # user rebuilds them or restores the old backend.
    save_reference(reference)

    if changed:
        _log_change_banner(app, detail)
        _raise_notification(app, detail)
    else:
        logger.info('Voice embedding check passed (similarity %.6f)', similarity)
        _clear_notification(app)

    return get_status(app)


def _log_change_banner(app, detail):
    """Say it loudly in the startup output.

    An admin who restarts after repointing their ASR endpoint is reading the
    container log at exactly that moment, and a single ERROR line among the
    ordinary startup chatter is easy to scroll past. This is the one place
    where a banner earns its width.
    """
    logger_ = getattr(app, 'logger', logger)
    width = 78
    lines = [
        '',
        '=' * width,
        'VOICE EMBEDDING MODEL CHANGED'.center(width),
        '=' * width,
        detail or 'the transcription backend returned an unexpected embedding',
        '',
        'Voice profiles are kept separately for each embedding model. Profiles',
        'built with the previous model will not match new recordings; profiles',
        'for this model are learned as speakers are named. Either:',
        '',
        '  * restore the previous transcription backend, which brings its',
        '    profiles back and clears this on the next check, or',
        '  * accept the change: reset the reference from the admin area once',
        '    the profiles have been rebuilt for the new model.',
        '',
        'Set DISABLE_VOICE_EMBEDDING_CHECK=true to stop checking.',
        '=' * width,
        '',
    ]
    for line in lines:
        logger_.error(line)


def _raise_notification(app, detail):
    """Put the warning in front of every admin, not only in the logs."""
    try:
        from src.models import KIND_VOICE_EMBEDDING_CHANGED
        from src.services.notifications import notify
        notify(
            KIND_VOICE_EMBEDDING_CHANGED,
            'notifications.voiceEmbeddingChanged',
            admins=True,
            level='error',
            params={'detail': detail or ''},
            link='/admin#voice-embeddings',
        )
    except Exception as e:
        logger.warning('Could not raise the voice embedding notification: %s', e)


def _clear_notification(app):
    """The condition ended, so the notice goes away on its own.

    This is what makes restoring the previous backend a complete fix rather
    than something the admin also has to acknowledge.
    """
    try:
        from src.models import KIND_VOICE_EMBEDDING_CHANGED
        from src.services.notifications import resolve
        resolve(KIND_VOICE_EMBEDDING_CHANGED)
    except Exception as e:
        logger.warning('Could not clear the voice embedding notification: %s', e)


def canary_failure_message(error):
    """Log text for a check that could not run: what it was, and what to do.

    The connector logs the failed request as an error on its own; without
    this the line reads like a failed recording.
    """
    detail = str(error)
    message = (
        f"Voice embedding check could not run: {detail}. This came from the "
        f"voice embedding check, which sends the bundled test clip "
        f"{CANARY_FILENAME} to the transcription service; no recording was "
        f"affected and nothing was changed."
    )
    lowered = detail.lower()
    if 'invalid model size' in lowered or ("model" in lowered and "''" in detail):
        message += (
            " The service has no default model: set PRELOAD_MODEL (or "
            "DEFAULT_MODEL on whisperx-asr-service 0.4.2 and later) on the ASR "
            "service, or choose a default transcription model in Speakr's admin "
            "settings."
        )
    return message


def _spaces(fn):
    """Run a voice-space update without ever failing the check."""
    try:
        from src.services import voice_profiles
        return fn(voice_profiles)
    except Exception as e:
        logger.warning(f"Voice embedding space bookkeeping failed: {e}")
        return None


def _initialize_spaces(vp, reference):
    from src.models import VoiceEmbeddingSpace
    if VoiceEmbeddingSpace.query.count() == 0:
        vp.register_space(reference['embedding'], reference.get('backend_fingerprint'))


def spaces_status():
    """Voice spaces for the admin card: which is current, samples, threshold."""
    try:
        from src.models import SpeakerVoiceSample, VoiceEmbeddingSpace
        from src.services import voice_profiles as vp
        current, legacy = vp.current_space_id(), vp.legacy_space_id()
        counts = {}
        for (space_id,) in SpeakerVoiceSample.query.with_entities(SpeakerVoiceSample.space_id).all():
            key = vp.effective_space(space_id)
            counts[key] = counts.get(key, 0) + 1
        # Profiles from before samples existed count as one sample each in
        # the legacy space until their next update writes them out.
        from src.models import Speaker
        with_samples = {sid for (sid,) in SpeakerVoiceSample.query.with_entities(SpeakerVoiceSample.speaker_id).distinct()}
        legacy_only = sum(1 for (sid,) in Speaker.query.filter(Speaker.average_embedding.isnot(None))
                          .with_entities(Speaker.id).all() if sid not in with_samples)
        if legacy_only and legacy is not None:
            counts[legacy] = counts.get(legacy, 0) + legacy_only
        out = []
        for space in VoiceEmbeddingSpace.query.order_by(VoiceEmbeddingSpace.id).all():
            item = space.to_dict()
            item['current'] = space.id == current
            item['legacy'] = space.id == legacy
            item['sample_count'] = counts.get(space.id, 0)
            item['calibrated_threshold'] = vp.calibrated_threshold(space.id)
            out.append(item)
        return out
    except Exception as e:
        logger.debug(f"Could not read voice spaces: {e}")
        return []


def rebaseline(app):
    """Accept the current backend as correct and discard the old reference.

    Called after the user rebuilds their voice profiles, or when they know the
    change was intentional.
    """
    from src.database import db
    from src.models import SystemSetting
    setting = SystemSetting.query.filter_by(key=SETTING_KEY).first()
    if setting:
        db.session.delete(setting)
        db.session.commit()
    return check_voice_embeddings(app, force=True)