Skip to content

Speakr is a personal, self-hosted web application designed for transcribing audio recordings

Python

feat: detect when the transcription backend changes its voice embedding model and warn until it is restored or profiles are rebuilt (#380)

Murtaza Nasir50cabe2297c4
diff --git a/config/env.asr.example b/config/env.asr.example
index 5f7e8207..82b5450e 100644
--- a/config/env.asr.example
+++ b/config/env.asr.example
@@ -428,6 +428,16 @@ MAX_CONCURRENT_UPLOADS=3
# Transcription queue workers (for ASR processing, default: 2)
JOB_QUEUE_WORKERS=2
+# Voice embedding compatibility check
+# Voice profiles are only comparable with embeddings from the model that made
+# them. If this endpoint starts returning embeddings from a different model,
+# existing profiles silently stop matching. Speakr detects that by sending a
+# short bundled clip and comparing the result with a stored reference, when the
+# transcription configuration changes and on demand from the admin area.
+# Set to true to disable it. Costs one short transcription per run.
+# DISABLE_VOICE_EMBEDDING_CHECK=false
+
+
# Summary queue workers (for LLM summarization, default: 2)
SUMMARY_QUEUE_WORKERS=2
diff --git a/docs/features.md b/docs/features.md
index 9afc406d..c57e56bc 100644
--- a/docs/features.md
+++ b/docs/features.md
@@ -125,7 +125,13 @@ Implement intelligent data lifecycle management with flexible retention policies
Build and maintain a library of speaker profiles that persist across recordings. [Identify speakers](user-guide/transcripts.md#speaker-identification) after each transcription to build your library. Once identified, speakers are remembered and suggested in future recordings. The system tracks usage statistics, showing how often each speaker appears and when they were last identified. Bulk management tools help maintain a clean, organized speaker library.
-**Voice Profile Recognition**: The system builds voice recognition profiles from speaker identifications across multiple recordings. These profiles use 256-dimensional voice embeddings to suggest speaker matches when processing new recordings, making speaker identification faster and more consistent. The voice matching system calculates similarity scores to help identify speakers across recordings, even when audio quality varies. Voice profiles are automatically updated as more recordings are processed, improving recognition accuracy over time.
+**Voice Profile Recognition**: The system builds voice recognition profiles from speaker identifications across multiple recordings. These profiles use voice embeddings supplied by the transcription backend (256-dimensional with WhisperX, though nothing in Speakr requires that size) to suggest speaker matches when processing new recordings, making speaker identification faster and more consistent. The voice matching system calculates similarity scores to help identify speakers across recordings, even when audio quality varies. Voice profiles are automatically updated as more recordings are processed, improving recognition accuracy over time.
+
+**Embedding Model Changes**: Voice profiles are only comparable with embeddings from the model that produced them. If the transcription backend starts returning embeddings from a different model, whether because the endpoint was repointed or because the service upgraded itself, existing profiles stop matching anything. On its own this failure is silent: matching simply returns nothing.
+
+Speakr detects it by sending a short fixed clip to the backend and comparing the embedding it gets back against the one recorded when voice matching first worked. The check runs when the transcription configuration changes, and on demand from the admin area. When it detects a change, Speakr reports it and keeps the original reference, so restoring the previous backend clears the warning by itself. If the change was intended, rebuild the affected voice profiles and then re-baseline to accept the new model. A backend that is simply unreachable is reported as a failed check rather than a changed model.
+
+Set `DISABLE_VOICE_EMBEDDING_CHECK=true` to turn the check off. The clip costs one short transcription each time it runs.
**Auto Speaker Labeling**: Enable automatic speaker identification in your [account settings](user-guide/settings.md#processing-preferences). When enabled, new recordings are automatically analyzed against your speaker profiles, and matching speakers are labeled without manual intervention. Configure the confidence threshold to balance between more matches (low threshold) and higher accuracy (high threshold). This feature requires WhisperX ASR with speaker embeddings enabled.
diff --git a/src/api/admin.py b/src/api/admin.py
index caf3abc4..13eef2e3 100644
--- a/src/api/admin.py
+++ b/src/api/admin.py
@@ -1576,3 +1576,54 @@ def admin_inquire_status():
# --- Group Management API (Admin Only) ---
+
+
+# --- Voice embedding compatibility (#380) ---------------------------------
+
+@admin_bp.route('/admin/voice-embeddings/status', methods=['GET'])
+@login_required
+def admin_voice_embedding_status():
+ """Report whether the backend still produces compatible voice embeddings.
+
+ Reads the stored verdict only. Probing costs a real transcription, so it
+ is never done on a page load.
+ """
+ if not current_user.is_admin:
+ return jsonify({'error': 'Unauthorized'}), 403
+
+ from src.services.voice_embedding_check import get_status
+ return jsonify(get_status(current_app))
+
+
+@admin_bp.route('/admin/voice-embeddings/check', methods=['POST'])
+@login_required
+def admin_voice_embedding_check():
+ """Re-run the check now, regardless of whether the backend looks changed."""
+ if not current_user.is_admin:
+ return jsonify({'error': 'Unauthorized'}), 403
+
+ from src.services.voice_embedding_check import check_voice_embeddings
+ result = check_voice_embeddings(current_app, force=True)
+ current_app.config['VOICE_EMBEDDING_STATUS'] = result
+ return jsonify(result)
+
+
+@admin_bp.route('/admin/voice-embeddings/rebaseline', methods=['POST'])
+@login_required
+def admin_voice_embedding_rebaseline():
+ """Accept the current backend as the reference.
+
+ For when the change was intentional and the voice profiles have been
+ rebuilt against the new model, or are being abandoned. This discards the
+ embedding the existing profiles were built against, so it cannot be undone
+ by restoring the old backend.
+ """
+ if not current_user.is_admin:
+ return jsonify({'error': 'Unauthorized'}), 403
+
+ from src.services.voice_embedding_check import rebaseline
+ result = rebaseline(current_app)
+ current_app.config['VOICE_EMBEDDING_STATUS'] = result
+ current_app.logger.info(
+ "Voice embedding reference re-baselined by admin user %s", current_user.id)
+ return jsonify(result)
diff --git a/src/assets/README.md b/src/assets/README.md
new file mode 100644
index 00000000..b82b9623
--- /dev/null
+++ b/src/assets/README.md
@@ -0,0 +1,27 @@
+# Bundled assets
+
+## voice_canary_v1.wav
+
+A fixed 12 second clip used to detect that the transcription backend's voice
+embedding model has changed. See `src/services/voice_embedding_check.py`.
+
+It is synthetic on purpose. Synthesised speech carries no licensing question,
+is regenerable from the command below rather than being an opaque blob, and
+puts no real person's voice in a public repository, which matters more than
+usual for a file whose whole purpose is voice embedding.
+
+Regenerating it produces a DIFFERENT file, and any difference invalidates every
+stored reference embedding (the same voice saying the same words, merely
+truncated, measures 0.52 against the original). So do not regenerate it in
+place. To change the clip, add `voice_canary_v2.wav` and bump
+`CANARY_CLIP_VERSION`, which re-baselines every instance on upgrade instead of
+warning them all.
+
+```bash
+docker run --rm -v "$PWD":/out debian:bookworm-slim bash -c '
+apt-get update -qq && apt-get install -y -qq espeak-ng ffmpeg
+TEXT="This is a Speakr self test recording. It is used only to check that voice embeddings from the transcription service are still compatible with the voice profiles already stored. Nothing here is transcribed or saved."
+espeak-ng -v en-us -s 145 -p 45 -w /tmp/raw.wav "$TEXT"
+ffmpeg -i /tmp/raw.wav -ac 1 -ar 16000 -sample_fmt s16 \
+ -af "loudnorm=I=-18:TP=-2:LRA=7,apad=pad_dur=0.3" -t 12 /out/voice_canary_v1.wav'
+```
diff --git a/src/assets/voice_canary_v1.wav b/src/assets/voice_canary_v1.wav
new file mode 100644
index 00000000..819cb4d7
Binary files /dev/null and b/src/assets/voice_canary_v1.wav differ
diff --git a/src/config/startup.py b/src/config/startup.py
index 255b8e5b..7131a395 100644
--- a/src/config/startup.py
+++ b/src/config/startup.py
@@ -34,6 +34,7 @@ FILE_MONITOR_OWNER_LOCK = 'speakr.file_monitor_owner'
AUTO_DELETION_OWNER_LOCK = 'speakr.auto_deletion_owner'
SESSION_CLEANUP_OWNER_LOCK = 'speakr.session_cleanup_owner'
WEBHOOK_DISPATCHER_OWNER_LOCK = 'speakr.webhook_dispatcher_owner'
+VOICE_EMBEDDING_CHECK_OWNER_LOCK = 'speakr.voice_embedding_check_owner'
# Every lock name this module elects on. The test suite asserts that each
# background starter in run_startup_tasks() goes through one of these.
@@ -43,6 +44,7 @@ SINGLE_INSTANCE_LOCKS = (
AUTO_DELETION_OWNER_LOCK,
SESSION_CLEANUP_OWNER_LOCK,
WEBHOOK_DISPATCHER_OWNER_LOCK,
+ VOICE_EMBEDDING_CHECK_OWNER_LOCK,
)
@@ -265,6 +267,32 @@ def initialize_recording_session_cleanup(app):
app.logger.info("✅ Recording-session cleanup scheduler initialized")
+def initialize_voice_embedding_check(app):
+ """Check the voice embedding backend on a background thread (#380).
+
+ Off the boot path on purpose: it makes a real transcription request, and a
+ slow or unreachable ASR service must not delay the app coming up. A
+ failure to check is logged and leaves the previous verdict alone; a
+ backend that is down is not a backend that changed.
+ """
+ import threading
+
+ if os.environ.get('DISABLE_VOICE_EMBEDDING_CHECK', 'false').lower() == 'true':
+ app.logger.info("Voice embedding check disabled (DISABLE_VOICE_EMBEDDING_CHECK=true)")
+ return
+
+ def _run():
+ from src.services.voice_embedding_check import check_voice_embeddings
+ try:
+ with app.app_context():
+ result = check_voice_embeddings(app)
+ app.config['VOICE_EMBEDDING_STATUS'] = result
+ except Exception as e:
+ app.logger.warning(f"Voice embedding check failed to run: {e}")
+
+ threading.Thread(target=_run, daemon=True, name="VoiceEmbeddingCheck").start()
+
+
def run_startup_tasks(app):
"""Run all startup tasks that need to happen after app creation."""
from src.models import SystemSetting
@@ -324,6 +352,13 @@ def run_startup_tasks(app):
app, WEBHOOK_DISPATCHER_OWNER_LOCK,
lambda: start_dispatcher_thread(app), 'webhook dispatcher')
+ # Checks that the backend still returns voice embeddings compatible
+ # with the stored profiles (#380). Runs on a thread because it makes a
+ # real transcription request and must not hold up the boot.
+ start_single_instance(
+ app, VOICE_EMBEDDING_CHECK_OWNER_LOCK,
+ lambda: initialize_voice_embedding_check(app), 'voice embedding check')
+
# Not elected: this only creates a directory and logs. No thread, no
# sweep, nothing to duplicate. If it ever grows a background task it
# needs to move up into the block above.
diff --git a/src/services/speaker_embedding_matcher.py b/src/services/speaker_embedding_matcher.py
index 9af2b643..3730fbe7 100644
--- a/src/services/speaker_embedding_matcher.py
+++ b/src/services/speaker_embedding_matcher.py
@@ -13,6 +13,7 @@ Uses 256-dimensional embeddings from WhisperX diarization.
"""
import json
+import logging
import numpy as np
from datetime import datetime
try:
@@ -22,6 +23,8 @@ except ImportError:
from src.database import db
from src.models import Speaker
+logger = logging.getLogger(__name__)
+
def serialize_embedding(embedding_array):
"""
@@ -90,11 +93,27 @@ def find_matching_speakers(target_embedding, user_id, threshold=0.70):
if not speakers:
return []
+ target_dims = len(target_embedding) if target_embedding is not None else 0
+
matches = []
+ dimension_mismatches = 0
+ mismatched_dims = set()
for speaker in speakers:
try:
# Deserialize and compare
speaker_emb = deserialize_embedding(speaker.average_embedding)
+
+ # Checked explicitly rather than left to raise below. A stored
+ # profile of a different length is not a corrupt profile: it is a
+ # profile built by a different embedding model, and treating it as
+ # corruption is what made a backend change look like voice
+ # matching quietly ceasing to work, with nothing in the log to say
+ # why (#380).
+ if target_dims and len(speaker_emb) != target_dims:
+ dimension_mismatches += 1
+ mismatched_dims.add(len(speaker_emb))
+ continue
+
similarity = calculate_similarity(target_embedding, speaker_emb)
if similarity >= threshold:
@@ -106,9 +125,25 @@ def find_matching_speakers(target_embedding, user_id, threshold=0.70):
'embedding_count': speaker.embedding_count or 0
})
except Exception as e:
- # Skip speakers with corrupted embeddings
+ # Genuinely unreadable stored bytes. Logged rather than swallowed,
+ # so a systematic problem is visible instead of showing up only as
+ # an empty match list.
+ logger.warning(
+ "Skipping speaker %s: its stored embedding could not be compared: %s",
+ getattr(speaker, 'id', '?'), e)
continue
+ if dimension_mismatches:
+ # One line per call rather than per speaker, so this stays readable
+ # when a whole library is affected, which is the usual case.
+ sizes = ', '.join(str(d) for d in sorted(mismatched_dims))
+ logger.error(
+ "%d of %d stored voice profiles have %s dimensions and cannot be compared "
+ "with the %d-dimensional embeddings this backend now returns. Voice matching "
+ "will find nothing until the previous transcription backend is restored or "
+ "the profiles are rebuilt.",
+ dimension_mismatches, len(speakers), sizes, target_dims)
+
# Sort by similarity (highest first)
return sorted(matches, key=lambda x: x['similarity'], reverse=True)
diff --git a/src/services/voice_embedding_check.py b/src/services/voice_embedding_check.py
new file mode 100644
index 00000000..40d75070
--- /dev/null
+++ b/src/services/voice_embedding_check.py
@@ -0,0 +1,299 @@
+"""Detect that the transcription backend's voice embedding model has changed.
+
+Voice profiles are stored as raw vectors with no record of what produced them.
+If the backend starts returning embeddings from a different model, those
+vectors live in a different space and stop matching, and the failure is
+completely silent: `find_matching_speakers()` compares each stored profile in
+turn and a mismatched pair raises, which it treats as a corrupt profile and
+skips. The user sees voice matching quietly stop working, with nothing in the
+logs to explain it (#380).
+
+Dimension alone does not detect this. Two different models can both emit 256
+dimensions, and nothing in Speakr enforces a dimension anyway: serialize is
+`np.array(...).tobytes()`, deserialize is `np.frombuffer(...)`, and similarity
+reshapes to (1, -1). All three work at any length.
+
+The backend cannot be asked either. An ASR response carries the embeddings and
+nothing about the model that made them; the connector knows which
+*transcription* model it requested and never learns which *embedding* model the
+service chose.
+
+So Speakr asks the only question that has a reliable answer: it sends a fixed
+clip and compares the embedding that comes back with the one stored when the
+reference was taken. Measured against a whisperx-asr-service backend:
+
+ same clip, same model, repeated cosine 1.000000 (bit-identical)
+ different voice, same model cosine 0.270
+ same voice, same words, 5s not 12s cosine 0.516
+
+A different model puts the clip in an unrelated space, so the separation is
+wide and the threshold below has a lot of room. The last row is the reason the
+clip is a committed fixture rather than anything generated: this comparison is
+sensitive to the exact audio, so the reference is only meaningful against
+byte-identical input.
+"""
+
+import hashlib
+import json
+import logging
+import math
+import os
+from datetime import datetime
+
+logger = logging.getLogger(__name__)
+
+# Bump this when the clip changes. A new version re-baselines every instance on
+# upgrade rather than warning all of them, because a different clip cannot be
+# compared with a reference taken from the old one.
+CANARY_CLIP_VERSION = 'v1'
+CANARY_FILENAME = f'voice_canary_{CANARY_CLIP_VERSION}.wav'
+CANARY_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ 'assets', CANARY_FILENAME)
+
+# Same input through the same model measured exactly 1.0 here, and the nearest
+# wrong answer measured 0.52. 0.98 leaves room for a backend whose GPU kernels
+# are not bit-deterministic (that kind of noise stays above 0.9999) while
+# staying far above anything that is genuinely a different model.
+SIMILARITY_THRESHOLD = 0.98
+
+SETTING_KEY = 'voice_embedding_reference'
+
+STATUS_OK = 'ok'
+STATUS_CHANGED = 'changed'
+STATUS_UNKNOWN = 'unknown'
+
+
+class CanaryUnavailable(Exception):
+ """The probe could not be run, which is not the same as a mismatch."""
+
+
+def cosine_similarity(a, b):
+ """Plain cosine, no numpy, so this works without the embeddings extra."""
+ if not a or not b or len(a) != len(b):
+ return None
+ dot = sum(x * y for x, y in zip(a, b))
+ na = math.sqrt(sum(x * x for x in a))
+ nb = math.sqrt(sum(y * y for y in b))
+ if not na or not nb:
+ return None
+ return dot / (na * nb)
+
+
+def backend_fingerprint():
+ """Identify the configured backend well enough to notice reconfiguration.
+
+ This is a TRIGGER, never the alarm. Renaming a host changes it without
+ changing a single embedding, and a service that quietly upgrades its own
+ model changes every embedding without touching it. Either way the probe
+ decides; this only says when it is worth running one.
+ """
+ try:
+ from src.services.transcription.registry import get_registry
+ registry = get_registry()
+ connector = registry.get_active_connector()
+ parts = [
+ registry.get_active_connector_name() or '',
+ str(getattr(connector, 'base_url', '') or ''),
+ str(getattr(connector, 'model', '') or ''),
+ ]
+ except Exception as e:
+ logger.debug(f"Could not fingerprint the transcription backend: {e}")
+ parts = ['unknown']
+ return hashlib.sha256('|'.join(parts).encode()).hexdigest()[:16]
+
+
+def embeddings_supported():
+ """Whether the active connector returns speaker embeddings at all."""
+ try:
+ from src.services.transcription.base import TranscriptionCapability
+ from src.services.transcription.registry import get_registry
+ connector = get_registry().get_active_connector()
+ return bool(connector.supports(TranscriptionCapability.SPEAKER_EMBEDDINGS))
+ except Exception as e:
+ logger.debug(f"Could not determine speaker-embedding support: {e}")
+ return False
+
+
+def probe_backend():
+ """Send the canary through the active connector and return its embedding.
+
+ Raises CanaryUnavailable when the probe cannot be completed. That is
+ deliberately distinct from a mismatch: a backend that is merely down must
+ never be reported as a changed model.
+ """
+ from src.services.transcription.base import TranscriptionRequest
+ from src.services.transcription.registry import get_registry
+
+ if not os.path.exists(CANARY_PATH):
+ raise CanaryUnavailable(f'canary clip missing at {CANARY_PATH}')
+
+ try:
+ connector = get_registry().get_active_connector()
+ except Exception as e:
+ raise CanaryUnavailable(f'no active transcription connector: {e}')
+
+ try:
+ with open(CANARY_PATH, 'rb') as f:
+ response = connector.transcribe(TranscriptionRequest(
+ audio_file=f,
+ filename=CANARY_FILENAME,
+ mime_type='audio/wav',
+ language='en',
+ diarize=True,
+ min_speakers=1,
+ max_speakers=1,
+ ))
+ except Exception as e:
+ raise CanaryUnavailable(f'transcription of the canary failed: {e}')
+
+ embeddings = getattr(response, 'speaker_embeddings', None) or {}
+ if len(embeddings) != 1:
+ # More than one speaker means the backend's diarizer split a
+ # single-voice clip, so the result is not comparable with a reference
+ # taken from one segment.
+ raise CanaryUnavailable(
+ f'expected exactly one speaker in the canary, got {len(embeddings)}')
+
+ vector = list(embeddings.values())[0]
+ if not vector:
+ raise CanaryUnavailable('backend returned an empty embedding')
+ return [float(x) for x in vector]
+
+
+def load_reference():
+ from src.models import SystemSetting
+ raw = SystemSetting.get_setting(SETTING_KEY, None)
+ if not raw:
+ return None
+ try:
+ return json.loads(raw)
+ except (TypeError, ValueError):
+ logger.warning('Stored voice embedding reference is not valid JSON; ignoring it')
+ return None
+
+
+def save_reference(data):
+ from src.models import SystemSetting
+ # set_setting commits for us.
+ SystemSetting.set_setting(
+ SETTING_KEY, json.dumps(data), setting_type='string',
+ description='Reference embedding of the bundled canary clip, used to detect '
+ 'that the transcription backend changed its voice embedding model')
+
+
+def get_status(app=None):
+ """What the admin UI shows. Never runs a probe."""
+ reference = load_reference() or {}
+ return {
+ 'status': reference.get('status', STATUS_UNKNOWN),
+ 'dimension': reference.get('dimension'),
+ 'clip_version': reference.get('clip_version'),
+ 'checked_at': reference.get('checked_at'),
+ 'similarity': reference.get('last_similarity'),
+ 'detail': reference.get('detail'),
+ 'supported': embeddings_supported(),
+ }
+
+
+def check_voice_embeddings(app, force=False):
+ """Probe the backend and compare against the stored reference.
+
+ Returns the same dict shape as get_status(). Never raises: a failure to
+ check is reported, not thrown, because this runs on a startup thread.
+ """
+ if not embeddings_supported():
+ logger.debug('Active connector returns no speaker embeddings; skipping the canary')
+ return {'status': STATUS_UNKNOWN, 'supported': False,
+ 'detail': 'the active connector does not return speaker embeddings'}
+
+ reference = load_reference()
+ fingerprint = backend_fingerprint()
+
+ # A stored reference from a different clip cannot be compared with this
+ # one, so treat it as no reference and re-baseline.
+ if reference and reference.get('clip_version') != CANARY_CLIP_VERSION:
+ logger.info('Canary clip changed since the reference was taken; re-baselining')
+ reference = None
+
+ if reference and not force and reference.get('backend_fingerprint') == fingerprint:
+ # Nothing about the configuration moved. The probe costs a real
+ # transcription, so it is not run on every boot for no reason.
+ logger.debug('Transcription backend unchanged since the last voice embedding check')
+ return get_status(app)
+
+ try:
+ vector = probe_backend()
+ except CanaryUnavailable as e:
+ logger.warning(f'Voice embedding check could not run: {e}')
+ # Leave any existing reference and its status untouched. A backend
+ # that is down is not a backend that changed.
+ status = get_status(app)
+ status['detail'] = f'check could not run: {e}'
+ return status
+
+ now = datetime.utcnow().isoformat()
+
+ if reference is None:
+ save_reference({
+ 'clip_version': CANARY_CLIP_VERSION,
+ 'dimension': len(vector),
+ 'embedding': vector,
+ 'backend_fingerprint': fingerprint,
+ 'checked_at': now,
+ 'status': STATUS_OK,
+ 'last_similarity': 1.0,
+ 'detail': 'baseline recorded',
+ })
+ logger.info(f'Recorded a voice embedding baseline ({len(vector)} dimensions)')
+ return get_status(app)
+
+ stored = reference.get('embedding') or []
+ if len(stored) != len(vector):
+ detail = (f'the backend now returns {len(vector)}-dimensional embeddings, '
+ f'and stored voice profiles are {len(stored)}-dimensional')
+ similarity = None
+ else:
+ similarity = cosine_similarity(stored, vector)
+ detail = None
+
+ changed = similarity is None or similarity < SIMILARITY_THRESHOLD
+ if changed and detail is None:
+ detail = (f'the same clip now embeds differently (similarity '
+ f'{similarity:.4f}, expected at least {SIMILARITY_THRESHOLD})')
+
+ reference.update({
+ 'backend_fingerprint': fingerprint,
+ 'checked_at': now,
+ 'last_similarity': similarity,
+ 'status': STATUS_CHANGED if changed else STATUS_OK,
+ 'detail': detail if changed else None,
+ })
+ # The reference embedding itself is NOT replaced on a mismatch. It is what
+ # the existing voice profiles were built against, so it stays until the
+ # user rebuilds them or restores the old backend.
+ save_reference(reference)
+
+ if changed:
+ logger.error(
+ 'Voice embedding model appears to have changed: %s. Existing voice profiles '
+ 'will not match new recordings until either the previous transcription '
+ 'backend is restored or the profiles are rebuilt.', detail)
+ else:
+ logger.info('Voice embedding check passed (similarity %.6f)', similarity)
+
+ return get_status(app)
+
+
+def rebaseline(app):
+ """Accept the current backend as correct and discard the old reference.
+
+ Called after the user rebuilds their voice profiles, or when they know the
+ change was intentional.
+ """
+ from src.database import db
+ from src.models import SystemSetting
+ setting = SystemSetting.query.filter_by(key=SETTING_KEY).first()
+ if setting:
+ db.session.delete(setting)
+ db.session.commit()
+ return check_voice_embeddings(app, force=True)
diff --git a/tests/test_voice_embedding_check.py b/tests/test_voice_embedding_check.py
new file mode 100644
index 00000000..919036a7
--- /dev/null
+++ b/tests/test_voice_embedding_check.py
@@ -0,0 +1,425 @@
+"""Detecting that the transcription backend changed its voice embedding model (#380).
+
+The failure being guarded against is silent, so most of these tests are about
+NOT crying wolf. A backend that is merely unreachable, or a Speakr upgrade that
+ships a new canary clip, must not be reported as a changed model: a warning
+nobody can act on trains people to ignore the one that matters.
+
+The numbers here come from measuring a real whisperx-asr-service backend:
+the same clip re-embedded scores 1.0 and is bit-identical, a different voice
+through the same model scores 0.27, and the same voice truncated from 12s to 5s
+scores 0.52. The threshold sits at 0.98.
+"""
+
+import json
+from unittest.mock import patch
+
+import pytest
+
+from src.app import app
+from src.database import db
+from src.models import SystemSetting
+from src.services import voice_embedding_check as vec
+
+
+REF = [0.10, -0.20, 0.30, 0.40] * 64 # 256 dims, stands in for the baseline
+SAME = list(REF)
+SCALED = [x * 3 for x in REF] # same direction, different magnitude
+DIFFERENT = [((i * 37) % 19) / 19 - 0.5 for i in range(256)]
+SHORTER = [0.1] * 192 # what a 192-dim model would return
+
+
[email protected](autouse=True)
+def clean_reference():
+ """Each test starts with no stored baseline."""
+ with app.app_context():
+ SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
+ db.session.commit()
+ yield
+ with app.app_context():
+ SystemSetting.query.filter_by(key=vec.SETTING_KEY).delete()
+ db.session.commit()
+
+
+def _supported():
+ return patch.object(vec, 'embeddings_supported', return_value=True)
+
+
+def _baseline(vector=REF, fingerprint='backend-a'):
+ """Record a baseline the way a first startup would."""
+ with _supported(), \
+ patch.object(vec, 'probe_backend', return_value=vector), \
+ patch.object(vec, 'backend_fingerprint', return_value=fingerprint):
+ return vec.check_voice_embeddings(app)
+
+
+# --- the core claim ------------------------------------------------------
+
+def test_a_first_run_records_a_baseline_rather_than_warning():
+ with app.app_context():
+ result = _baseline()
+ assert result['status'] == vec.STATUS_OK
+ assert result['dimension'] == 256
+
+
+def test_the_same_embedding_is_not_reported_as_a_change():
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SAME), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_OK
+ assert result['similarity'] == pytest.approx(1.0)
+
+
+def test_a_different_embedding_is_reported_as_a_change():
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_CHANGED
+ assert result['similarity'] < vec.SIMILARITY_THRESHOLD
+ assert 'rebuil' in result['detail'] or 'embeds differently' in result['detail']
+
+
+def test_magnitude_alone_is_not_a_change():
+ """Cosine ignores scale, and it should: a rescaled vector is the same
+ direction in the same space, so voice matching still works."""
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SCALED), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_OK
+
+
+def test_a_dimension_change_is_reported_as_dimensions_not_as_similarity():
+ """192-dim vs 256-dim is the case from the issue, and cosine cannot even
+ be computed for it, so the message has to say so."""
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SHORTER), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_CHANGED
+ assert result['similarity'] is None
+ assert '192' in result['detail'] and '256' in result['detail']
+
+
+# --- not crying wolf -----------------------------------------------------
+
+def test_an_unreachable_backend_is_not_reported_as_a_changed_model():
+ """The most important false positive to avoid. A down ASR service is a
+ different problem with a different fix, and telling the user their model
+ changed would send them to rebuild profiles for nothing."""
+ with app.app_context():
+ _baseline()
+ with _supported(), \
+ patch.object(vec, 'probe_backend',
+ side_effect=vec.CanaryUnavailable('connection refused')), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_OK, 'a down backend must not read as a changed model'
+ assert 'could not run' in result['detail']
+
+
+def test_a_canary_that_diarizes_into_several_speakers_is_unavailable_not_changed():
+ """A backend whose VAD splits the single-voice clip gives a result that is
+ not comparable, which is missing data rather than evidence of a change."""
+ with app.app_context():
+ _baseline()
+
+ class _Resp:
+ speaker_embeddings = {'SPEAKER_00': REF, 'SPEAKER_01': DIFFERENT}
+
+ class _Conn:
+ def transcribe(self, request):
+ return _Resp()
+
+ with _supported(), \
+ patch('src.services.transcription.registry.get_registry') as reg:
+ reg.return_value.get_active_connector.return_value = _Conn()
+ with pytest.raises(vec.CanaryUnavailable) as e:
+ vec.probe_backend()
+ assert 'one speaker' in str(e.value)
+
+
+def test_a_new_canary_clip_rebaselines_instead_of_warning_everyone():
+ """Shipping a different clip must not light up every instance on upgrade.
+ A reference taken from the old clip is not comparable with the new one."""
+ with app.app_context():
+ _baseline()
+ stored = json.loads(SystemSetting.get_setting(vec.SETTING_KEY))
+ stored['clip_version'] = 'v0'
+ SystemSetting.set_setting(vec.SETTING_KEY, json.dumps(stored))
+
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
+ result = vec.check_voice_embeddings(app)
+
+ assert result['status'] == vec.STATUS_OK
+ assert result['clip_version'] == vec.CANARY_CLIP_VERSION
+
+
+def test_a_connector_without_embeddings_is_unknown_not_broken():
+ with app.app_context():
+ with patch.object(vec, 'embeddings_supported', return_value=False):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_UNKNOWN
+ assert result['supported'] is False
+
+
+# --- cost ---------------------------------------------------------------
+
+def test_an_unchanged_backend_is_not_re_probed():
+ """The probe is a real transcription. Running it every boot would put a
+ pointless job through the user's ASR service five times per container."""
+ with app.app_context():
+ _baseline(fingerprint='backend-a')
+ with _supported(), patch.object(vec, 'probe_backend') as probe, \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
+ vec.check_voice_embeddings(app)
+ probe.assert_not_called()
+
+
+def test_force_re_probes_even_when_nothing_changed():
+ with app.app_context():
+ _baseline(fingerprint='backend-a')
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SAME) as probe, \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
+ vec.check_voice_embeddings(app, force=True)
+ probe.assert_called_once()
+
+
+# --- recovery ------------------------------------------------------------
+
+def test_the_baseline_survives_a_mismatch_so_the_backend_can_be_restored():
+ """If a mismatch overwrote the reference, the warning would clear itself on
+ the next check and restoring the old backend would then look like a second
+ change. The reference is what the stored profiles were built against."""
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ vec.check_voice_embeddings(app)
+ stored = vec.load_reference()
+ assert stored['embedding'] == REF, 'the reference was replaced by the mismatching one'
+
+
+def test_restoring_the_original_backend_clears_the_warning_by_itself():
+ """The user should not have to click anything to resolve a change they
+ undid."""
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ assert vec.check_voice_embeddings(app)['status'] == vec.STATUS_CHANGED
+
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SAME), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
+ result = vec.check_voice_embeddings(app)
+ assert result['status'] == vec.STATUS_OK
+ assert result['detail'] is None
+
+
+def test_rebaselining_accepts_the_new_backend():
+ """The other resolution: the change was intended, profiles were rebuilt."""
+ with app.app_context():
+ _baseline()
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ vec.check_voice_embeddings(app)
+
+ with _supported(), patch.object(vec, 'probe_backend', return_value=DIFFERENT), \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-b'):
+ result = vec.rebaseline(app)
+ assert result['status'] == vec.STATUS_OK
+ with app.app_context():
+ assert vec.load_reference()['embedding'] == DIFFERENT
+
+
+# --- the clip itself -----------------------------------------------------
+
+def test_the_canary_clip_ships_and_is_the_expected_audio():
+ """A missing or re-encoded clip silently invalidates every reference, so
+ its identity is pinned here rather than assumed."""
+ import hashlib
+ import os
+ import wave
+
+ assert os.path.exists(vec.CANARY_PATH), f'canary clip missing at {vec.CANARY_PATH}'
+
+ with wave.open(vec.CANARY_PATH, 'rb') as w:
+ assert w.getnchannels() == 1, 'must be mono'
+ assert w.getframerate() == 16000, 'must be 16 kHz'
+ assert w.getsampwidth() == 2, 'must be 16-bit PCM'
+ duration = w.getnframes() / w.getframerate()
+ assert 10 <= duration <= 14, f'expected roughly 12s of audio, got {duration:.1f}s'
+
+ digest = hashlib.sha256(open(vec.CANARY_PATH, 'rb').read()).hexdigest()
+ assert digest == '2f4440d771f4206e67d53469002dd0241fd2a19e18db46ba2b79600da51a2741', (
+ 'the canary clip changed. Every stored reference embedding was taken against the '
+ 'old bytes and is now meaningless, so bump CANARY_CLIP_VERSION and ship the new '
+ 'clip under a new filename rather than replacing this one in place.'
+ )
+
+
+def test_cosine_similarity_edge_cases():
+ assert vec.cosine_similarity([1, 0], [1, 0]) == pytest.approx(1.0)
+ assert vec.cosine_similarity([1, 0], [0, 1]) == pytest.approx(0.0)
+ # Lengths that differ cannot be compared; None, not an exception, because
+ # the caller distinguishes "cannot compare" from "compares badly".
+ assert vec.cosine_similarity([1, 0], [1, 0, 0]) is None
+ assert vec.cosine_similarity([0, 0], [1, 0]) is None
+ assert vec.cosine_similarity([], []) is None
+
+
+# --- the matcher's side of it --------------------------------------------
+#
+# The check above tells the admin the model changed. These cover what happens
+# to a user in the meantime, which is where the original silence lived.
+
+def _speaker(user_id, name, vector):
+ from src.models import Speaker
+ from src.services.speaker_embedding_matcher import serialize_embedding
+ return Speaker(user_id=user_id, name=name,
+ average_embedding=serialize_embedding(vector))
+
+
+def speaker_owner():
+ from src.models import User
+ import os
+ with app.app_context():
+ user = User(username=f'vec_owner_{os.getpid()}',
+ email=f'vec_owner_{os.getpid()}@example.com', password='x')
+ db.session.add(user)
+ db.session.commit()
+ uid = user.id
+ yield uid
+ with app.app_context():
+ from src.models import Speaker, User
+ Speaker.query.filter_by(user_id=uid).delete()
+ User.query.filter_by(id=uid).delete()
+ db.session.commit()
+
+
+def test_profiles_of_another_dimension_are_skipped_and_said_so_in_the_log(speaker_owner, caplog):
+ """The original behaviour: a mismatched pair raised inside the loop, the
+ blanket `except Exception: continue` treated it as a corrupt profile, and
+ the user got an empty match list forever with nothing in the log."""
+ from src.services.speaker_embedding_matcher import find_matching_speakers
+
+ with app.app_context():
+ db.session.add(_speaker(speaker_owner, 'Old Profile A', REF))
+ db.session.add(_speaker(speaker_owner, 'Old Profile B', REF))
+ db.session.commit()
+
+ with caplog.at_level('ERROR'):
+ matches = find_matching_speakers(SHORTER, speaker_owner, threshold=0.5)
+
+ assert matches == []
+ logged = ' '.join(r.getMessage() for r in caplog.records)
+ assert '2 of 2' in logged, f'the skip was not reported: {logged!r}'
+ assert '256 dimensions' in logged, f'the stored size must be named: {logged!r}'
+ assert '192-dimensional' in logged, f'the new size must be named: {logged!r}'
+ assert 'rebuilt' in logged or 'restored' in logged, 'the log must say what to do'
+
+
+def test_matching_still_works_when_the_dimensions_agree(speaker_owner):
+ """The guard must not cost normal matching."""
+ from src.services.speaker_embedding_matcher import find_matching_speakers
+
+ with app.app_context():
+ db.session.add(_speaker(speaker_owner, 'Alice', REF))
+ db.session.commit()
+ matches = find_matching_speakers(SAME, speaker_owner, threshold=0.7)
+
+ assert [m['name'] for m in matches] == ['Alice']
+ assert matches[0]['similarity'] == pytest.approx(100.0, abs=0.1)
+
+
+def test_a_genuinely_unreadable_profile_is_skipped_and_logged(speaker_owner, caplog):
+ """Corruption still has to be tolerated, just not silently."""
+ from src.models import Speaker
+ from src.services.speaker_embedding_matcher import find_matching_speakers
+
+ with app.app_context():
+ broken = Speaker(user_id=speaker_owner, name='Corrupt',
+ average_embedding=b'\x01\x02\x03') # not a whole float32
+ db.session.add(broken)
+ db.session.add(_speaker(speaker_owner, 'Alice', REF))
+ db.session.commit()
+
+ with caplog.at_level('WARNING'):
+ matches = find_matching_speakers(SAME, speaker_owner, threshold=0.7)
+
+ assert [m['name'] for m in matches] == ['Alice'], 'one bad profile must not lose the good ones'
+
+
+# --- the admin surface ---------------------------------------------------
+
+def admin_client():
+ """The endpoints are CSRF-protected, which these tests are not about."""
+ import os
+ from src.models import User
+ previous_csrf = app.config.get('WTF_CSRF_ENABLED')
+ app.config['WTF_CSRF_ENABLED'] = False
+ with app.app_context():
+ admin = User(username=f'vec_admin_{os.getpid()}',
+ email=f'vec_admin_{os.getpid()}@example.com',
+ password='x', is_admin=True)
+ plain = User(username=f'vec_plain_{os.getpid()}',
+ email=f'vec_plain_{os.getpid()}@example.com',
+ password='x', is_admin=False)
+ db.session.add_all([admin, plain])
+ db.session.commit()
+ ids = (admin.id, plain.id)
+ yield app.test_client(), ids
+ app.config['WTF_CSRF_ENABLED'] = previous_csrf
+ with app.app_context():
+ from src.models import User
+ User.query.filter(User.id.in_(ids)).delete(synchronize_session=False)
+ db.session.commit()
+
+
+def _login(client, user_id):
+ with client.session_transaction() as sess:
+ sess['_user_id'] = str(user_id)
+ sess['_fresh'] = True
+
+
+def test_status_endpoint_never_probes(admin_client):
+ """A page load must not put a transcription job through the ASR service."""
+ client, (admin_id, _) = admin_client
+ _login(client, admin_id)
+ with patch.object(vec, 'probe_backend') as probe:
+ resp = client.get('/admin/voice-embeddings/status')
+ assert resp.status_code == 200
+ assert 'status' in resp.get_json()
+ probe.assert_not_called()
+
+
+def test_a_non_admin_cannot_read_or_change_the_verdict(admin_client):
+ client, (_, plain_id) = admin_client
+ _login(client, plain_id)
+ for method, path in (('get', '/admin/voice-embeddings/status'),
+ ('post', '/admin/voice-embeddings/check'),
+ ('post', '/admin/voice-embeddings/rebaseline')):
+ resp = getattr(client, method)(path)
+ assert resp.status_code == 403, f'{method.upper()} {path} was not refused'
+
+
+def test_the_check_endpoint_forces_a_probe(admin_client):
+ client, (admin_id, _) = admin_client
+ _login(client, admin_id)
+ with app.app_context():
+ _baseline(fingerprint='backend-a')
+ with _supported(), patch.object(vec, 'probe_backend', return_value=SAME) as probe, \
+ patch.object(vec, 'backend_fingerprint', return_value='backend-a'):
+ resp = client.post('/admin/voice-embeddings/check')
+ assert resp.status_code == 200
+ assert resp.get_json()['status'] == vec.STATUS_OK
+ probe.assert_called_once()